Matchcompiler parse all Match|simpleMatch on the same line (#2993)
* MatchCompiler: Neaten error messages Especially the added space makes it a little more readable. * MatchCompiler: Add spaces between operators * Matchcompiler: Don't bailout if non-const pattern If matchcompiler found a call to Token::Match() or Token::simpleMatch() with an unknown string argument, subsequent calls to Token::Match() or Token::simpleMatch() on the same line would not be processed by matchcompiler. To fix this, keep track of the last index we found a match, and update it accordingly when the line is modified. To avoid having to keep track of if "Match" or "simpleMatch" is the first match we find, just make a loop over them.
This commit is contained in:
Rikard Falkeborn
GitHub
parent
e469436fe1
commit
8dc8aa0459
|
|
@ -92,7 +92,7 @@ class MatchCompiler:
|
||||||
elif tok == '%bool%':
|
elif tok == '%bool%':
|
||||||
return 'tok->isBoolean()'
|
return 'tok->isBoolean()'
|
||||||
elif tok == '%char%':
|
elif tok == '%char%':
|
||||||
return '(tok->tokType()==Token::eChar)'
|
return '(tok->tokType() == Token::eChar)'
|
||||||
elif tok == '%comp%':
|
elif tok == '%comp%':
|
||||||
return 'tok->isComparisonOp()'
|
return 'tok->isComparisonOp()'
|
||||||
elif tok == '%num%':
|
elif tok == '%num%':
|
||||||
|
|
@ -102,24 +102,24 @@ class MatchCompiler:
|
||||||
elif tok == '%op%':
|
elif tok == '%op%':
|
||||||
return 'tok->isOp()'
|
return 'tok->isOp()'
|
||||||
elif tok == '%or%':
|
elif tok == '%or%':
|
||||||
return '(tok->tokType() == Token::eBitOp && tok->str()==MatchCompiler::makeConstString("|") )'
|
return '(tok->tokType() == Token::eBitOp && tok->str() == MatchCompiler::makeConstString("|") )'
|
||||||
elif tok == '%oror%':
|
elif tok == '%oror%':
|
||||||
return '(tok->tokType() == Token::eLogicalOp && tok->str()==MatchCompiler::makeConstString("||"))'
|
return '(tok->tokType() == Token::eLogicalOp && tok->str() == MatchCompiler::makeConstString("||"))'
|
||||||
elif tok == '%str%':
|
elif tok == '%str%':
|
||||||
return '(tok->tokType()==Token::eString)'
|
return '(tok->tokType() == Token::eString)'
|
||||||
elif tok == '%type%':
|
elif tok == '%type%':
|
||||||
return '(tok->isName() && tok->varId()==0U && (tok->str() != "delete" || !tok->isKeyword()))'
|
return '(tok->isName() && tok->varId() == 0U && (tok->str() != "delete" || !tok->isKeyword()))'
|
||||||
elif tok == '%name%':
|
elif tok == '%name%':
|
||||||
return 'tok->isName()'
|
return 'tok->isName()'
|
||||||
elif tok == '%var%':
|
elif tok == '%var%':
|
||||||
return '(tok->varId() != 0)'
|
return '(tok->varId() != 0)'
|
||||||
elif tok == '%varid%':
|
elif tok == '%varid%':
|
||||||
return '(tok->isName() && tok->varId()==varid)'
|
return '(tok->isName() && tok->varId() == varid)'
|
||||||
elif (len(tok) > 2) and (tok[0] == "%"):
|
elif (len(tok) > 2) and (tok[0] == "%"):
|
||||||
print("unhandled:" + tok)
|
print("unhandled:" + tok)
|
||||||
|
|
||||||
return (
|
return (
|
||||||
'(tok->str()==MatchCompiler::makeConstString("' + tok + '"))'
|
'(tok->str() == MatchCompiler::makeConstString("' + tok + '"))'
|
||||||
)
|
)
|
||||||
|
|
||||||
def _compilePattern(self, pattern, nr, varid,
|
def _compilePattern(self, pattern, nr, varid,
|
||||||
|
|
@ -155,7 +155,7 @@ class MatchCompiler:
|
||||||
|
|
||||||
# [abc]
|
# [abc]
|
||||||
if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'):
|
if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'):
|
||||||
ret += ' if (!tok || tok->str().size()!=1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n'
|
ret += ' if (!tok || tok->str().size() != 1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n'
|
||||||
ret += ' ' + returnStatement
|
ret += ' ' + returnStatement
|
||||||
|
|
||||||
# a|b|c
|
# a|b|c
|
||||||
|
|
@ -330,8 +330,8 @@ class MatchCompiler:
|
||||||
# ret += ' std::cout << "tok: " << tok->str();\n'
|
# ret += ' std::cout << "tok: " << tok->str();\n'
|
||||||
# ret += ' if (tok->next())\n'
|
# ret += ' if (tok->next())\n'
|
||||||
# ret += ' std::cout << "tok next: " << tok->next()->str();\n'
|
# ret += ' std::cout << "tok next: " << tok->next()->str();\n'
|
||||||
ret += ' throw InternalError(tok, "Internal error.' +\
|
ret += ' throw InternalError(tok, "Internal error. ' +\
|
||||||
'compiled match returned different result than parsed match: ' + pattern + '");\n'
|
'Compiled match returned different result than parsed match: ' + pattern + '");\n'
|
||||||
ret += ' }\n'
|
ret += ' }\n'
|
||||||
ret += ' return res_compiled_match;\n'
|
ret += ' return res_compiled_match;\n'
|
||||||
ret += '}\n'
|
ret += '}\n'
|
||||||
|
|
@ -380,44 +380,46 @@ class MatchCompiler:
|
||||||
)
|
)
|
||||||
|
|
||||||
def _replaceTokenMatch(self, line, linenr, filename):
|
def _replaceTokenMatch(self, line, linenr, filename):
|
||||||
while True:
|
for func in ('Match', 'simpleMatch'):
|
||||||
is_simplematch = False
|
is_simplematch = func == 'simpleMatch'
|
||||||
pos1 = line.find('Token::Match(')
|
pattern_start = 0
|
||||||
if pos1 == -1:
|
while True:
|
||||||
is_simplematch = True
|
pos1 = line.find('Token::' + func + '(', pattern_start)
|
||||||
pos1 = line.find('Token::simpleMatch(')
|
if pos1 == -1:
|
||||||
if pos1 == -1:
|
break
|
||||||
break
|
|
||||||
|
|
||||||
res = self.parseMatch(line, pos1)
|
res = self.parseMatch(line, pos1)
|
||||||
if res is None:
|
if res is None:
|
||||||
break
|
break
|
||||||
|
|
||||||
# assert that Token::Match has either 2 or 3 arguments
|
# assert that Token::Match has either 2 or 3 arguments
|
||||||
assert(len(res) == 3 or len(res) == 4)
|
assert(len(res) == 3 or len(res) == 4)
|
||||||
|
|
||||||
end_pos = len(res[0])
|
end_pos = len(res[0])
|
||||||
tok = res[1]
|
tok = res[1]
|
||||||
raw_pattern = res[2]
|
raw_pattern = res[2]
|
||||||
varId = None
|
varId = None
|
||||||
if len(res) == 4:
|
if len(res) == 4:
|
||||||
varId = res[3]
|
varId = res[3]
|
||||||
|
|
||||||
res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern)
|
pattern_start = pos1 + end_pos
|
||||||
if res is None:
|
res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern)
|
||||||
if self._showSkipped:
|
if res is None:
|
||||||
print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern)
|
if self._showSkipped:
|
||||||
break # Non-const pattern - bailout
|
print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern)
|
||||||
|
continue # Non-const pattern - bailout
|
||||||
|
|
||||||
pattern = res.group(1)
|
pattern = res.group(1)
|
||||||
line = self._replaceSpecificTokenMatch(
|
orig_len = len(line)
|
||||||
is_simplematch,
|
line = self._replaceSpecificTokenMatch(
|
||||||
line,
|
is_simplematch,
|
||||||
pos1,
|
line,
|
||||||
end_pos,
|
pos1,
|
||||||
pattern,
|
end_pos,
|
||||||
tok,
|
pattern,
|
||||||
varId)
|
tok,
|
||||||
|
varId)
|
||||||
|
pattern_start += len(line) - orig_len
|
||||||
|
|
||||||
return line
|
return line
|
||||||
|
|
||||||
|
|
@ -459,7 +461,7 @@ class MatchCompiler:
|
||||||
# We also need to verify builds in 'release' mode
|
# We also need to verify builds in 'release' mode
|
||||||
ret += ' if (res_parsed_findmatch != res_compiled_findmatch) {\n'
|
ret += ' if (res_parsed_findmatch != res_compiled_findmatch) {\n'
|
||||||
ret += ' throw InternalError(tok, "Internal error. ' +\
|
ret += ' throw InternalError(tok, "Internal error. ' +\
|
||||||
'compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n'
|
'Compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n'
|
||||||
ret += ' }\n'
|
ret += ' }\n'
|
||||||
ret += ' return res_compiled_findmatch;\n'
|
ret += ' return res_compiled_findmatch;\n'
|
||||||
ret += '}\n'
|
ret += '}\n'
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,10 @@ class MatchCompilerTest(unittest.TestCase):
|
||||||
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
self.assertEqual(output, 'if (match1(tok)) {')
|
self.assertEqual(output, 'if (match1(tok)) {')
|
||||||
|
|
||||||
|
input = 'if (Token::simpleMatch(tok, "foobar")) {'
|
||||||
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
|
self.assertEqual(output, 'if (match1(tok)) {')
|
||||||
|
|
||||||
input = 'if (Token::Match(tok->next()->next(), "foobar %type% %num%")) {'
|
input = 'if (Token::Match(tok->next()->next(), "foobar %type% %num%")) {'
|
||||||
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
self.assertEqual(output, 'if (match2(tok->next()->next())) {')
|
self.assertEqual(output, 'if (match2(tok->next()->next())) {')
|
||||||
|
|
@ -60,6 +64,25 @@ class MatchCompilerTest(unittest.TestCase):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
output, 'if (Token::Match(tok, "extern \"C\" " + varname)) {')
|
output, 'if (Token::Match(tok, "extern \"C\" " + varname)) {')
|
||||||
|
|
||||||
|
# test that multiple patterns on the same line are replaced
|
||||||
|
input = 'if (Token::Match(tok, "foo") && Token::Match(tok->next(), "baz")) {'
|
||||||
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
|
self.assertEqual(output, 'if (match4(tok) && match5(tok->next())) {')
|
||||||
|
|
||||||
|
# test that second pattern is replaced, even if there is a bailout on the first pattern
|
||||||
|
input = 'if (Token::Match(tok, foo) && Token::Match(tok->next(), "foobaz")) {'
|
||||||
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
|
self.assertEqual(output, 'if (Token::Match(tok, foo) && match6(tok->next())) {')
|
||||||
|
|
||||||
|
# test mixing Match and simpleMatch on the same line
|
||||||
|
input = 'if (Token::Match(tok, "a") && Token::simpleMatch(tok->next(), "b")) {'
|
||||||
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
|
self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {')
|
||||||
|
|
||||||
|
input = 'if (Token::simpleMatch(tok, "a") && Token::Match(tok->next(), "b")) {'
|
||||||
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
|
self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {')
|
||||||
|
|
||||||
def test_replaceTokenMatchWithVarId(self):
|
def test_replaceTokenMatchWithVarId(self):
|
||||||
input = 'if (Token::Match(tok, "foobar %varid%", 123)) {'
|
input = 'if (Token::Match(tok, "foobar %varid%", 123)) {'
|
||||||
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue