From 8dc8aa04595a793ee1de3d112adca1523bc7cd6f Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Thu, 31 Dec 2020 19:28:06 +0100 Subject: [PATCH] Matchcompiler parse all Match|simpleMatch on the same line (#2993) * MatchCompiler: Neaten error messages Especially the added space makes it a little more readable. * MatchCompiler: Add spaces between operators * Matchcompiler: Don't bailout if non-const pattern If matchcompiler found a call to Token::Match() or Token::simpleMatch() with an unknown string argument, subsequent calls to Token::Match() or Token::simpleMatch() on the same line would not be processed by matchcompiler. To fix this, keep track of the last index we found a match, and update it accordingly when the line is modified. To avoid having to keep track of if "Match" or "simpleMatch" is the first match we find, just make a loop over them. --- tools/matchcompiler.py | 90 +++++++++++++++++++------------------ tools/test_matchcompiler.py | 23 ++++++++++ 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/tools/matchcompiler.py b/tools/matchcompiler.py index 6e333cffe..4b0e01a4e 100755 --- a/tools/matchcompiler.py +++ b/tools/matchcompiler.py @@ -92,7 +92,7 @@ class MatchCompiler: elif tok == '%bool%': return 'tok->isBoolean()' elif tok == '%char%': - return '(tok->tokType()==Token::eChar)' + return '(tok->tokType() == Token::eChar)' elif tok == '%comp%': return 'tok->isComparisonOp()' elif tok == '%num%': @@ -102,24 +102,24 @@ class MatchCompiler: elif tok == '%op%': return 'tok->isOp()' elif tok == '%or%': - return '(tok->tokType() == Token::eBitOp && tok->str()==MatchCompiler::makeConstString("|") )' + return '(tok->tokType() == Token::eBitOp && tok->str() == MatchCompiler::makeConstString("|") )' elif tok == '%oror%': - return '(tok->tokType() == Token::eLogicalOp && tok->str()==MatchCompiler::makeConstString("||"))' + return '(tok->tokType() == Token::eLogicalOp && tok->str() == MatchCompiler::makeConstString("||"))' elif tok == '%str%': - return '(tok->tokType()==Token::eString)' + return '(tok->tokType() == Token::eString)' elif tok == '%type%': - return '(tok->isName() && tok->varId()==0U && (tok->str() != "delete" || !tok->isKeyword()))' + return '(tok->isName() && tok->varId() == 0U && (tok->str() != "delete" || !tok->isKeyword()))' elif tok == '%name%': return 'tok->isName()' elif tok == '%var%': return '(tok->varId() != 0)' elif tok == '%varid%': - return '(tok->isName() && tok->varId()==varid)' + return '(tok->isName() && tok->varId() == varid)' elif (len(tok) > 2) and (tok[0] == "%"): print("unhandled:" + tok) return ( - '(tok->str()==MatchCompiler::makeConstString("' + tok + '"))' + '(tok->str() == MatchCompiler::makeConstString("' + tok + '"))' ) def _compilePattern(self, pattern, nr, varid, @@ -155,7 +155,7 @@ class MatchCompiler: # [abc] if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'): - ret += ' if (!tok || tok->str().size()!=1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n' + ret += ' if (!tok || tok->str().size() != 1U || !strchr("' + tok[1:-1] + '", tok->str()[0]))\n' ret += ' ' + returnStatement # a|b|c @@ -330,8 +330,8 @@ class MatchCompiler: # ret += ' std::cout << "tok: " << tok->str();\n' # ret += ' if (tok->next())\n' # ret += ' std::cout << "tok next: " << tok->next()->str();\n' - ret += ' throw InternalError(tok, "Internal error.' +\ - 'compiled match returned different result than parsed match: ' + pattern + '");\n' + ret += ' throw InternalError(tok, "Internal error. ' +\ + 'Compiled match returned different result than parsed match: ' + pattern + '");\n' ret += ' }\n' ret += ' return res_compiled_match;\n' ret += '}\n' @@ -380,44 +380,46 @@ class MatchCompiler: ) def _replaceTokenMatch(self, line, linenr, filename): - while True: - is_simplematch = False - pos1 = line.find('Token::Match(') - if pos1 == -1: - is_simplematch = True - pos1 = line.find('Token::simpleMatch(') - if pos1 == -1: - break + for func in ('Match', 'simpleMatch'): + is_simplematch = func == 'simpleMatch' + pattern_start = 0 + while True: + pos1 = line.find('Token::' + func + '(', pattern_start) + if pos1 == -1: + break - res = self.parseMatch(line, pos1) - if res is None: - break + res = self.parseMatch(line, pos1) + if res is None: + break - # assert that Token::Match has either 2 or 3 arguments - assert(len(res) == 3 or len(res) == 4) + # assert that Token::Match has either 2 or 3 arguments + assert(len(res) == 3 or len(res) == 4) - end_pos = len(res[0]) - tok = res[1] - raw_pattern = res[2] - varId = None - if len(res) == 4: - varId = res[3] + end_pos = len(res[0]) + tok = res[1] + raw_pattern = res[2] + varId = None + if len(res) == 4: + varId = res[3] - res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern) - if res is None: - if self._showSkipped: - print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern) - break # Non-const pattern - bailout + pattern_start = pos1 + end_pos + res = re.match(r'\s*"((?:.|\\")*?)"\s*$', raw_pattern) + if res is None: + if self._showSkipped: + print(filename + ":" + str(linenr) + " skipping match pattern:" + raw_pattern) + continue # Non-const pattern - bailout - pattern = res.group(1) - line = self._replaceSpecificTokenMatch( - is_simplematch, - line, - pos1, - end_pos, - pattern, - tok, - varId) + pattern = res.group(1) + orig_len = len(line) + line = self._replaceSpecificTokenMatch( + is_simplematch, + line, + pos1, + end_pos, + pattern, + tok, + varId) + pattern_start += len(line) - orig_len return line @@ -459,7 +461,7 @@ class MatchCompiler: # We also need to verify builds in 'release' mode ret += ' if (res_parsed_findmatch != res_compiled_findmatch) {\n' ret += ' throw InternalError(tok, "Internal error. ' +\ - 'compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n' + 'Compiled findmatch returned different result than parsed findmatch: ' + pattern + '");\n' ret += ' }\n' ret += ' return res_compiled_findmatch;\n' ret += '}\n' diff --git a/tools/test_matchcompiler.py b/tools/test_matchcompiler.py index f51645596..57f07ae3a 100755 --- a/tools/test_matchcompiler.py +++ b/tools/test_matchcompiler.py @@ -39,6 +39,10 @@ class MatchCompilerTest(unittest.TestCase): output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") self.assertEqual(output, 'if (match1(tok)) {') + input = 'if (Token::simpleMatch(tok, "foobar")) {' + output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") + self.assertEqual(output, 'if (match1(tok)) {') + input = 'if (Token::Match(tok->next()->next(), "foobar %type% %num%")) {' output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") self.assertEqual(output, 'if (match2(tok->next()->next())) {') @@ -60,6 +64,25 @@ class MatchCompilerTest(unittest.TestCase): self.assertEqual( output, 'if (Token::Match(tok, "extern \"C\" " + varname)) {') + # test that multiple patterns on the same line are replaced + input = 'if (Token::Match(tok, "foo") && Token::Match(tok->next(), "baz")) {' + output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") + self.assertEqual(output, 'if (match4(tok) && match5(tok->next())) {') + + # test that second pattern is replaced, even if there is a bailout on the first pattern + input = 'if (Token::Match(tok, foo) && Token::Match(tok->next(), "foobaz")) {' + output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") + self.assertEqual(output, 'if (Token::Match(tok, foo) && match6(tok->next())) {') + + # test mixing Match and simpleMatch on the same line + input = 'if (Token::Match(tok, "a") && Token::simpleMatch(tok->next(), "b")) {' + output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") + self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {') + + input = 'if (Token::simpleMatch(tok, "a") && Token::Match(tok->next(), "b")) {' + output = self.mc._replaceTokenMatch(input, 0, "foo.cpp") + self.assertEqual(output, 'if (match7(tok) && match8(tok->next())) {') + def test_replaceTokenMatchWithVarId(self): input = 'if (Token::Match(tok, "foobar %varid%", 123)) {' output = self.mc._replaceTokenMatch(input, 0, "foo.cpp")