walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Match compiler: Turn code into a python class 

This will make passing around internal states a lot easier
This commit is contained in:
Thomas Jarosch 2013-01-04 03:38:40 +01:00
parent 7f0bc73e8e
commit 71a236b3df
1 changed files with 339 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
tools/matchcompiler.py
View File

@ -3,7 +3,11 @@
import re import re
import glob import glob
def insertMatchStr(matchStrs, look_for): class MatchCompiler:
def __init__(self):
self._selftests()
def _insertMatchStr(self, matchStrs, look_for):
prefix = 'matchStr' prefix = 'matchStr'
# Add entry if needed # Add entry if needed
@ -13,7 +17,7 @@ def insertMatchStr(matchStrs, look_for):
return prefix + str(matchStrs[look_for]) return prefix + str(matchStrs[look_for])
def compileCmd(tok, matchStrs): def _compileCmd(self, tok, matchStrs):
if tok == '%any%': if tok == '%any%':
return 'true' return 'true'
elif tok == '%bool%': elif tok == '%bool%':
@ -27,13 +31,13 @@ def compileCmd(tok, matchStrs):
elif tok == '%op%': elif tok == '%op%':
return 'tok->isOp()' return 'tok->isOp()'
elif tok == '%or%': elif tok == '%or%':
return '(tok->str()==' + insertMatchStr(matchStrs, '|') + ')/* | */' return '(tok->str()==' + self._insertMatchStr(matchStrs, '|') + ')/* | */'
elif tok == '%oror%': elif tok == '%oror%':
return '(tok->str()==' + insertMatchStr(matchStrs, '||') + ')/* || */' return '(tok->str()==' + self._insertMatchStr(matchStrs, '||') + ')/* || */'
elif tok == '%str%': elif tok == '%str%':
return '(tok->type()==Token::eString)' return '(tok->type()==Token::eString)'
elif tok == '%type%': elif tok == '%type%':
return '(tok->isName() && tok->varId()==0U && tok->str() != ' + insertMatchStr(matchStrs, 'delete') + '/* delete */)' return '(tok->isName() && tok->varId()==0U && tok->str() != ' + self._insertMatchStr(matchStrs, 'delete') + '/* delete */)'
elif tok == '%var%': elif tok == '%var%':
return 'tok->isName()' return 'tok->isName()'
elif tok == '%varid%': elif tok == '%varid%':
@ -41,9 +45,9 @@ def compileCmd(tok, matchStrs):
elif (len(tok)>2) and (tok[0]=="%"): elif (len(tok)>2) and (tok[0]=="%"):
print ("unhandled:" + tok) print ("unhandled:" + tok)
return '(tok->str()==' + insertMatchStr(matchStrs, tok) + ')/* ' + tok + ' */' return '(tok->str()==' + self._insertMatchStr(matchStrs, tok) + ')/* ' + tok + ' */'
def compilePattern(matchStrs, pattern, nr, varid, isFindMatch=False): def _compilePattern(self, matchStrs, pattern, nr, varid, isFindMatch=False):
ret = '' ret = ''
returnStatement = '' returnStatement = ''
@ -99,7 +103,7 @@ def compilePattern(matchStrs, pattern, nr, varid, isFindMatch=False):
if not first: if not first:
ret += logicalOp ret += logicalOp
first = False first = False
ret += neg + compileCmd(tok2, matchStrs) ret += neg + self._compileCmd(tok2, matchStrs)
if "" in tokens2: if "" in tokens2:
ret += '))\n' ret += '))\n'
@ -111,12 +115,12 @@ def compilePattern(matchStrs, pattern, nr, varid, isFindMatch=False):
# !!a # !!a
elif tok[0:2]=="!!": elif tok[0:2]=="!!":
ret += ' if (tok && tok->str() == ' + insertMatchStr(matchStrs, tok[2:]) + ')/* ' + tok[2:] + ' */\n' ret += ' if (tok && tok->str() == ' + self._insertMatchStr(matchStrs, tok[2:]) + ')/* ' + tok[2:] + ' */\n'
ret += ' ' + returnStatement ret += ' ' + returnStatement
gotoNextToken = ' tok = tok ? tok->next() : NULL;\n' gotoNextToken = ' tok = tok ? tok->next() : NULL;\n'
else: else:
ret += ' if (!tok || !' + compileCmd(tok, matchStrs) + ')\n' ret += ' if (!tok || !' + self._compileCmd(tok, matchStrs) + ')\n'
ret += ' ' + returnStatement ret += ' ' + returnStatement
if isFindMatch: if isFindMatch:
@ -127,7 +131,7 @@ def compilePattern(matchStrs, pattern, nr, varid, isFindMatch=False):
return ret return ret
def compileFindPattern(matchFunctions, matchStrs, pattern, findmatchnr, endToken, varId): def _compileFindPattern(self, matchFunctions, matchStrs, pattern, findmatchnr, endToken, varId):
more_args = '' more_args = ''
endCondition = '' endCondition = ''
if endToken: if endToken:
@ -140,13 +144,13 @@ def compileFindPattern(matchFunctions, matchStrs, pattern, findmatchnr, endToken
ret += 'static const Token *findmatch' + str(findmatchnr) + '(const Token *start_tok'+more_args+') {\n' ret += 'static const Token *findmatch' + str(findmatchnr) + '(const Token *start_tok'+more_args+') {\n'
ret += ' for (; start_tok' + endCondition + '; start_tok = start_tok->next()) {\n' ret += ' for (; start_tok' + endCondition + '; start_tok = start_tok->next()) {\n'
ret += compilePattern(matchStrs, pattern, -1, varId, True) ret += self._compilePattern(matchStrs, pattern, -1, varId, True)
ret += ' }\n' ret += ' }\n'
ret += ' return NULL;\n}\n' ret += ' return NULL;\n}\n'
return ret return ret
def parseMatch(line, pos1): def parseMatch(self, line, pos1):
parlevel = 0 parlevel = 0
args = [] args = []
argstart = 0 argstart = 0
@ -180,7 +184,7 @@ def parseMatch(line, pos1):
return None return None
def parseStringComparison(line, pos1): def _parseStringComparison(self, line, pos1):
startPos = 0 startPos = 0
endPos = 0 endPos = 0
pos = pos1 pos = pos1
@ -200,7 +204,7 @@ def parseStringComparison(line, pos1):
return None return None
def replaceTokenMatch(matchFunctions, matchStrs, line): def _replaceTokenMatch(self, matchFunctions, matchStrs, line):
while True: while True:
pos1 = line.find('Token::Match(') pos1 = line.find('Token::Match(')
if pos1 == -1: if pos1 == -1:
@ -208,7 +212,7 @@ def replaceTokenMatch(matchFunctions, matchStrs, line):
if pos1 == -1: if pos1 == -1:
break break
res = parseMatch(line, pos1) res = self.parseMatch(line, pos1)
if res == None: if res == None:
break break
else: else:
@ -231,11 +235,11 @@ def replaceTokenMatch(matchFunctions, matchStrs, line):
a3 = ',' + arg3 a3 = ',' + arg3
patternNumber = len(matchFunctions) + 1 patternNumber = len(matchFunctions) + 1
line = line[:pos1]+'match'+str(patternNumber)+'('+arg1+a3+')'+line[pos1+len(g0):] line = line[:pos1]+'match'+str(patternNumber)+'('+arg1+a3+')'+line[pos1+len(g0):]
matchFunctions.append(compilePattern(matchStrs, arg2, patternNumber, arg3)) matchFunctions.append(self._compilePattern(matchStrs, arg2, patternNumber, arg3))
return line return line
def replaceTokenFindMatch(matchFunctions, matchStrs, line): def _replaceTokenFindMatch(self, matchFunctions, matchStrs, line):
pos1 = 0 pos1 = 0
while True: while True:
is_findmatch = False is_findmatch = False
@ -246,7 +250,7 @@ def replaceTokenFindMatch(matchFunctions, matchStrs, line):
if pos1 == -1: if pos1 == -1:
break break
res = parseMatch(line, pos1) res = self.parseMatch(line, pos1)
if res == None: if res == None:
break break
else: else:
@ -291,11 +295,11 @@ def replaceTokenFindMatch(matchFunctions, matchStrs, line):
a3 += ',' + varId a3 += ',' + varId
findMatchNumber = len(matchFunctions) + 1 findMatchNumber = len(matchFunctions) + 1
line = line[:pos1]+'findmatch'+str(findMatchNumber)+'('+arg1+a3+')'+line[pos1+len(g0):] line = line[:pos1]+'findmatch'+str(findMatchNumber)+'('+arg1+a3+')'+line[pos1+len(g0):]
matchFunctions.append(compileFindPattern(matchFunctions, matchStrs, pattern, findMatchNumber, endToken, varId)) matchFunctions.append(self._compileFindPattern(matchFunctions, matchStrs, pattern, findMatchNumber, endToken, varId))
return line return line
def replaceCStrings(matchStrs, line): def _replaceCStrings(self, matchStrs, line):
while True: while True:
match = re.search('str\(\) (==|!=) "', line) match = re.search('str\(\) (==|!=) "', line)
if not match: if not match:
@ -303,18 +307,18 @@ def replaceCStrings(matchStrs, line):
if not match: if not match:
break break
res = parseStringComparison(line, match.start()) res = self._parseStringComparison(line, match.start())
if res == None: if res == None:
break break
startPos = res[0] startPos = res[0]
endPos = res[1] endPos = res[1]
text = line[startPos+1:endPos-1] text = line[startPos+1:endPos-1]
line = line[:startPos] + insertMatchStr(matchStrs, text) + line[endPos:] line = line[:startPos] + self._insertMatchStr(matchStrs, text) + line[endPos:]
return line return line
def convertFile(srcname, destname): def convertFile(self, srcname, destname):
fin = open(srcname, "rt") fin = open(srcname, "rt")
srclines = fin.readlines() srclines = fin.readlines()
fin.close() fin.close()
@ -329,14 +333,14 @@ def convertFile(srcname, destname):
matchStrs = {} matchStrs = {}
for line in srclines: for line in srclines:
# Compile Token::Match and Token::simpleMatch # Compile Token::Match and Token::simpleMatch
line = replaceTokenMatch(matchFunctions, matchStrs, line) line = self._replaceTokenMatch(matchFunctions, matchStrs, line)
# Compile Token::findsimplematch # Compile Token::findsimplematch
# NOTE: Not enabled for now since the generated code is slower than before. # NOTE: Not enabled for now since the generated code is slower than before.
# line = replaceTokenFindMatch(matchFunctions, matchStrs, line) # line = self._replaceTokenFindMatch(matchFunctions, matchStrs, line)
# Cache plain C-strings in C++ strings # Cache plain C-strings in C++ strings
line = replaceCStrings(matchStrs, line) line = self._replaceCStrings(matchStrs, line)
code += line code += line
@ -354,18 +358,23 @@ def convertFile(srcname, destname):
fout.write(header+stringList+strFunctions+code) fout.write(header+stringList+strFunctions+code)
fout.close() fout.close()
# selftests.. def _assertEquals(self,actual,expected):
def assertEquals(actual,expected):
if actual!=expected: if actual!=expected:
print ('Assertion failed:') print ('Assertion failed:')
print (actual) print (actual)
print (expected) print (expected)
assert actual == expected assert actual == expected
assertEquals(parseMatch(' Token::Match(tok, ";") ',2), ['Token::Match(tok, ";")','tok',' ";"'])
assertEquals(parseMatch(' Token::Match(tok,', 2), None) # multiline Token::Match is not supported yet def _selftests(self):
assertEquals(parseMatch(' Token::Match(Token::findsimplematch(tok,")"), ";")', 2), ['Token::Match(Token::findsimplematch(tok,")"), ";")', 'Token::findsimplematch(tok,")")', ' ";"']) # inner function call self._assertEquals(self.parseMatch(' Token::Match(tok, ";") ',2), ['Token::Match(tok, ";")','tok',' ";"'])
self._assertEquals(self.parseMatch(' Token::Match(tok,', 2), None) # multiline Token::Match is not supported yet
self._assertEquals(self.parseMatch(' Token::Match(Token::findsimplematch(tok,")"), ";")', 2), ['Token::Match(Token::findsimplematch(tok,")"), ";")', 'Token::findsimplematch(tok,")")', ' ";"']) # inner function call
# Main program
mc = MatchCompiler()
# convert all lib/*.cpp files # convert all lib/*.cpp files
for f in glob.glob('lib/*.cpp'): for f in glob.glob('lib/*.cpp'):
print (f + ' => build/' + f[4:]) print (f + ' => build/' + f[4:])
convertFile(f, 'build/'+f[4:]) mc.convertFile(f, 'build/'+f[4:])