walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

matchcompiler: autopep8 formatting

This commit is contained in:
Daniel Marjamäki 2013-10-13 11:01:50 +02:00
parent a7e925283a
commit 398ed973f4
1 changed files with 138 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
tools/matchcompiler.py
View File

@ -24,6 +24,7 @@ import argparse
class MatchCompiler: class MatchCompiler:
def __init__(self, verify_mode=False): def __init__(self, verify_mode=False):
self._verifyMode = verify_mode self._verifyMode = verify_mode
self._reset() self._reset()
@ -33,7 +34,8 @@ class MatchCompiler:
self._matchStrs = {} self._matchStrs = {}
self._matchFunctionCache = {} self._matchFunctionCache = {}
def _generateCacheSignature(self, pattern, endToken=None, varId=None, isFindMatch=False): def _generateCacheSignature(
self, pattern, endToken=None, varId=None, isFindMatch=False):
sig = pattern sig = pattern
if endToken: if endToken:
@ -53,19 +55,28 @@ class MatchCompiler:
return sig return sig
def _lookupMatchFunctionId(self, pattern, endToken=None, varId=None, isFindMatch=False): def _lookupMatchFunctionId(
signature = self._generateCacheSignature(pattern, endToken, varId, isFindMatch) self, pattern, endToken=None, varId=None, isFindMatch=False):
signature = self._generateCacheSignature(
pattern, endToken, varId, isFindMatch)
if signature in self._matchFunctionCache: if signature in self._matchFunctionCache:
return self._matchFunctionCache[signature] return self._matchFunctionCache[signature]
return None return None
def _insertMatchFunctionId(self, id, pattern, endToken=None, varId=None, isFindMatch=False): def _insertMatchFunctionId(
signature = self._generateCacheSignature(pattern, endToken, varId, isFindMatch) self, id, pattern, endToken=None, varId=None, isFindMatch=False):
signature = self._generateCacheSignature(
pattern, endToken, varId, isFindMatch)
# function signature should not be in the cache # function signature should not be in the cache
assert(self._lookupMatchFunctionId(pattern, endToken, varId, isFindMatch) == None) assert(
self._lookupMatchFunctionId(
pattern,
endToken,
varId,
isFindMatch) is None)
self._matchFunctionCache[signature] = id self._matchFunctionCache[signature] = id
@ -101,7 +112,10 @@ class MatchCompiler:
elif tok == '%str%': elif tok == '%str%':
return '(tok->type()==Token::eString)' return '(tok->type()==Token::eString)'
elif tok == '%type%': elif tok == '%type%':
return '(tok->isName() && tok->varId()==0U && tok->str() != ' + self._insertMatchStr('delete') + '/* delete */)' return (
'(tok->isName() && tok->varId()==0U && tok->str() != ' +
self._insertMatchStr('delete') + '/* delete */)'
)
elif tok == '%var%': elif tok == '%var%':
return 'tok->isName()' return 'tok->isName()'
elif tok == '%varid%': elif tok == '%varid%':
@ -109,9 +123,12 @@ class MatchCompiler:
elif (len(tok) > 2) and (tok[0] == "%"): elif (len(tok) > 2) and (tok[0] == "%"):
print ("unhandled:" + tok) print ("unhandled:" + tok)
return '(tok->str()==' + self._insertMatchStr(tok) + ')/* ' + tok + ' */' return (
'(tok->str()==' + self._insertMatchStr(tok) + ')/* ' + tok + ' */'
)
def _compilePattern(self, pattern, nr, varid, isFindMatch=False, tokenType="const Token"): def _compilePattern(self, pattern, nr, varid,
isFindMatch=False, tokenType="const Token"):
ret = '' ret = ''
returnStatement = '' returnStatement = ''
@ -124,7 +141,8 @@ class MatchCompiler:
arg2 = ', const unsigned int varid' arg2 = ', const unsigned int varid'
ret = '// pattern: ' + pattern + '\n' ret = '// pattern: ' + pattern + '\n'
ret += 'static bool match' + str(nr) + '(' + tokenType + '* tok'+arg2+') {\n' ret += 'static bool match' + \
str(nr) + '(' + tokenType + '* tok' + arg2 + ') {\n'
returnStatement = 'return false;\n' returnStatement = 'return false;\n'
tokens = pattern.split(' ') tokens = pattern.split(' ')
@ -144,7 +162,8 @@ class MatchCompiler:
# [abc] # [abc]
if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'): if (len(tok) > 2) and (tok[0] == '[') and (tok[-1] == ']'):
ret += ' if (!tok || tok->str().size()!=1U || !strchr("'+tok[1:-1]+'", tok->str()[0]))\n' ret += ' if (!tok || tok->str().size()!=1U || !strchr("' + tok[
1:-1] + '", tok->str()[0]))\n'
ret += ' ' + returnStatement ret += ' ' + returnStatement
# a|b|c # a|b|c
@ -179,7 +198,8 @@ class MatchCompiler:
# !!a # !!a
elif tok[0:2] == "!!": elif tok[0:2] == "!!":
ret += ' if (tok && tok->str() == ' + self._insertMatchStr(tok[2:]) + ')/* ' + tok[2:] + ' */\n' ret += ' if (tok && tok->str() == ' + self._insertMatchStr(
tok[2:]) + ')/* ' + tok[2:] + ' */\n'
ret += ' ' + returnStatement ret += ' ' + returnStatement
gotoNextToken = ' tok = tok ? tok->next() : NULL;\n' gotoNextToken = ' tok = tok ? tok->next() : NULL;\n'
@ -205,8 +225,10 @@ class MatchCompiler:
more_args += ', unsigned int varid' more_args += ', unsigned int varid'
ret = '// pattern: ' + pattern + '\n' ret = '// pattern: ' + pattern + '\n'
ret += 'template<class T> static T * findmatch' + str(findmatchnr) + '(T * start_tok'+more_args+') {\n' ret += 'template<class T> static T * findmatch' + \
ret += ' for (; start_tok' + endCondition + '; start_tok = start_tok->next()) {\n' str(findmatchnr) + '(T * start_tok' + more_args + ') {\n'
ret += ' for (; start_tok' + endCondition + \
'; start_tok = start_tok->next()) {\n'
ret += self._compilePattern(pattern, -1, varId, True, 'T') ret += self._compilePattern(pattern, -1, varId, True, 'T')
ret += ' }\n' ret += ' }\n'
@ -236,7 +258,7 @@ class MatchCompiler:
parlevel -= 1 parlevel -= 1
if parlevel == 0: if parlevel == 0:
ret = [] ret = []
ret.append(line[pos1:pos+1]) ret.append(line[pos1:pos + 1])
for arg in args: for arg in args:
ret.append(arg) ret.append(arg)
ret.append(line[argstart:pos]) ret.append(line[argstart:pos])
@ -259,7 +281,7 @@ class MatchCompiler:
pos += 1 pos += 1
elif line[pos] == '"': elif line[pos] == '"':
inString = False inString = False
endPos = pos+1 endPos = pos + 1
return (startPos, endPos) return (startPos, endPos)
elif line[pos] == '"': elif line[pos] == '"':
startPos = pos startPos = pos
@ -268,24 +290,28 @@ class MatchCompiler:
return None return None
def _compileVerifyTokenMatch(self, is_simplematch, verifyNumber, pattern, patternNumber, varId): def _compileVerifyTokenMatch(
self, is_simplematch, verifyNumber, pattern, patternNumber, varId):
more_args = '' more_args = ''
if varId: if varId:
more_args = ', const unsigned int varid' more_args = ', const unsigned int varid'
ret = 'static bool match_verify' + str(verifyNumber) + '(const Token *tok'+more_args+') {\n' ret = 'static bool match_verify' + \
str(verifyNumber) + '(const Token *tok' + more_args + ') {\n'
origMatchName = 'Match' origMatchName = 'Match'
if is_simplematch: if is_simplematch:
origMatchName = 'simpleMatch' origMatchName = 'simpleMatch'
assert(varId == None) assert(varId is None)
ret += ' bool res_compiled_match = match'+str(patternNumber)+'(tok' ret += ' bool res_compiled_match = match' + \
str(patternNumber) + '(tok'
if varId: if varId:
ret += ', varid' ret += ', varid'
ret += ');\n' ret += ');\n'
ret += ' bool res_parsed_match = Token::' + origMatchName + '(tok, "' + pattern + '"' ret += ' bool res_parsed_match = Token::' + \
origMatchName + '(tok, "' + pattern + '"'
if varId: if varId:
ret += ', varid' ret += ', varid'
ret += ');\n' ret += ');\n'
@ -306,29 +332,46 @@ class MatchCompiler:
return ret return ret
def _replaceSpecificTokenMatch(self, is_simplematch, line, start_pos, end_pos, pattern, tok, varId): def _replaceSpecificTokenMatch(
self, is_simplematch, line, start_pos, end_pos, pattern, tok, varId):
more_args = '' more_args = ''
if varId: if varId:
more_args = ',' + varId more_args = ',' + varId
# Compile function or use previously compiled one # Compile function or use previously compiled one
patternNumber = self._lookupMatchFunctionId(pattern, None, varId, False) patternNumber = self._lookupMatchFunctionId(
pattern, None, varId, False)
if patternNumber is None: if patternNumber is None:
patternNumber = len(self._rawMatchFunctions) + 1 patternNumber = len(self._rawMatchFunctions) + 1
self._insertMatchFunctionId(patternNumber, pattern, None, varId, False) self._insertMatchFunctionId(
self._rawMatchFunctions.append(self._compilePattern(pattern, patternNumber, varId)) patternNumber,
pattern,
None,
varId,
False)
self._rawMatchFunctions.append(
self._compilePattern(pattern, patternNumber, varId))
functionName = "match" functionName = "match"
if self._verifyMode: if self._verifyMode:
verifyNumber = len(self._rawMatchFunctions) + 1 verifyNumber = len(self._rawMatchFunctions) + 1
self._rawMatchFunctions.append(self._compileVerifyTokenMatch(is_simplematch, verifyNumber, pattern, patternNumber, varId)) self._rawMatchFunctions.append(
self._compileVerifyTokenMatch(
is_simplematch,
verifyNumber,
pattern,
patternNumber,
varId))
# inject verify function # inject verify function
functionName = "match_verify" functionName = "match_verify"
patternNumber = verifyNumber patternNumber = verifyNumber
return line[:start_pos]+functionName+str(patternNumber)+'('+tok+more_args+')'+line[start_pos+end_pos:] return (
line[:start_pos] + functionName + str(
patternNumber) + '(' + tok + more_args + ')' + line[start_pos + end_pos:]
)
def _replaceTokenMatch(self, line): def _replaceTokenMatch(self, line):
while True: while True:
@ -344,7 +387,8 @@ class MatchCompiler:
if res is None: if res is None:
break break
assert(len(res) == 3 or len(res) == 4) # assert that Token::Match has either 2 or 3 arguments # assert that Token::Match has either 2 or 3 arguments
assert(len(res) == 3 or len(res) == 4)
end_pos = len(res[0]) end_pos = len(res[0])
tok = res[1] tok = res[1]
@ -358,32 +402,43 @@ class MatchCompiler:
break # Non-const pattern - bailout break # Non-const pattern - bailout
pattern = res.group(1) pattern = res.group(1)
line = self._replaceSpecificTokenMatch(is_simplematch, line, pos1, end_pos, pattern, tok, varId) line = self._replaceSpecificTokenMatch(
is_simplematch,
line,
pos1,
end_pos,
pattern,
tok,
varId)
return line return line
def _compileVerifyTokenFindMatch(self, is_findsimplematch, verifyNumber, pattern, patternNumber, endToken, varId): def _compileVerifyTokenFindMatch(
self, is_findsimplematch, verifyNumber, pattern, patternNumber, endToken, varId):
more_args = '' more_args = ''
if endToken: if endToken:
more_args += ', const Token * endToken' more_args += ', const Token * endToken'
if varId: if varId:
more_args += ', const unsigned int varid' more_args += ', const unsigned int varid'
ret = 'template < class T > static T * findmatch_verify' + str(verifyNumber) + '(T * tok'+more_args+') {\n' ret = 'template < class T > static T * findmatch_verify' + \
str(verifyNumber) + '(T * tok' + more_args + ') {\n'
origFindMatchName = 'findmatch' origFindMatchName = 'findmatch'
if is_findsimplematch: if is_findsimplematch:
origMatchName = 'findsimplematch' origMatchName = 'findsimplematch'
assert(varId == None) assert(varId is None)
ret += ' T * res_compiled_findmatch = findmatch'+str(patternNumber)+'(tok' ret += ' T * res_compiled_findmatch = findmatch' + \
str(patternNumber) + '(tok'
if endToken: if endToken:
ret += ', endToken' ret += ', endToken'
if varId: if varId:
ret += ', varid' ret += ', varid'
ret += ');\n' ret += ');\n'
ret += ' T * res_parsed_findmatch = Token::' + origFindMatchName + '(tok, "' + pattern + '"' ret += ' T * res_parsed_findmatch = Token::' + \
origFindMatchName + '(tok, "' + pattern + '"'
if endToken: if endToken:
ret += ', endToken' ret += ', endToken'
if varId: if varId:
@ -401,7 +456,8 @@ class MatchCompiler:
return ret return ret
def _replaceSpecificFindTokenMatch(self, is_findsimplematch, line, start_pos, end_pos, pattern, tok, endToken, varId): def _replaceSpecificFindTokenMatch(
self, is_findsimplematch, line, start_pos, end_pos, pattern, tok, endToken, varId):
more_args = '' more_args = ''
if endToken: if endToken:
more_args += ',' + endToken more_args += ',' + endToken
@ -409,23 +465,44 @@ class MatchCompiler:
more_args += ',' + varId more_args += ',' + varId
# Compile function or use previously compiled one # Compile function or use previously compiled one
findMatchNumber = self._lookupMatchFunctionId(pattern, endToken, varId, True) findMatchNumber = self._lookupMatchFunctionId(
pattern, endToken, varId, True)
if findMatchNumber is None: if findMatchNumber is None:
findMatchNumber = len(self._rawMatchFunctions) + 1 findMatchNumber = len(self._rawMatchFunctions) + 1
self._insertMatchFunctionId(findMatchNumber, pattern, endToken, varId, True) self._insertMatchFunctionId(
self._rawMatchFunctions.append(self._compileFindPattern(pattern, findMatchNumber, endToken, varId)) findMatchNumber,
pattern,
endToken,
varId,
True)
self._rawMatchFunctions.append(
self._compileFindPattern(
pattern,
findMatchNumber,
endToken,
varId))
functionName = "findmatch" functionName = "findmatch"
if self._verifyMode: if self._verifyMode:
verifyNumber = len(self._rawMatchFunctions) + 1 verifyNumber = len(self._rawMatchFunctions) + 1
self._rawMatchFunctions.append(self._compileVerifyTokenFindMatch(is_findsimplematch, verifyNumber, pattern, findMatchNumber, endToken, varId)) self._rawMatchFunctions.append(
self._compileVerifyTokenFindMatch(
is_findsimplematch,
verifyNumber,
pattern,
findMatchNumber,
endToken,
varId))
# inject verify function # inject verify function
functionName = "findmatch_verify" functionName = "findmatch_verify"
findMatchNumber = verifyNumber findMatchNumber = verifyNumber
return line[:start_pos]+functionName+str(findMatchNumber)+'('+tok+more_args+')'+line[start_pos+end_pos:] return (
line[:start_pos] + functionName + str(
findMatchNumber) + '(' + tok + more_args + ')' + line[start_pos + end_pos:]
)
def _replaceTokenFindMatch(self, line): def _replaceTokenFindMatch(self, line):
pos1 = 0 pos1 = 0
@ -442,7 +519,9 @@ class MatchCompiler:
if res is None: if res is None:
break break
assert(len(res) >= 3 or len(res) < 6) # assert that Token::find(simple)match has either 2, 3 or four arguments assert(len(res) >= 3 or len(res) < 6)
# assert that Token::find(simple)match has either 2, 3 or
# four arguments
g0 = res[0] g0 = res[0]
tok = res[1] tok = res[1]
@ -461,7 +540,8 @@ class MatchCompiler:
# Token *findsimplematch(const Token *tok, const char pattern[]); # Token *findsimplematch(const Token *tok, const char pattern[]);
# Token *findsimplematch(const Token *tok, const char pattern[], const Token *end); # Token *findsimplematch(const Token *tok, const char pattern[], const Token *end);
# Token *findmatch(const Token *tok, const char pattern[], unsigned int varId = 0); # Token *findmatch(const Token *tok, const char pattern[], unsigned int varId = 0);
# Token *findmatch(const Token *tok, const char pattern[], const Token *end, unsigned int varId = 0); # Token *findmatch(const Token *tok, const char pattern[], const
# Token *end, unsigned int varId = 0);
endToken = None endToken = None
if is_findsimplematch is True and len(res) == 4: if is_findsimplematch is True and len(res) == 4:
endToken = res[3] endToken = res[3]
@ -476,7 +556,15 @@ class MatchCompiler:
break # Non-const pattern - bailout break # Non-const pattern - bailout
pattern = res.group(1) pattern = res.group(1)
line = self._replaceSpecificFindTokenMatch(is_findsimplematch, line, pos1, len(g0), pattern, tok, endToken, varId) line = self._replaceSpecificFindTokenMatch(
is_findsimplematch,
line,
pos1,
len(g0),
pattern,
tok,
endToken,
varId)
return line return line
@ -494,7 +582,7 @@ class MatchCompiler:
startPos = res[0] startPos = res[0]
endPos = res[1] endPos = res[1]
text = line[startPos+1:endPos-1] text = line[startPos + 1:endPos - 1]
line = line[:startPos] + self._insertMatchStr(text) + line[endPos:] line = line[:startPos] + self._insertMatchStr(text) + line[endPos:]
return line return line
@ -528,7 +616,8 @@ class MatchCompiler:
# Compute string list # Compute string list
stringList = '' stringList = ''
for match in sorted(self._matchStrs, key=self._matchStrs.get): for match in sorted(self._matchStrs, key=self._matchStrs.get):
stringList += 'static const std::string matchStr' + str(self._matchStrs[match]) + '("' + match + '");\n' stringList += 'static const std::string matchStr' + \
str(self._matchStrs[match]) + '("' + match + '");\n'
# Compute matchFunctions # Compute matchFunctions
strFunctions = '' strFunctions = ''
@ -536,7 +625,7 @@ class MatchCompiler:
strFunctions += function strFunctions += function
fout = open(destname, 'wt') fout = open(destname, 'wt')
fout.write(header+stringList+strFunctions+code) fout.write(header + stringList + strFunctions + code)
fout.close() fout.close()
@ -556,7 +645,8 @@ def main():
raise Exception(build_dir + ' is not a directory') raise Exception(build_dir + ' is not a directory')
# Argument handling # Argument handling
parser = argparse.ArgumentParser(description='Compile Token::Match() calls into native C++ code') parser = argparse.ArgumentParser(
description='Compile Token::Match() calls into native C++ code')
parser.add_argument('--verify', action='store_true', default=False, parser.add_argument('--verify', action='store_true', default=False,
help='verify compiled matches against on-the-fly parser. Slow!') help='verify compiled matches against on-the-fly parser. Slow!')
args = parser.parse_args() args = parser.parse_args()
@ -566,7 +656,7 @@ def main():
# convert all lib/*.cpp files # convert all lib/*.cpp files
for f in glob.glob('lib/*.cpp'): for f in glob.glob('lib/*.cpp'):
print (f + ' => ' + build_dir + '/' + f[4:]) print (f + ' => ' + build_dir + '/' + f[4:])
mc.convertFile(f, build_dir + '/'+f[4:]) mc.convertFile(f, build_dir + '/' + f[4:])
if __name__ == '__main__': if __name__ == '__main__':
main() main()