walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Tokenizer::tokenize() can now be called without AST being created

This commit is contained in:
PKEuS 2014-06-04 18:04:04 +02:00
parent d93d7401c6
commit 8db0790407
4 changed files with 20 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/preprocessor.cpp
View File

@ -1806,7 +1806,7 @@ std::string Preprocessor::getcode(const std::string &filedata, const std::string
Tokenizer tokenizer(_settings, _errorLogger); Tokenizer tokenizer(_settings, _errorLogger);
line.erase(0, sizeof("#pragma endasm")); line.erase(0, sizeof("#pragma endasm"));
std::istringstream tempIstr(line); std::istringstream tempIstr(line);
tokenizer.tokenize(tempIstr, ""); tokenizer.tokenize(tempIstr, "", "", true);
if (Token::Match(tokenizer.tokens(), "( %var% = %any% )")) { if (Token::Match(tokenizer.tokens(), "( %var% = %any% )")) {
ret << "asm(" << tokenizer.tokens()->strAt(1) << ");"; ret << "asm(" << tokenizer.tokens()->strAt(1) << ");";
} }

28
lib/tokenize.cpp
View File

@ -1581,7 +1581,8 @@ void Tokenizer::simplifyMulAndParens()
bool Tokenizer::tokenize(std::istream &code, bool Tokenizer::tokenize(std::istream &code,
const char FileName[], const char FileName[],
const std::string &configuration) const std::string &configuration,
bool noSymbolDB_AST)
{ {
// make sure settings specified // make sure settings specified
assert(_settings); assert(_settings);
@ -1598,22 +1599,23 @@ bool Tokenizer::tokenize(std::istream &code,
Token::isCPP(isCPP()); Token::isCPP(isCPP());
if (simplifyTokenList1(FileName)) { if (simplifyTokenList1(FileName)) {
createSymbolDatabase(); if (!noSymbolDB_AST) {
createSymbolDatabase();
// Use symbol database to identify rvalue references. Split && to & &. This is safe, since it doesn't delete any tokens (which might be referenced by symbol database) // Use symbol database to identify rvalue references. Split && to & &. This is safe, since it doesn't delete any tokens (which might be referenced by symbol database)
for (std::size_t i = 0; i < _symbolDatabase->getVariableListSize(); i++) { for (std::size_t i = 0; i < _symbolDatabase->getVariableListSize(); i++) {
const Variable* var = _symbolDatabase->getVariableFromVarId(i); const Variable* var = _symbolDatabase->getVariableFromVarId(i);
if (var && var->isRValueReference()) { if (var && var->isRValueReference()) {
const_cast<Token*>(var->typeEndToken())->str("&"); const_cast<Token*>(var->typeEndToken())->str("&");
const_cast<Token*>(var->typeEndToken())->insertToken("&"); const_cast<Token*>(var->typeEndToken())->insertToken("&");
const_cast<Token*>(var->typeEndToken()->next())->scope(var->typeEndToken()->scope()); const_cast<Token*>(var->typeEndToken()->next())->scope(var->typeEndToken()->scope());
}
} }
list.createAst();
ValueFlow::setValues(&list, _errorLogger, _settings);
} }
list.createAst();
ValueFlow::setValues(&list, _errorLogger, _settings);
return true; return true;
} }
return false; return false;

5
lib/tokenize.h
View File

@ -85,12 +85,13 @@ public:
* *
* @param FileName The filename * @param FileName The filename
* @param configuration E.g. "A" for code where "#ifdef A" is true * @param configuration E.g. "A" for code where "#ifdef A" is true
* @param noSymbolDB_AST Disable creation of SymbolDatabase and AST
* @return false if source code contains syntax errors * @return false if source code contains syntax errors
*/ */
bool tokenize(std::istream &code, bool tokenize(std::istream &code,
const char FileName[], const char FileName[],
const std::string &configuration = ""); const std::string &configuration = "",
bool noSymbolDB_AST = false);
/** /**
* tokenize condition and run simple simplifications on it * tokenize condition and run simple simplifications on it
* @param code code * @param code code

2
test/testsimplifytokens.cpp
View File

@ -2581,7 +2581,7 @@ private:
Tokenizer tokenizer(&settings, this); Tokenizer tokenizer(&settings, this);
std::istringstream istr(code); std::istringstream istr(code);
tokenizer.tokenize(istr, "test.cpp"); tokenizer.tokenize(istr, "test.cpp", "", true);
return TemplateSimplifier::templateParameters(tokenizer.tokens()); return TemplateSimplifier::templateParameters(tokenizer.tokens());
} }