walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Tokenizer::tokenize() can now be called without AST being created

This commit is contained in:
PKEuS 2014-06-04 18:04:04 +02:00
parent d93d7401c6
commit 8db0790407
4 changed files with 20 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/preprocessor.cpp
View File

@ -1806,7 +1806,7 @@ std::string Preprocessor::getcode(const std::string &filedata, const std::string
Tokenizer tokenizer(_settings, _errorLogger);
line.erase(0, sizeof("#pragma endasm"));
std::istringstream tempIstr(line);
tokenizer.tokenize(tempIstr, "");
tokenizer.tokenize(tempIstr, "", "", true);
if (Token::Match(tokenizer.tokens(), "( %var% = %any% )")) {
ret << "asm(" << tokenizer.tokens()->strAt(1) << ");";
}

28
lib/tokenize.cpp
View File

@ -1581,7 +1581,8 @@ void Tokenizer::simplifyMulAndParens()
bool Tokenizer::tokenize(std::istream &code,
const char FileName[],
const std::string &configuration)
const std::string &configuration,
bool noSymbolDB_AST)
{
// make sure settings specified
assert(_settings);
@ -1598,22 +1599,23 @@ bool Tokenizer::tokenize(std::istream &code,
Token::isCPP(isCPP());
if (simplifyTokenList1(FileName)) {
createSymbolDatabase();
if (!noSymbolDB_AST) {
createSymbolDatabase();
// Use symbol database to identify rvalue references. Split && to & &. This is safe, since it doesn't delete any tokens (which might be referenced by symbol database)
for (std::size_t i = 0; i < _symbolDatabase->getVariableListSize(); i++) {
const Variable* var = _symbolDatabase->getVariableFromVarId(i);
if (var && var->isRValueReference()) {
const_cast<Token*>(var->typeEndToken())->str("&");
const_cast<Token*>(var->typeEndToken())->insertToken("&");
const_cast<Token*>(var->typeEndToken()->next())->scope(var->typeEndToken()->scope());
// Use symbol database to identify rvalue references. Split && to & &. This is safe, since it doesn't delete any tokens (which might be referenced by symbol database)
for (std::size_t i = 0; i < _symbolDatabase->getVariableListSize(); i++) {
const Variable* var = _symbolDatabase->getVariableFromVarId(i);
if (var && var->isRValueReference()) {
const_cast<Token*>(var->typeEndToken())->str("&");
const_cast<Token*>(var->typeEndToken())->insertToken("&");
const_cast<Token*>(var->typeEndToken()->next())->scope(var->typeEndToken()->scope());
}
}
list.createAst();
ValueFlow::setValues(&list, _errorLogger, _settings);
}
list.createAst();
ValueFlow::setValues(&list, _errorLogger, _settings);
return true;
}
return false;

5
lib/tokenize.h
View File

@ -85,12 +85,13 @@ public:
*
* @param FileName The filename
* @param configuration E.g. "A" for code where "#ifdef A" is true
* @param noSymbolDB_AST Disable creation of SymbolDatabase and AST
* @return false if source code contains syntax errors
*/
bool tokenize(std::istream &code,
const char FileName[],
const std::string &configuration = "");
const std::string &configuration = "",
bool noSymbolDB_AST = false);
/**
* tokenize condition and run simple simplifications on it
* @param code code

2
test/testsimplifytokens.cpp
View File

@ -2581,7 +2581,7 @@ private:
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.tokenize(istr, "test.cpp");
tokenizer.tokenize(istr, "test.cpp", "", true);
return TemplateSimplifier::templateParameters(tokenizer.tokens());
}