walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Tokenizer: Add special tokenize method for the Preprocessor with only basic simplifications

This commit is contained in:
Daniel Marjamäki 2013-01-27 17:58:54 +01:00
parent 0a88a136ca
commit 4391f0880f
4 changed files with 176 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/preprocessor.cpp
View File

@ -1335,8 +1335,7 @@ std::list<std::string> Preprocessor::getcfgs(const std::string &filedata, const
if (s.find("&&") != std::string::npos) {
Tokenizer tokenizer(_settings, _errorLogger);
std::istringstream tempIstr(s);
if (!tokenizer.tokenize(tempIstr, filename.c_str(), "", true)) {
if (!tokenizer.tokenizeCondition(s)) {
std::ostringstream lineStream;
lineStream << __LINE__;
@ -1444,8 +1443,7 @@ void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &c
{
const Settings settings;
Tokenizer tokenizer(&settings, _errorLogger);
std::istringstream istr("(" + condition + ")");
if (!tokenizer.tokenize(istr, "", "", true)) {
if (!tokenizer.tokenizeCondition("(" + condition + ")")) {
// If tokenize returns false, then there is syntax error in the
// code which we can't handle. So stop here.
return;
@ -1505,8 +1503,7 @@ void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &c
if (!it->second.empty()) {
// Tokenize the value
Tokenizer tokenizer2(&settings,NULL);
std::istringstream istr2(it->second);
tokenizer2.tokenize(istr2,"","",true);
tokenizer2.tokenizeCondition(it->second);
// Copy the value tokens
std::stack<Token *> link;

182
lib/tokenize.cpp
View File

@ -1554,8 +1554,7 @@ void Tokenizer::simplifyMulAndParens()
bool Tokenizer::tokenize(std::istream &code,
const char FileName[],
const std::string &configuration,
const bool preprocessorCondition)
const std::string &configuration)
{
// make sure settings specified
assert(_settings);
@ -1743,11 +1742,9 @@ bool Tokenizer::tokenize(std::istream &code,
// ";a+=b;" => ";a=a+b;"
simplifyCompoundAssignment();
if (!preprocessorCondition) {
if (hasComplicatedSyntaxErrorsInTemplates()) {
list.deallocateTokens();
return false;
}
if (hasComplicatedSyntaxErrorsInTemplates()) {
list.deallocateTokens();
return false;
}
simplifyDefaultAndDeleteInsideClass();
@ -2002,20 +1999,18 @@ bool Tokenizer::tokenize(std::istream &code,
// Split up variable declarations.
simplifyVarDecl(false);
if (!preprocessorCondition) {
if (m_timerResults) {
Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults);
setVarId();
} else {
setVarId();
}
createLinks2();
// Change initialisation of variable to assignment
simplifyInitVar();
if (m_timerResults) {
Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults);
setVarId();
} else {
setVarId();
}
createLinks2();
// Change initialisation of variable to assignment
simplifyInitVar();
// Convert e.g. atol("0") into 0
simplifyMathFunctions();
@ -2040,6 +2035,155 @@ bool Tokenizer::tokenize(std::istream &code,
}
//---------------------------------------------------------------------------
bool Tokenizer::tokenizeCondition(const std::string &code) {
assert(_settings);
// Fill the map _typeSize..
_typeSize.clear();
_typeSize["char"] = 1;
_typeSize["bool"] = _settings->sizeof_bool;
_typeSize["short"] = _settings->sizeof_short;
_typeSize["int"] = _settings->sizeof_int;
_typeSize["long"] = _settings->sizeof_long;
_typeSize["float"] = _settings->sizeof_float;
_typeSize["double"] = _settings->sizeof_double;
_typeSize["wchar_t"] = _settings->sizeof_wchar_t;
_typeSize["size_t"] = _settings->sizeof_size_t;
_typeSize["*"] = _settings->sizeof_pointer;
{
std::istringstream istr(code);
if (!list.createTokens(istr, "")) {
cppcheckError(0);
return false;
}
}
// Combine wide strings
for (Token *tok = list.front();
tok;
tok = tok->next()) {
while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
// Combine 'L "string"'
tok->str(tok->next()->str());
tok->deleteNext();
tok->isLong(true);
}
}
// Combine strings
for (Token *tok = list.front();
tok;
tok = tok->next()) {
if (tok->str()[0] != '"')
continue;
tok->str(simplifyString(tok->str()));
while (tok->next() && tok->next()->type() == Token::eString) {
tok->next()->str(simplifyString(tok->next()->str()));
// Two strings after each other, combine them
tok->concatStr(tok->next()->str());
tok->deleteNext();
}
}
// Remove "volatile", "inline", "register", and "restrict"
simplifyKeyword();
// convert platform dependent types to standard types
// 32 bits: size_t -> unsigned long
// 64 bits: size_t -> unsigned long long
simplifyPlatformTypes();
// collapse compound standard types into a single token
// unsigned long long int => long _isUnsigned=true,_isLong=true
simplifyStdType();
// Concatenate double sharp: 'a ## b' -> 'ab'
concatenateDoubleSharp();
if (!createLinks()) {
// Source has syntax errors, can't proceed
return false;
}
// replace 'NULL' and similar '0'-defined macros with '0'
simplifyNull();
// replace 'sin(0)' to '0' and other similar math expressions
simplifyMathExpressions();
// combine "- %num%"
concatenateNegativeNumberAndAnyPositive();
// simplify simple calculations
for (Token *tok = list.front() ? list.front()->next() : NULL;
tok;
tok = tok->next()) {
if (tok->isNumber())
TemplateSimplifier::simplifyNumericCalculations(tok->previous());
}
// Combine tokens..
for (Token *tok = list.front();
tok && tok->next();
tok = tok->next()) {
const char c1 = tok->str()[0];
if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
const char c2 = tok->next()->str()[0];
// combine +-*/ and =
if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) {
tok->str(tok->str() + c2);
tok->deleteNext();
continue;
}
// replace "->" with "."
else if (c1 == '-' && c2 == '>') {
tok->str(".");
tok->deleteNext();
continue;
}
}
else if (tok->str() == ">>" && tok->next()->str() == "=") {
tok->str(">>=");
tok->deleteNext();
}
else if (tok->str() == "<<" && tok->next()->str() == "=") {
tok->str("<<=");
tok->deleteNext();
}
else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
tok->str(tok->str() + ":");
tok->deleteNext();
continue;
}
}
}
simplifyRedundantParentheses();
for (Token *tok = list.front();
tok;
tok = tok->next())
while (TemplateSimplifier::simplifyNumericCalculations(tok));
while (simplifyLogicalOperators()) { }
// Convert e.g. atol("0") into 0
simplifyMathFunctions();
simplifyDoublePlusAndDoubleMinus();
return true;
}
bool Tokenizer::hasComplicatedSyntaxErrorsInTemplates()
{
const Token *tok = TemplateSimplifier::hasComplicatedSyntaxErrorsInTemplates(list.front());

10
lib/tokenize.h
View File

@ -89,8 +89,14 @@ public:
*/
bool tokenize(std::istream &code,
const char FileName[],
const std::string &configuration = "",
const bool preprocessorCondition = false);
const std::string &configuration = "");
/**
* tokenize condition and run simple simplifications on it
* @param code code
* @return true if success.
*/
bool tokenizeCondition(const std::string &code);
/** Set variable id */
void setVarId();

4
test/testtokenize.cpp
View File

@ -5285,7 +5285,7 @@ private:
Settings settings;
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.tokenize(istr, "test.cpp", "", false);
tokenizer.tokenize(istr, "test.cpp", "");
ASSERT_EQUALS(true, tokenizer.validate());
}
@ -5296,7 +5296,7 @@ private:
Settings settings;
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.tokenize(istr, "test.cpp", "", false);
tokenizer.tokenize(istr, "test.cpp", "");
ASSERT_EQUALS(true, tokenizer.validate());
}