diff --git a/lib/preprocessor.cpp b/lib/preprocessor.cpp index 91346ef47..bb2643e5f 100644 --- a/lib/preprocessor.cpp +++ b/lib/preprocessor.cpp @@ -1335,8 +1335,7 @@ std::list Preprocessor::getcfgs(const std::string &filedata, const if (s.find("&&") != std::string::npos) { Tokenizer tokenizer(_settings, _errorLogger); - std::istringstream tempIstr(s); - if (!tokenizer.tokenize(tempIstr, filename.c_str(), "", true)) { + if (!tokenizer.tokenizeCondition(s)) { std::ostringstream lineStream; lineStream << __LINE__; @@ -1444,8 +1443,7 @@ void Preprocessor::simplifyCondition(const std::map &c { const Settings settings; Tokenizer tokenizer(&settings, _errorLogger); - std::istringstream istr("(" + condition + ")"); - if (!tokenizer.tokenize(istr, "", "", true)) { + if (!tokenizer.tokenizeCondition("(" + condition + ")")) { // If tokenize returns false, then there is syntax error in the // code which we can't handle. So stop here. return; @@ -1505,8 +1503,7 @@ void Preprocessor::simplifyCondition(const std::map &c if (!it->second.empty()) { // Tokenize the value Tokenizer tokenizer2(&settings,NULL); - std::istringstream istr2(it->second); - tokenizer2.tokenize(istr2,"","",true); + tokenizer2.tokenizeCondition(it->second); // Copy the value tokens std::stack link; diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 36b96caa3..27a9ba8c5 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -1554,8 +1554,7 @@ void Tokenizer::simplifyMulAndParens() bool Tokenizer::tokenize(std::istream &code, const char FileName[], - const std::string &configuration, - const bool preprocessorCondition) + const std::string &configuration) { // make sure settings specified assert(_settings); @@ -1743,11 +1742,9 @@ bool Tokenizer::tokenize(std::istream &code, // ";a+=b;" => ";a=a+b;" simplifyCompoundAssignment(); - if (!preprocessorCondition) { - if (hasComplicatedSyntaxErrorsInTemplates()) { - list.deallocateTokens(); - return false; - } + if (hasComplicatedSyntaxErrorsInTemplates()) { + list.deallocateTokens(); + return false; } simplifyDefaultAndDeleteInsideClass(); @@ -2002,20 +1999,18 @@ bool Tokenizer::tokenize(std::istream &code, // Split up variable declarations. simplifyVarDecl(false); - if (!preprocessorCondition) { - if (m_timerResults) { - Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults); - setVarId(); - } else { - setVarId(); - } - - createLinks2(); - - // Change initialisation of variable to assignment - simplifyInitVar(); + if (m_timerResults) { + Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults); + setVarId(); + } else { + setVarId(); } + createLinks2(); + + // Change initialisation of variable to assignment + simplifyInitVar(); + // Convert e.g. atol("0") into 0 simplifyMathFunctions(); @@ -2040,6 +2035,155 @@ bool Tokenizer::tokenize(std::istream &code, } //--------------------------------------------------------------------------- +bool Tokenizer::tokenizeCondition(const std::string &code) { + assert(_settings); + + // Fill the map _typeSize.. + _typeSize.clear(); + _typeSize["char"] = 1; + _typeSize["bool"] = _settings->sizeof_bool; + _typeSize["short"] = _settings->sizeof_short; + _typeSize["int"] = _settings->sizeof_int; + _typeSize["long"] = _settings->sizeof_long; + _typeSize["float"] = _settings->sizeof_float; + _typeSize["double"] = _settings->sizeof_double; + _typeSize["wchar_t"] = _settings->sizeof_wchar_t; + _typeSize["size_t"] = _settings->sizeof_size_t; + _typeSize["*"] = _settings->sizeof_pointer; + + { + std::istringstream istr(code); + if (!list.createTokens(istr, "")) { + cppcheckError(0); + return false; + } + } + + // Combine wide strings + for (Token *tok = list.front(); + tok; + tok = tok->next()) { + while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) { + // Combine 'L "string"' + tok->str(tok->next()->str()); + tok->deleteNext(); + tok->isLong(true); + } + } + + // Combine strings + for (Token *tok = list.front(); + tok; + tok = tok->next()) { + if (tok->str()[0] != '"') + continue; + + tok->str(simplifyString(tok->str())); + while (tok->next() && tok->next()->type() == Token::eString) { + tok->next()->str(simplifyString(tok->next()->str())); + + // Two strings after each other, combine them + tok->concatStr(tok->next()->str()); + tok->deleteNext(); + } + } + + // Remove "volatile", "inline", "register", and "restrict" + simplifyKeyword(); + + // convert platform dependent types to standard types + // 32 bits: size_t -> unsigned long + // 64 bits: size_t -> unsigned long long + simplifyPlatformTypes(); + + // collapse compound standard types into a single token + // unsigned long long int => long _isUnsigned=true,_isLong=true + simplifyStdType(); + + // Concatenate double sharp: 'a ## b' -> 'ab' + concatenateDoubleSharp(); + + if (!createLinks()) { + // Source has syntax errors, can't proceed + return false; + } + + // replace 'NULL' and similar '0'-defined macros with '0' + simplifyNull(); + + // replace 'sin(0)' to '0' and other similar math expressions + simplifyMathExpressions(); + + // combine "- %num%" + concatenateNegativeNumberAndAnyPositive(); + + // simplify simple calculations + for (Token *tok = list.front() ? list.front()->next() : NULL; + tok; + tok = tok->next()) { + if (tok->isNumber()) + TemplateSimplifier::simplifyNumericCalculations(tok->previous()); + } + + // Combine tokens.. + for (Token *tok = list.front(); + tok && tok->next(); + tok = tok->next()) { + const char c1 = tok->str()[0]; + + if (tok->str().length() == 1 && tok->next()->str().length() == 1) { + const char c2 = tok->next()->str()[0]; + + // combine +-*/ and = + if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) { + tok->str(tok->str() + c2); + tok->deleteNext(); + continue; + } + + // replace "->" with "." + else if (c1 == '-' && c2 == '>') { + tok->str("."); + tok->deleteNext(); + continue; + } + } + + else if (tok->str() == ">>" && tok->next()->str() == "=") { + tok->str(">>="); + tok->deleteNext(); + } + + else if (tok->str() == "<<" && tok->next()->str() == "=") { + tok->str("<<="); + tok->deleteNext(); + } + + else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") { + if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") { + tok->str(tok->str() + ":"); + tok->deleteNext(); + continue; + } + } + } + + simplifyRedundantParentheses(); + for (Token *tok = list.front(); + tok; + tok = tok->next()) + while (TemplateSimplifier::simplifyNumericCalculations(tok)); + + while (simplifyLogicalOperators()) { } + + // Convert e.g. atol("0") into 0 + simplifyMathFunctions(); + + simplifyDoublePlusAndDoubleMinus(); + + return true; +} + bool Tokenizer::hasComplicatedSyntaxErrorsInTemplates() { const Token *tok = TemplateSimplifier::hasComplicatedSyntaxErrorsInTemplates(list.front()); diff --git a/lib/tokenize.h b/lib/tokenize.h index b1393a03b..a52f05a0e 100644 --- a/lib/tokenize.h +++ b/lib/tokenize.h @@ -89,8 +89,14 @@ public: */ bool tokenize(std::istream &code, const char FileName[], - const std::string &configuration = "", - const bool preprocessorCondition = false); + const std::string &configuration = ""); + + /** + * tokenize condition and run simple simplifications on it + * @param code code + * @return true if success. + */ + bool tokenizeCondition(const std::string &code); /** Set variable id */ void setVarId(); diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index ffbf55bf8..34366fd69 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -5285,7 +5285,7 @@ private: Settings settings; Tokenizer tokenizer(&settings, this); std::istringstream istr(code); - tokenizer.tokenize(istr, "test.cpp", "", false); + tokenizer.tokenize(istr, "test.cpp", ""); ASSERT_EQUALS(true, tokenizer.validate()); } @@ -5296,7 +5296,7 @@ private: Settings settings; Tokenizer tokenizer(&settings, this); std::istringstream istr(code); - tokenizer.tokenize(istr, "test.cpp", "", false); + tokenizer.tokenize(istr, "test.cpp", ""); ASSERT_EQUALS(true, tokenizer.validate()); }