Tokenizer: Add special tokenize method for the Preprocessor with only basic simplifications

lib/preprocessor.cpp

@@ -1335,8 +1335,7 @@ std::list<std::string> Preprocessor::getcfgs(const std::string &filedata, const
 
         if (s.find("&&") != std::string::npos) {
             Tokenizer tokenizer(_settings, _errorLogger);
-            std::istringstream tempIstr(s);
+            if (!tokenizer.tokenizeCondition(s)) {
-            if (!tokenizer.tokenize(tempIstr, filename.c_str(), "", true)) {
                 std::ostringstream lineStream;
                 lineStream << __LINE__;
 
@@ -1444,8 +1443,7 @@ void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &c
 {
     const Settings settings;
     Tokenizer tokenizer(&settings, _errorLogger);
-    std::istringstream istr("(" + condition + ")");
+    if (!tokenizer.tokenizeCondition("(" + condition + ")")) {
-    if (!tokenizer.tokenize(istr, "", "", true)) {
         // If tokenize returns false, then there is syntax error in the
         // code which we can't handle. So stop here.
         return;
@@ -1505,8 +1503,7 @@ void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &c
             if (!it->second.empty()) {
                 // Tokenize the value
                 Tokenizer tokenizer2(&settings,NULL);
-                std::istringstream istr2(it->second);
+                tokenizer2.tokenizeCondition(it->second);
-                tokenizer2.tokenize(istr2,"","",true);
 
                 // Copy the value tokens
                 std::stack<Token *> link;

lib/tokenize.cpp

@@ -1554,8 +1554,7 @@ void Tokenizer::simplifyMulAndParens()
 
 bool Tokenizer::tokenize(std::istream &code,
                          const char FileName[],
-                         const std::string &configuration,
+                         const std::string &configuration)
-                         const bool preprocessorCondition)
 {
     // make sure settings specified
     assert(_settings);
@@ -1743,12 +1742,10 @@ bool Tokenizer::tokenize(std::istream &code,
     // ";a+=b;" => ";a=a+b;"
     simplifyCompoundAssignment();
 
-    if (!preprocessorCondition) {
     if (hasComplicatedSyntaxErrorsInTemplates()) {
         list.deallocateTokens();
         return false;
     }
-    }
 
     simplifyDefaultAndDeleteInsideClass();
 
@@ -2002,7 +1999,6 @@ bool Tokenizer::tokenize(std::istream &code,
     // Split up variable declarations.
     simplifyVarDecl(false);
 
-    if (!preprocessorCondition) {
     if (m_timerResults) {
         Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults);
         setVarId();
@@ -2014,7 +2010,6 @@ bool Tokenizer::tokenize(std::istream &code,
 
     // Change initialisation of variable to assignment
     simplifyInitVar();
-    }
 
     // Convert e.g. atol("0") into 0
     simplifyMathFunctions();
@@ -2040,6 +2035,155 @@ bool Tokenizer::tokenize(std::istream &code,
 }
 //---------------------------------------------------------------------------
 
+bool Tokenizer::tokenizeCondition(const std::string &code) {
+    assert(_settings);
+
+    // Fill the map _typeSize..
+    _typeSize.clear();
+    _typeSize["char"] = 1;
+    _typeSize["bool"] = _settings->sizeof_bool;
+    _typeSize["short"] = _settings->sizeof_short;
+    _typeSize["int"] = _settings->sizeof_int;
+    _typeSize["long"] = _settings->sizeof_long;
+    _typeSize["float"] = _settings->sizeof_float;
+    _typeSize["double"] = _settings->sizeof_double;
+    _typeSize["wchar_t"] = _settings->sizeof_wchar_t;
+    _typeSize["size_t"] = _settings->sizeof_size_t;
+    _typeSize["*"] = _settings->sizeof_pointer;
+
+    {
+        std::istringstream istr(code);
+        if (!list.createTokens(istr, "")) {
+            cppcheckError(0);
+            return false;
+        }
+    }
+
+    // Combine wide strings
+    for (Token *tok = list.front();
+         tok;
+         tok = tok->next()) {
+        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
+            // Combine 'L "string"'
+            tok->str(tok->next()->str());
+            tok->deleteNext();
+            tok->isLong(true);
+        }
+    }
+
+    // Combine strings
+    for (Token *tok = list.front();
+         tok;
+         tok = tok->next()) {
+        if (tok->str()[0] != '"')
+            continue;
+
+        tok->str(simplifyString(tok->str()));
+        while (tok->next() && tok->next()->type() == Token::eString) {
+            tok->next()->str(simplifyString(tok->next()->str()));
+
+            // Two strings after each other, combine them
+            tok->concatStr(tok->next()->str());
+            tok->deleteNext();
+        }
+    }
+
+    // Remove "volatile", "inline", "register", and "restrict"
+    simplifyKeyword();
+
+    // convert platform dependent types to standard types
+    // 32 bits: size_t -> unsigned long
+    // 64 bits: size_t -> unsigned long long
+    simplifyPlatformTypes();
+
+    // collapse compound standard types into a single token
+    // unsigned long long int => long _isUnsigned=true,_isLong=true
+    simplifyStdType();
+
+    // Concatenate double sharp: 'a ## b' -> 'ab'
+    concatenateDoubleSharp();
+
+    if (!createLinks()) {
+        // Source has syntax errors, can't proceed
+        return false;
+    }
+
+    // replace 'NULL' and similar '0'-defined macros with '0'
+    simplifyNull();
+
+    // replace 'sin(0)' to '0' and other similar math expressions
+    simplifyMathExpressions();
+
+    // combine "- %num%"
+    concatenateNegativeNumberAndAnyPositive();
+
+    // simplify simple calculations
+    for (Token *tok = list.front() ? list.front()->next() : NULL;
+         tok;
+         tok = tok->next()) {
+        if (tok->isNumber())
+            TemplateSimplifier::simplifyNumericCalculations(tok->previous());
+    }
+
+    // Combine tokens..
+    for (Token *tok = list.front();
+         tok && tok->next();
+         tok = tok->next()) {
+        const char c1 = tok->str()[0];
+
+        if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
+            const char c2 = tok->next()->str()[0];
+
+            // combine +-*/ and =
+            if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) {
+                tok->str(tok->str() + c2);
+                tok->deleteNext();
+                continue;
+            }
+
+            // replace "->" with "."
+            else if (c1 == '-' && c2 == '>') {
+                tok->str(".");
+                tok->deleteNext();
+                continue;
+            }
+        }
+
+        else if (tok->str() == ">>" && tok->next()->str() == "=") {
+            tok->str(">>=");
+            tok->deleteNext();
+        }
+
+        else if (tok->str() == "<<" && tok->next()->str() == "=") {
+            tok->str("<<=");
+            tok->deleteNext();
+        }
+
+        else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
+            if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
+                tok->str(tok->str() + ":");
+                tok->deleteNext();
+                continue;
+            }
+        }
+    }
+
+    simplifyRedundantParentheses();
+    for (Token *tok = list.front();
+         tok;
+         tok = tok->next())
+        while (TemplateSimplifier::simplifyNumericCalculations(tok));
+
+    while (simplifyLogicalOperators()) { }
+
+    // Convert e.g. atol("0") into 0
+    simplifyMathFunctions();
+
+    simplifyDoublePlusAndDoubleMinus();
+
+    return true;
+}
+
 bool Tokenizer::hasComplicatedSyntaxErrorsInTemplates()
 {
     const Token *tok = TemplateSimplifier::hasComplicatedSyntaxErrorsInTemplates(list.front());

lib/tokenize.h

@@ -89,8 +89,14 @@ public:
      */
     bool tokenize(std::istream &code,
                   const char FileName[],
-                  const std::string &configuration = "",
+                  const std::string &configuration = "");
-                  const bool preprocessorCondition = false);
+
+    /**
+     * tokenize condition and run simple simplifications on it
+     * @param code code
+     * @return true if success.
+     */
+    bool tokenizeCondition(const std::string &code);
 
     /** Set variable id */
     void setVarId();

test/testtokenize.cpp

@@ -5285,7 +5285,7 @@ private:
         Settings settings;
         Tokenizer tokenizer(&settings, this);
         std::istringstream istr(code);
-        tokenizer.tokenize(istr, "test.cpp", "", false);
+        tokenizer.tokenize(istr, "test.cpp", "");
         ASSERT_EQUALS(true, tokenizer.validate());
     }
 
@@ -5296,7 +5296,7 @@ private:
         Settings settings;
         Tokenizer tokenizer(&settings, this);
         std::istringstream istr(code);
-        tokenizer.tokenize(istr, "test.cpp", "", false);
+        tokenizer.tokenize(istr, "test.cpp", "");
         ASSERT_EQUALS(true, tokenizer.validate());
     }