Add exprId to tokens (#2744)

lib/symboldatabase.cpp

@@ -21,6 +21,7 @@
 
 #include "astutils.h"
 #include "errorlogger.h"
+#include "mathlib.h"
 #include "platform.h"
 #include "settings.h"
 #include "token.h"
@@ -34,6 +35,7 @@
 #include <climits>
 #include <iomanip>
 #include <iostream>
+#include <unordered_map>
 //---------------------------------------------------------------------------
 
 SymbolDatabase::SymbolDatabase(const Tokenizer *tokenizer, const Settings *settings, ErrorLogger *errorLogger)
@@ -69,6 +71,7 @@ SymbolDatabase::SymbolDatabase(const Tokenizer *tokenizer, const Settings *setti
     createSymbolDatabaseEnums();
     createSymbolDatabaseEscapeFunctions();
     createSymbolDatabaseIncompleteVars();
+    createSymbolDatabaseExprIds();
 }
 
 static const Token* skipScopeIdentifiers(const Token* tok)
@@ -1425,6 +1428,49 @@ void SymbolDatabase::createSymbolDatabaseEscapeFunctions()
     }
 }
 
+void SymbolDatabase::createSymbolDatabaseExprIds()
+{
+    MathLib::bigint base = 0;
+    // Find highest varId
+    for (const Variable *var : mVariableList) {
+        if (!var)
+            continue;
+        base = std::max<MathLib::bigint>(base, var->declarationId());
+    }
+    MathLib::bigint id = base+1;
+    for (const Scope * scope : functionScopes) {
+        std::unordered_map<std::string, std::vector<Token*>> exprs;
+
+        // Assign IDs
+        for (Token* tok = const_cast<Token*>(scope->bodyStart); tok != scope->bodyEnd; tok = tok->next()) {
+            if (tok->varId() > 0) {
+                tok->exprId(tok->varId());
+            } else if (Token::Match(tok, "(|.|%cop%")) {
+                exprs[tok->str()].push_back(tok);
+                tok->exprId(id++);
+            }
+        }
+
+        // Apply CSE
+        for(const auto& p:exprs) {
+            const std::vector<Token*>& tokens = p.second;
+            for(Token* tok1:tokens) {
+                for(Token* tok2:tokens) {
+                    if (tok1 == tok2)
+                        continue;
+                    if (tok1->exprId() == tok2->exprId())
+                        continue;
+                    if (!isSameExpression(isCPP(), true, tok1, tok2, mSettings->library, true, false))
+                        continue;
+                    MathLib::bigint cid = std::min(tok1->exprId(), tok2->exprId());
+                    tok1->exprId(cid);
+                    tok2->exprId(cid);
+                }
+            }
+        }
+    }
+}
+
 void SymbolDatabase::setArrayDimensionsUsingValueFlow()
 {
     // set all unknown array dimensions

lib/symboldatabase.h

@@ -1350,6 +1350,7 @@ private:
     void createSymbolDatabaseEnums();
     void createSymbolDatabaseEscapeFunctions();
     void createSymbolDatabaseIncompleteVars();
+    void createSymbolDatabaseExprIds();
 
     void addClassFunction(Scope **scope, const Token **tok, const Token *argStart);
     Function *addGlobalFunctionDecl(Scope*& scope, const Token* tok, const Token *argStart, const Token* funcStart);

lib/token.cpp

@@ -1140,14 +1140,14 @@ void Token::printOut(const char *title) const
 {
     if (title && title[0])
         std::cout << "\n### " << title << " ###\n";
-    std::cout << stringifyList(true, true, true, true, true, nullptr, nullptr) << std::endl;
+    std::cout << stringifyList(stringifyOptions::forDebugExprId(), nullptr, nullptr) << std::endl;
 }
 
 void Token::printOut(const char *title, const std::vector<std::string> &fileNames) const
 {
     if (title && title[0])
         std::cout << "\n### " << title << " ###\n";
-    std::cout << stringifyList(true, true, true, true, true, &fileNames, nullptr) << std::endl;
+    std::cout << stringifyList(stringifyOptions::forDebugExprId(), &fileNames, nullptr) << std::endl;
 }
 
 void Token::printLines(int lines) const
@@ -1155,12 +1155,12 @@ void Token::printLines(int lines) const
     const Token *end = this;
     while (end && end->linenr() < lines + linenr())
         end = end->next();
-    std::cout << stringifyList(true, true, true, true, true, nullptr, end) << std::endl;
+    std::cout << stringifyList(stringifyOptions::forDebugExprId(), nullptr, end) << std::endl;
 }
 
-void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro) const
+void Token::stringify(std::ostream& os, const stringifyOptions& options) const
 {
-    if (attributes) {
+    if (options.attributes) {
         if (isUnsigned())
             os << "unsigned ";
         else if (isSigned())
@@ -1172,7 +1172,7 @@ void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro)
                 os << "long ";
         }
     }
-    if (macro && isExpandedMacro())
+    if (options.macro && isExpandedMacro())
         os << "$";
     if (isName() && mStr.find(' ') != std::string::npos) {
         for (char i : mStr) {
@@ -1189,19 +1189,30 @@ void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro)
                 os << i;
         }
     }
-    if (varid && mImpl->mVarId != 0)
+    if (options.varid && mImpl->mVarId != 0)
         os << '@' << mImpl->mVarId;
+    if (options.exprid && mImpl->mExprId != 0)
+        os << '@' << mImpl->mExprId;
 }
 
-std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers, bool linebreaks, bool files, const std::vector<std::string>* fileNames, const Token* end) const
+void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro) const
+{
+    stringifyOptions options;
+    options.varid = varid;
+    options.attributes = attributes;
+    options.macro = macro;
+    stringify(os, options);
+}
+
+std::string Token::stringifyList(const stringifyOptions& options, const std::vector<std::string>* fileNames, const Token* end) const
 {
     if (this == end)
         return "";
 
     std::ostringstream ret;
 
-    unsigned int lineNumber = mImpl->mLineNumber - (linenumbers ? 1U : 0U);
+    unsigned int lineNumber = mImpl->mLineNumber - (options.linenumbers ? 1U : 0U);
-    unsigned int fileIndex = files ? ~0U : mImpl->mFileIndex;
+    unsigned int fileIndex = options.files ? ~0U : mImpl->mFileIndex;
     std::map<int, unsigned int> lineNumbers;
     for (const Token *tok = this; tok != end; tok = tok->next()) {
         bool fileChange = false;
@@ -1211,7 +1222,7 @@ std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers,
             }
 
             fileIndex = tok->mImpl->mFileIndex;
-            if (files) {
+            if (options.files) {
                 ret << "\n\n##file ";
                 if (fileNames && fileNames->size() > tok->mImpl->mFileIndex)
                     ret << fileNames->at(tok->mImpl->mFileIndex);
@@ -1224,18 +1235,18 @@ std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers,
             fileChange = true;
         }
 
-        if (linebreaks && (lineNumber != tok->linenr() || fileChange)) {
+        if (options.linebreaks && (lineNumber != tok->linenr() || fileChange)) {
             if (lineNumber+4 < tok->linenr() && fileIndex == tok->mImpl->mFileIndex) {
                 ret << '\n' << lineNumber+1 << ":\n|\n";
                 ret << tok->linenr()-1 << ":\n";
                 ret << tok->linenr() << ": ";
-            } else if (this == tok && linenumbers) {
+            } else if (this == tok && options.linenumbers) {
                 ret << tok->linenr() << ": ";
             } else {
                 while (lineNumber < tok->linenr()) {
                     ++lineNumber;
                     ret << '\n';
-                    if (linenumbers) {
+                    if (options.linenumbers) {
                         ret << lineNumber << ':';
                         if (lineNumber == tok->linenr())
                             ret << ' ';
@@ -1245,14 +1256,25 @@ std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers,
             lineNumber = tok->linenr();
         }
 
-        tok->stringify(ret, varid, attributes, attributes); // print token
+        tok->stringify(ret, options); // print token
-        if (tok->next() != end && (!linebreaks || (tok->next()->linenr() <= tok->linenr() && tok->next()->fileIndex() == tok->fileIndex())))
+        if (tok->next() != end && (!options.linebreaks || (tok->next()->linenr() <= tok->linenr() && tok->next()->fileIndex() == tok->fileIndex())))
             ret << ' ';
     }
-    if (linebreaks && (files || linenumbers))
+    if (options.linebreaks && (options.files || options.linenumbers))
         ret << '\n';
     return ret.str();
 }
+std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers, bool linebreaks, bool files, const std::vector<std::string>* fileNames, const Token* end) const
+{
+    stringifyOptions options;
+    options.varid = varid;
+    options.attributes = attributes;
+    options.macro = attributes;
+    options.linenumbers = linenumbers;
+    options.linebreaks = linebreaks;
+    options.files = files;
+    return stringifyList(options, fileNames, end);
+}
 
 std::string Token::stringifyList(const Token* end, bool attributes) const
 {

lib/token.h

@@ -65,6 +65,7 @@ struct TokenImpl {
     nonneg int mFileIndex;
     nonneg int mLineNumber;
     nonneg int mColumn;
+    MathLib::bigint mExprId;
 
     // AST..
     Token *mAstOperand1;
@@ -129,6 +130,7 @@ struct TokenImpl {
         , mFileIndex(0)
         , mLineNumber(0)
         , mColumn(0)
+        , mExprId(0)
         , mAstOperand1(nullptr)
         , mAstOperand2(nullptr)
         , mAstParent(nullptr)
@@ -790,6 +792,13 @@ public:
         }
     }
 
+    MathLib::bigint exprId() const {
+        return mImpl->mExprId;
+    }
+    void exprId(MathLib::bigint id) {
+        mImpl->mExprId = id;
+    }
+
     /**
      * For debugging purposes, prints token and all tokens
      * followed by it.
@@ -822,6 +831,35 @@ public:
      */
     static void replace(Token *replaceThis, Token *start, Token *end);
 
+    struct stringifyOptions {
+        bool varid = false;
+        bool exprid = false;
+        bool attributes = false;
+        bool macro = false;
+        bool linenumbers = false;
+        bool linebreaks = false;
+        bool files = false;
+        static stringifyOptions forDebug() {
+            stringifyOptions options;
+            options.attributes = true;
+            options.macro = true;
+            options.linenumbers = true;
+            options.linebreaks = true;
+            options.files = true;
+            return options;
+        }
+        static stringifyOptions forDebugVarId() {
+            stringifyOptions options = forDebug();
+            options.varid = true;
+            return options;
+        }
+        static stringifyOptions forDebugExprId() {
+            stringifyOptions options = forDebug();
+            options.exprid = true;
+            return options;
+        }
+    };
+
     /**
      * Stringify a token
      * @param os The result is shifted into that output stream
@@ -829,6 +867,7 @@ public:
      * @param attributes Print attributes of tokens like "unsigned" in front of it.
      * @param macro Prints $ in front of the token if it was expanded from a macro.
      */
+    void stringify(std::ostream& os, const stringifyOptions& options) const;
     void stringify(std::ostream& os, bool varid, bool attributes, bool macro) const;
 
     /**
@@ -842,6 +881,7 @@ public:
      * @param end Stringification ends before this token is reached. 0 to stringify until end of list.
      * @return Stringified token list as a string
      */
+    std::string stringifyList(const stringifyOptions& options, const std::vector<std::string>* fileNames = nullptr, const Token* end = nullptr) const;
     std::string stringifyList(bool varid, bool attributes, bool linenumbers, bool linebreaks, bool files, const std::vector<std::string>* fileNames = nullptr, const Token* end = nullptr) const;
     std::string stringifyList(const Token* end, bool attributes = true) const;
     std::string stringifyList(bool varid = false) const;

test/testvarid.cpp

@@ -202,6 +202,8 @@ private:
         TEST_CASE(setVarIdStructMembers1);
 
         TEST_CASE(decltype1);
+        
+        TEST_CASE(exprid1);
     }
 
     std::string tokenize(const char code[], const char filename[] = "test.cpp") {
@@ -217,7 +219,27 @@ private:
         tokenizer.tokenize(istr, filename);
 
         // result..
-        return tokenizer.tokens()->stringifyList(true,true,true,true,false);
+        Token::stringifyOptions options = Token::stringifyOptions::forDebugVarId();
+        options.files = false;
+        return tokenizer.tokens()->stringifyList(options);
+    }
+
+    std::string tokenizeExpr(const char code[], const char filename[] = "test.cpp") {
+        errout.str("");
+
+        Settings settings;
+        settings.platform(Settings::Unix64);
+        settings.standards.c   = Standards::C89;
+        settings.standards.cpp = Standards::CPP11;
+
+        Tokenizer tokenizer(&settings, this);
+        std::istringstream istr(code);
+        tokenizer.tokenize(istr, filename);
+
+        // result..
+        Token::stringifyOptions options = Token::stringifyOptions::forDebugExprId();
+        options.files = false;
+        return tokenizer.tokens()->stringifyList(options);
     }
 
     std::string compareVaridsForVariable(const char code[], const char varname[], const char filename[] = "test.cpp") {
@@ -3179,6 +3201,29 @@ private:
         const char expected[] = "1: void foo ( int x@1 , decltype ( A :: b ) * p@2 ) ;\n";
         ASSERT_EQUALS(expected, tokenize(code));
     }
+
+    void exprid1() {
+        const std::string actual = tokenizeExpr(
+                                       "struct A {\n"
+                                       "    int x, y;\n"
+                                       "};\n"
+                                       "int f(A a, A b) {\n"
+                                       "    int x = a.x + b.x;\n"
+                                       "    int y = b.x + a.x;\n"
+                                       "    return x + y + a.y + b.y;\n"
+                                       "}\n");
+
+        const char expected[] = "1: struct A {\n"
+                                "2: int x ; int y ;\n"
+                                "3: } ;\n"
+                                "4: int f ( A a , A b ) {\n"
+                                "5: int x@5 ; x@5 = a@3 .@9 x@6 +@10 b@4 .@11 x@7 ;\n"
+                                "6: int y@8 ; y@8 = b@4 .@11 x@7 +@10 a@3 .@9 x@6 ;\n"
+                                "7: return x@5 +@15 y@8 +@16 a@3 .@17 y@9 +@18 b@4 .@19 y@10 ;\n"
+                                "8: }\n";
+
+        ASSERT_EQUALS(expected, actual);
+    }
 };
 
 REGISTER_TEST(TestVarID)