Revert "Cleanup: Removed Tokenizer::simplifyTokenList2. As a side-effect, rules for "simple" token list are now executed on normal token list." (#2666)

This reverts commit 187cde183d.

cli/cmdlineparser.cpp

@@ -258,6 +258,10 @@ bool CmdLineParser::parseFromArgs(int argc, const char* const argv[])
             else if (std::strcmp(argv[i], "--debug-bug-hunting") == 0)
                 mSettings->debugBugHunting = true;
 
+            // Flag used for various purposes during debugging
+            else if (std::strcmp(argv[i], "--debug-simplified") == 0)
+                mSettings->debugSimplified = true;
+
             // Show template information
             else if (std::strcmp(argv[i], "--debug-template") == 0)
                 mSettings->debugtemplate = true;

lib/cppcheck.cpp

@@ -249,6 +249,7 @@ CppCheck::CppCheck(ErrorLogger &errorLogger,
     , mSuppressInternalErrorFound(false)
     , mUseGlobalSuppressions(useGlobalSuppressions)
     , mTooManyConfigs(false)
+    , mSimplify(true)
     , mExecuteCommand(executeCommand)
 {
 }
@@ -384,7 +385,7 @@ unsigned int CppCheck::check(const std::string &path)
         clangimport::parseClangAstDump(&tokenizer, ast);
         ValueFlow::setValues(&tokenizer.list, const_cast<SymbolDatabase *>(tokenizer.getSymbolDatabase()), this, &mSettings);
         if (mSettings.debugnormal)
-            tokenizer.printDebugOutput();
+            tokenizer.printDebugOutput(1);
         checkNormalTokens(tokenizer);
         return mExitCode;
     }
@@ -755,8 +756,18 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
                     checkUnusedFunctions.parseTokens(mTokenizer, filename.c_str(), &mSettings);
 
                 // simplify more if required, skip rest of iteration if failed
-                if (hasRule("simple")) {
-                    std::cout << "Handling of \"simple\" rules was removed in Cppcheck 2.1. Rule is executed on normal token list instead." << std::endl;
+                if (mSimplify && hasRule("simple")) {
+                    std::cout << "Handling of \"simple\" rules is deprecated and will be removed in Cppcheck 2.5." << std::endl;
+
+                    // if further simplification fails then skip rest of iteration
+                    Timer timer3("Tokenizer::simplifyTokenList2", mSettings.showtime, &s_timerResults);
+                    result = mTokenizer.simplifyTokenList2();
+                    timer3.stop();
+                    if (!result)
+                        continue;
+
+                    if (!Settings::terminated())
+                        executeRules("simple", mTokenizer);
                 }
 
             } catch (const simplecpp::Output &o) {
@@ -960,7 +971,6 @@ void CppCheck::checkNormalTokens(const Tokenizer &tokenizer)
         }
 
         executeRules("normal", tokenizer);
-        executeRules("simple", tokenizer);
     }
 }
 

lib/cppcheck.h

@@ -128,6 +128,10 @@ public:
     void tooManyConfigsError(const std::string &file, const std::size_t numberOfConfigurations);
     void purgedConfigurationMessage(const std::string &file, const std::string& configuration);
 
+    void dontSimplify() {
+        mSimplify = false;
+    }
+
     /** Analyse whole program, run this after all TUs has been scanned.
      * This is deprecated and the plan is to remove this when
      * .analyzeinfo is good enough.
@@ -223,6 +227,9 @@ private:
     /** Are there too many configs? */
     bool mTooManyConfigs;
 
+    /** Simplify code? true by default */
+    bool mSimplify;
+
     /** File info used for whole program analysis */
     std::list<Check::FileInfo*> mFileInfo;
 

lib/settings.cpp

@@ -41,6 +41,7 @@ Settings::Settings()
       daca(false),
       debugBugHunting(false),
       debugnormal(false),
+      debugSimplified(false),
       debugtemplate(false),
       debugwarnings(false),
       dump(false),

lib/settings.h

@@ -130,6 +130,9 @@ public:
     /** @brief Is --debug-normal given? */
     bool debugnormal;
 
+    /** @brief Is --debug-simplified given? */
+    bool debugSimplified;
+
     /** @brief Is --debug-template given? */
     bool debugtemplate;
 

lib/tokenize.h

@@ -155,6 +155,14 @@ public:
     */
     bool simplifyTokenList1(const char FileName[]);
 
+    /**
+    * Most aggressive simplification of tokenlist
+    *
+    * @return false if there is an error that requires aborting
+    * the checking of this file.
+    */
+    bool simplifyTokenList2();
+
     /**
      * If --check-headers=no has been given; then remove unneeded code in headers.
      * - All executable code.
@@ -200,6 +208,10 @@ public:
      */
     bool isFunctionParameterPassedByValue(const Token *fpar) const;
 
+    /** Simplify assignment in function call "f(x=g());" => "x=g();f(x);"
+     */
+    void simplifyAssignmentInFunctionCall();
+
     /** Simplify assignment where rhs is a block : "x=({123;});" => "{x=123;}" */
     void simplifyAssignmentBlock();
 
@@ -210,6 +222,19 @@ public:
      */
     bool simplifyCalculations();
 
+    /**
+     * Simplify dereferencing a pointer offset by a number:
+     *     "*(ptr + num)" => "ptr[num]"
+     *     "*(ptr - num)" => "ptr[-num]"
+     */
+    void simplifyOffsetPointerDereference();
+
+    /**
+       * Simplify referencing a pointer offset:
+       *     "Replace "&str[num]" => "(str + num)"
+       */
+    void simplifyOffsetPointerReference();
+
     /** Insert array size where it isn't given */
     void arraySize();
 
@@ -235,9 +260,25 @@ public:
     /** Remove unknown macro in variable declarations: PROGMEM char x; */
     void removeMacroInVarDecl();
 
+    /** Remove redundant assignment */
+    void removeRedundantAssignment();
+
+    /** Simplifies some realloc usage like
+      * 'x = realloc (0, n);' => 'x = malloc(n);'
+      * 'x = realloc (y, 0);' => 'x = 0; free(y);'
+      */
+    void simplifyRealloc();
+
     /** Add parentheses for sizeof: sizeof x => sizeof(x) */
     void sizeofAddParentheses();
 
+    /**
+     * Replace sizeof() to appropriate size.
+     * @return true if modifications to token-list are done.
+     *         false if no modifications are done.
+     */
+    bool simplifySizeof();
+
     /**
      * Simplify variable declarations (split up)
      * \param only_k_r_fpar Only simplify K&R function parameters
@@ -252,6 +293,20 @@ public:
     void simplifyInitVar();
     Token * initVar(Token * tok);
 
+    /**
+     * Simplify easy constant '?:' operation
+     * Example: 0 ? (2/0) : 0 => 0
+     * @return true if something is modified
+     *         false if nothing is done.
+     */
+    bool simplifyConstTernaryOp();
+
+    /**
+     * Simplify compound assignments
+     * Example: ";a+=b;" => ";a=a+b;"
+     */
+    void simplifyCompoundAssignment();
+
     /**
      * Simplify the location of "static" and "const" qualifiers in
      * a variable declaration or definition.
@@ -260,6 +315,14 @@ public:
      */
     void simplifyStaticConst();
 
+    /**
+     * Simplify assignments in "if" and "while" conditions
+     * Example: "if(a=b);" => "a=b;if(a);"
+     * Example: "while(a=b) { f(a); }" => "a = b; while(a){ f(a); a = b; }"
+     * Example: "do { f(a); } while(a=b);" => "do { f(a); a = b; } while(a);"
+     */
+    void simplifyIfAndWhileAssign();
+
     /**
      * Simplify multiple assignments.
      * Example: "a = b = c = 0;" => "a = 0; b = 0; c = 0;"
@@ -275,6 +338,14 @@ public:
      */
     bool simplifyCAlternativeTokens();
 
+    /**
+     * Simplify comma into a semicolon when possible:
+     * - "delete a, delete b" => "delete a; delete b;"
+     * - "a = 0, b = 0;" => "a = 0; b = 0;"
+     * - "return a(), b;" => "a(); return b;"
+     */
+    void simplifyComma();
+
     /** Add braces to an if-block, for-block, etc.
      * @return true if no syntax errors
      */
@@ -317,18 +388,82 @@ public:
      */
     bool simplifyUsing();
 
+    /**
+     * Simplify casts
+     */
+    void simplifyCasts();
+
+    /**
+     * Change (multiple) arrays to (multiple) pointers.
+     */
+    void simplifyUndefinedSizeArray();
+
+    /**
+     * A simplify function that replaces a variable with its value in cases
+     * when the value is known. e.g. "x=10; if(x)" => "x=10;if(10)"
+     *
+     * @return true if modifications to token-list are done.
+     *         false if no modifications are done.
+     */
+    bool simplifyKnownVariables();
+
+    /**
+     * Utility function for simplifyKnownVariables. Get data about an
+     * assigned variable.
+     */
+    static bool simplifyKnownVariablesGetData(nonneg int varid, Token **_tok2, Token **_tok3, std::string &value, nonneg int &valueVarId, bool &valueIsPointer, bool floatvar);
+
+    /**
+     * utility function for simplifyKnownVariables. Perform simplification
+     * of a given variable
+     */
+    bool simplifyKnownVariablesSimplify(Token **tok2, Token *tok3, nonneg int varid, const std::string &structname, std::string &value, nonneg int valueVarId, bool valueIsPointer, const Token * const valueToken, int indentlevel) const;
+
     /** Simplify useless C++ empty namespaces, like: 'namespace %name% { }'*/
     void simplifyEmptyNamespaces();
 
+    /** Simplify redundant code placed after control flow statements :
+     * 'return', 'throw', 'goto', 'break' and 'continue'
+     */
+    void simplifyFlowControl();
+
+    /** Expand nested strcat() calls. */
+    void simplifyNestedStrcat();
+
     /** Simplify "if else" */
     void elseif();
 
+    /** Simplify conditions
+     * @return true if something is modified
+     *         false if nothing is done.
+     */
+    bool simplifyConditions();
+
+    /** Remove redundant code, e.g. if( false ) { int a; } should be
+     * removed, because it is never executed.
+     * @return true if something is modified
+     *         false if nothing is done.
+     */
+    bool removeRedundantConditions();
+
+    /**
+     * Remove redundant for:
+     * "for (x=0;x<1;x++) { }" => "{ x = 1; }"
+     */
+    void removeRedundantFor();
+
 
     /**
      * Reduces "; ;" to ";", except in "( ; ; )"
      */
     void removeRedundantSemicolons();
 
+    /** Simplify function calls - constant return value
+     * @return true if something is modified
+     *         false if nothing is done.
+     */
+    bool simplifyFunctionReturn();
+
     /** Struct simplification
      * "struct S { } s;" => "struct S { }; S s;"
      */
@@ -346,6 +481,11 @@ public:
      */
     bool simplifyRedundantParentheses();
 
+    void simplifyCharAt();
+
+    /** Simplify references */
+    void simplifyReference();
+
     /**
      * Simplify functions like "void f(x) int x; {"
      * into "void f(int x) {"
@@ -385,6 +525,11 @@ public:
 
     void findComplicatedSyntaxErrorsInTemplates();
 
+    /**
+     * Simplify e.g. 'atol("0")' into '0'
+     */
+    void simplifyMathFunctions();
+
     /**
      * Simplify e.g. 'sin(0)' into '0'
      */
@@ -417,6 +562,26 @@ public:
 
 private:
 
+    /**
+     * simplify "while (0)"
+     */
+    void simplifyWhile0();
+
+    /**
+     * Simplify while(func() && errno==EINTR)
+     */
+    void simplifyErrNoInWhile();
+
+    /**
+     * Simplify while(func(f))
+     */
+    void simplifyFuncInWhile();
+
+    /**
+     * Remove "std::" before some function names
+     */
+    void simplifyStd();
+
     /** Simplify pointer to standard type (C only) */
     void simplifyPointerToStandardType();
 
@@ -573,6 +738,12 @@ private:
     */
     void simplifyCPPAttribute();
 
+    /**
+     * Replace strlen(str)
+     * @return true if any replacement took place, false else
+     * */
+    bool simplifyStrlen();
+
     /**
      * Convert namespace aliases
      */
@@ -619,6 +790,12 @@ private:
                                std::map<int, std::map<std::string,int> >& structMembers,
                                nonneg int *varId_);
 
+    /**
+     * Simplify e.g. 'return(strncat(temp,"a",1));' into
+     * strncat(temp,"a",1); return temp;
+     */
+    void simplifyReturnStrncat();
+
     /**
      * Output list of unknown types.
      */
@@ -653,7 +830,7 @@ public:
      * 1=1st simplifications
      * 2=2nd simplifications
      */
-    void printDebugOutput() const;
+    void printDebugOutput(int simplification) const;
 
     void dump(std::ostream &out) const;
 

test/testcmdlineparser.cpp

@@ -52,6 +52,7 @@ private:
         TEST_CASE(optionwithoutfile);
         TEST_CASE(verboseshort);
         TEST_CASE(verboselong);
+        TEST_CASE(debugSimplified);
         TEST_CASE(debugwarnings);
         TEST_CASE(forceshort);
         TEST_CASE(forcelong);
@@ -223,6 +224,14 @@ private:
         ASSERT_EQUALS(true, settings.verbose);
     }
 
+    void debugSimplified() {
+        REDIRECT;
+        const char * const argv[] = {"cppcheck", "--debug-simplified", "file.cpp"};
+        settings.debugSimplified = false;
+        ASSERT(defParser.parseFromArgs(3, argv));
+        ASSERT_EQUALS(true, settings.debugSimplified);
+    }
+
     void debugwarnings() {
         REDIRECT;
         const char * const argv[] = {"cppcheck", "--debug-warnings", "file.cpp"};

test/testsimplifytypedef.cpp

@@ -2491,7 +2491,7 @@ private:
                             "Dummy<intvec> y;";
         const char exp [] = "Dummy < int [ 1 ] > y ;";
         ASSERT_EQUALS(exp, tok(code, false));
-        ASSERT_EQUALS("[test.cpp:2]: (debug) Unknown type 'Dummy<int[1]>'.\n", errout.str());
+        ASSERT_EQUALS("", errout.str());
     }
 
     void simplifyTypedef124() { // ticket #7792

test/testvalueflow.cpp

@@ -1456,8 +1456,7 @@ private:
                 "}\n");
         ASSERT_EQUALS_WITHOUT_LINENUMBERS(
             "[test.cpp:2]: (debug) valueflow.cpp:1035:valueFlowReverse bailout: assignment of abc\n"
-            "[test.cpp:8]: (debug) valueflow.cpp:1131:valueFlowReverse bailout: variable abc stopping on goto label\n"
-            "[test.cpp:2]: (debug) Unknown type 'ABC'.\n",
+            "[test.cpp:8]: (debug) valueflow.cpp:1131:valueFlowReverse bailout: variable abc stopping on goto label\n",
             errout.str());
     }