Fixed #5700 (Defect: False positives due to failure to expand struct tag macros)

lib/preprocessor.cpp

@@ -2627,7 +2627,7 @@ private:
                         std::string innercode;
                         std::map<std::string,PreprocessorMacro *> innermacros = macros;
                         innermacros.erase(innerMacroName);
-                        innerMacro->code(innerparams, innermacros, innercode);
+                        innerMacro->code(innerparams, innerparams, innermacros, innercode);
                         params2[ipar] = innercode;
                     }
                 }
@@ -2708,12 +2708,13 @@ public:
 
     /**
      * get expanded code for this macro
-     * @param params2 macro parameters
+     * @param params2 completely expanded macro parameters
+     * @param params3 non-expanded macro parameters
      * @param macros macro definitions (recursion)
      * @param macrocode output string
      * @return true if the expanding was successful
      */
-    bool code(const std::vector<std::string> &params2, const std::map<std::string, PreprocessorMacro *> &macros, std::string &macrocode) const {
+    bool code(const std::vector<std::string> &params2, const std::vector<std::string> &params3, const std::map<std::string, PreprocessorMacro *> &macros, std::string &macrocode) const {
         if (_nopar || (_params.empty() && _variadic)) {
             macrocode = _macro.substr(1 + _macro.find(')'));
             if (macrocode.empty())
@@ -2768,6 +2769,7 @@ public:
         else {
             const std::vector<std::string> givenparams = expandInnerMacros(params2, macros);
 
+            bool noprescan = false;
             const Token *tok = tokens();
             while (tok && tok->str() != ")")
                 tok = tok->next();
@@ -2776,8 +2778,10 @@ public:
                 while (nullptr != (tok = tok->next())) {
                     std::string str = tok->str();
                     if (str[0] == '#' || tok->isName()) {
-                        if (str == "##")
+                        if (str == "##") {
+                            noprescan = true;
                             continue;
+                        }
 
                         const bool stringify(str[0] == '#');
                         if (stringify) {
@@ -2799,8 +2803,14 @@ public:
                                     // Macro had more parameters than caller used.
                                     macrocode = "";
                                     return false;
-                                } else
+                                } else {
-                                    str = givenparams[i];
+                                    if (noprescan) {
+                                        noprescan = false;
+                                        str = params3[i];
+                                    } else {
+                                        str = givenparams[i];
+                                    }
+                                }
 
                                 break;
                             }
@@ -3084,13 +3094,13 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
             // the macro again, the macro should not be expanded again.
             // The limits are used to prevent recursive expanding.
             // * When a macro is expanded its limit position is set to
-            //   the last expanded character.
+            //   the first expanded character.
             // * macros are only allowed to be expanded when the
-            //   the position is beyond the limit.
+            //   the position is before the limit.
-            // * The limit is relative to the end of the "line"
+            // * The limit is relative to the start of the "line"
-            //   variable. Inserting and deleting text before the limit
+            //   variable. Inserting and deleting text after the limit
             //   without updating the limit is safe.
-            // * when pos goes beyond a limit the limit needs to be
+            // * when base_pos is prior to a limit the limit needs to be
             //   deleted because it is unsafe to insert/delete text
             //   after the limit otherwise
             std::map<const PreprocessorMacro *, std::size_t> limits;
@@ -3098,9 +3108,54 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
             // pos is the current position in line
             std::string::size_type pos = 0;
 
+            // base_pos is the starting point of the current macro expansion
+            // macro expansion only commences once all macros have been scanned
+            // recursively and have been pushed on the macro expansion stack
+            std::string::size_type base_pos;
+
+            // Offset from the end of the line from where further macro
+            // expansion is not necessary
+            std::string::size_type end_pos = 0;
+
             // scan line to see if there are any macros to expand..
             unsigned int tmpLinenr = 0;
-            while (pos < line.size()) {
+
+            // The stack of starting positions of macros to be expanded along
+            // with their unexpanded parameters
+            std::stack<std::pair<std::string::size_type, std::vector<std::string>>> pos_stack;
+
+            // Flag that indicates whether we are scanning line or processing the macro stack
+            bool processing_stack = false;
+
+            std::vector<std::string> unexpanded_params;
+
+            while ((pos < (line.size() - end_pos)) || !(pos_stack.empty())) {
+                if (pos >= line.size() - end_pos) {
+                    // We are at the end of the line so start processing the stack
+                    pos = pos_stack.top().first;
+                    unexpanded_params = pos_stack.top().second;
+                    pos_stack.pop();
+                    if (processing_stack) {
+                        // Remove old limits
+                        for (std::map<const PreprocessorMacro *, std::size_t>::iterator iter = limits.begin();
+                             iter != limits.end();) {
+                            if (base_pos <= iter->second) {
+                                // We have gone past this limit range, so just delete it
+                                limits.erase(iter++);
+                            } else {
+                                ++iter;
+                            }
+                        }
+                    }
+                    base_pos = pos;
+                    if (pos >= (line.size() - end_pos)) {
+                        // The macro expansion popped off the stack exceeds the
+                        // end position for macro expansion so discard it
+                        continue;
+                    }
+                    processing_stack = true;
+                }
+
                 if (line[pos] == '\n')
                     ++tmpLinenr;
 
@@ -3133,7 +3188,7 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
 
                 // found an identifier..
                 // the "while" is used in case the expanded macro will immediately call another macro
-                while (pos < line.length() && (std::isalpha((unsigned char)line[pos]) || line[pos] == '_')) {
+                while ((pos < (line.size() - end_pos)) && (std::isalpha((unsigned char)line[pos]) || line[pos] == '_')) {
                     // pos1 = start position of macro
                     const std::string::size_type pos1 = pos++;
 
@@ -3156,14 +3211,22 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
                     // macro
                     {
                         const std::map<const PreprocessorMacro *, std::size_t>::const_iterator it2 = limits.find(macro);
-                        if (it2 != limits.end() && pos <= line.length() - it2->second)
+                        if (it2 != limits.end() && pos1 > it2->second) {
+                            // Update end_pos to avoid trying to expand this
+                            // macro again
+                            end_pos = line.size() - pos1;
                             break;
+                        }
                     }
 
-                    // get parameters from line..
-                    if (macro->params().size() && pos >= line.length())
-                        break;
                     std::vector<std::string> params;
+
+                    // get parameters from line..
+                    if (macro->params().size() && pos >= line.length()) {
+                        pos_stack.push(std::make_pair(pos1, params));
+                        break;
+                    }
+
                     std::string::size_type pos2 = pos;
 
                     // number of newlines within macro use
@@ -3176,9 +3239,23 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
 
                         getparams(line,pos2,params,numberOfNewlines,endFound);
 
-                        // something went wrong so bail out
+                        // something went wrong (eg. perhaps the parameters
-                        if (!endFound)
+                        // require a prescan?) so bail out but push the
+                        // expansion onto the stack assuming we'll be able to
+                        // get the parameters after a prescan
+                        if (!endFound && params.size() != 0) {
+                            pos_stack.push(std::make_pair(pos1, params));
+                            processing_stack = false;
                             break;
+                        }
+                    }
+                    pos_stack.push(std::make_pair(pos1, params));
+                    if (processing_stack) {
+                        // Expand macro parameters
+                        processing_stack = false;
+                    } else {
+                        // Continue scanning
+                        continue;
                     }
 
                     // Just an empty parameter => clear
@@ -3186,12 +3263,14 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
                         params.clear();
 
                     // Check that it's the same number of parameters..
-                    if (!macro->variadic() && params.size() != macro->params().size())
+                    if (!macro->variadic() && params.size() != macro->params().size()) {
+                        pos_stack.pop();
                         break;
+                    }
 
                     // Create macro code..
                     std::string tempMacro;
-                    if (!macro->code(params, macros, tempMacro)) {
+                    if (!macro->code(params, unexpanded_params, macros, tempMacro)) {
                         // Syntax error in code
                         writeError(filename,
                                    linenr + tmpLinenr,
@@ -3213,19 +3292,8 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
                     if (macro->variadic() || macro->nopar() || !macro->params().empty())
                         ++pos2;
 
-                    // Remove old limits
+                    // don't allow this macro to be expanded again after pos1
-                    for (std::map<const PreprocessorMacro *, std::size_t>::iterator iter = limits.begin();
+                    limits[macro] = pos1;
-                         iter != limits.end();) {
-                        if ((line.length() - pos1) < iter->second) {
-                            // We have gone past this limit, so just delete it
-                            limits.erase(iter++);
-                        } else {
-                            ++iter;
-                        }
-                    }
-
-                    // don't allow this macro to be expanded again before pos2
-                    limits[macro] = line.length() - pos2;
 
                     // erase macro
                     line.erase(pos1, pos2 - pos1);

test/testpreprocessor.cpp

@@ -187,6 +187,7 @@ private:
         TEST_CASE(macro_incdec);  // separate ++ and -- with space when expanding such macro: '#define M(X)  A-X'
         TEST_CASE(macro_switchCase);
         TEST_CASE(macro_NULL); // skip #define NULL .. it is replaced in the tokenizer
+        TEST_CASE(macro_prescan); // #7563: Macro parameter prescan
         TEST_CASE(string1);
         TEST_CASE(string2);
         TEST_CASE(string3);
@@ -1992,6 +1993,25 @@ private:
         ASSERT_EQUALS("\nNULL", OurPreprocessor::expandMacros("#define NULL 0\nNULL"));
     }
 
+    void macro_prescan() const {
+        const char filedata[] = "#define CONCAT_(a, b) a##b\n"
+                                "#define CONCAT(a, b) CONCAT_(a, b)\n"
+                                "#define VALUE(value) CONCAT(CONCAT(text1, value), text3),\n"
+                                "enum {\n"
+                                "VALUE(1)\n"
+                                "VALUE(2)\n"
+                                "VALUE(3)\n"
+                                "NUM_VALUES,\n"
+                                "};\n";
+        ASSERT_EQUALS("\n\n\nenum {\n"
+                      "$$$text11text3,\n"
+                      "$$$text12text3,\n"
+                      "$$$text13text3,\n"
+                      "NUM_VALUES,\n"
+                      "};\n",
+                      OurPreprocessor::expandMacros(filedata));
+    }
+
     void string1() {
         const char filedata[] = "int main()"
                                 "{"