- Refactorizations in preprocessor.cpp and tokenizer.cpp

- Bugfix: tok->stringify(tok) returns now "".
- Removed most of the inline-asm handling in preprocessor; improved the remaining handling of #asm/#endasm: Simplified to "asm(...);" statement

lib/preprocessor.cpp

@@ -266,7 +266,7 @@ std::string Preprocessor::preprocessCleanupDirectives(const std::string &process
 
 static bool hasbom(const std::string &str)
 {
-    return bool(str.size() > 3 &&
+    return bool(str.size() >= 3 &&
                 static_cast<unsigned char>(str[0]) == 0xef &&
                 static_cast<unsigned char>(str[1]) == 0xbb &&
                 static_cast<unsigned char>(str[2]) == 0xbf);
@@ -284,13 +284,13 @@ static int tolowerWrapper(int c)
 static bool isFallThroughComment(std::string comment)
 {
     // convert comment to lower case without whitespace
-    std::transform(comment.begin(), comment.end(), comment.begin(), tolowerWrapper);
     for (std::string::iterator i = comment.begin(); i != comment.end();) {
         if (std::isspace(static_cast<unsigned char>(*i)))
             i = comment.erase(i);
         else
             ++i;
     }
+    std::transform(comment.begin(), comment.end(), comment.begin(), tolowerWrapper);
 
     return comment.find("fallthr") != std::string::npos ||
            comment.find("fallsthr") != std::string::npos ||
@@ -650,83 +650,16 @@ std::string Preprocessor::removeParentheses(const std::string &str)
 }
 
 
-
-static void _removeAsm(std::string &str, const std::string::size_type pos)
-{
-    unsigned int newlines = 0;
-    bool instr = false;
-    int parlevel = 0;
-    std::string::size_type pos2 = pos + 1;
-    while (pos2 < str.length()) {
-        if (str[pos2] == '\"')
-            instr = !instr;
-
-        else if (str[pos2] == '\n')
-            ++newlines;
-
-        else if (!instr) {
-            if (str[pos2] == '(')
-                ++parlevel;
-            else if (str[pos2] == ')') {
-                if (parlevel <= 1)
-                    break;
-                --parlevel;
-            }
-        }
-
-        ++pos2;
-    }
-    str.erase(pos + 1, pos2 - pos);
-    str.insert(pos, std::string(newlines, '\n'));
-}
-
 void Preprocessor::removeAsm(std::string &str)
 {
     std::string::size_type pos = 0;
-    while ((pos = str.find("\nasm(", pos)) != std::string::npos) {
-        _removeAsm(str, pos++);
-        str.insert(pos, "asm()");
-    }
-
-    pos = 0;
-    while ((pos = str.find("\nasm (", pos)) != std::string::npos) {
-        _removeAsm(str, pos++);
-        str.insert(pos, "asm()");
-    }
-
-    pos = 0;
-    while ((pos = str.find("\nasm __volatile(", pos)) != std::string::npos)
-        _removeAsm(str, pos);
-
-    pos = 0;
-    while ((pos = str.find("\nasm __volatile (", pos)) != std::string::npos)
-        _removeAsm(str, pos);
-
-    pos = 0;
     while ((pos = str.find("#asm\n", pos)) != std::string::npos) {
-        const std::string::size_type pos1 = pos;
+        str.replace(pos, 4, "asm(");
-        ++pos;
 
-        if (pos1 > 0 && str[pos1-1] != '\n')
+        std::string::size_type pos2 = str.find("#endasm", pos);
-            continue;
+        if (pos2 != std::string::npos) {
-
+            str.replace(pos2, 7, ");");
-        const std::string::size_type endpos = str.find("\n#endasm", pos1);
+            pos = pos2;
-        if (endpos != std::string::npos) {
-            if (endpos + 8U < str.size() && str[endpos+8U] != '\n')
-                break;
-
-            // Remove '#endasm'
-            str.erase(endpos+1, 7);
-
-            // Remove non-newline characters between pos1 and endpos
-            for (std::string::size_type p = endpos; p > pos1; --p) {
-                if (str[p] != '\n')
-                    str.erase(p,1);
-            }
-            str.erase(pos1,1);
-
-            // Insert 'asm();' to make the checks bailout properly
-            str.insert(pos1, ";asm();");
         }
     }
 }

lib/token.cpp

@@ -893,6 +893,9 @@ void Token::printOut(const char *title, const std::vector<std::string> &fileName
 
 std::string Token::stringify(const Token* end) const
 {
+    if (this == end)
+        return "";
+
     std::ostringstream ret;
 
     if (isUnsigned())

test/testpreprocessor.cpp

@@ -1549,21 +1549,13 @@ private:
     }
 
     void remove_asm() {
-        std::string str1("\nasm(\n\n\n);");
+        std::string str1("#asm\nmov ax,bx\n#endasm");
         Preprocessor::removeAsm(str1);
-        ASSERT_EQUALS("\nasm()\n\n\n;", str1);
+        ASSERT_EQUALS("asm(\nmov ax,bx\n);", str1);
 
-        std::string str2("\nasm __volatile(\"\nlw iScale, 0x00(pScale)\n\", ());");
+        std::string str2("\n#asm\nmov ax,bx\n#endasm\n");
         Preprocessor::removeAsm(str2);
-        ASSERT_EQUALS("\n\n\n;", str2);
+        ASSERT_EQUALS("\nasm(\nmov ax,bx\n);\n", str2);
-
-        std::string str3("#asm\nmov ax,bx\n#endasm");
-        Preprocessor::removeAsm(str3);
-        ASSERT_EQUALS(";asm();\n\n", str3);
-
-        std::string str4("\n#asm\nmov ax,bx\n#endasm\n");
-        Preprocessor::removeAsm(str4);
-        ASSERT_EQUALS("\n;asm();\n\n\n", str4);
     }
 
     void if_defined() {
@@ -2834,469 +2826,4 @@ private:
         }
 
         {
-            std::string actual = Preprocessor::getcode(src, "X=1;Y=2", "test.c", &settings, this);
+            std::string actual = Preprocessor::getcode(src, "X=1;Y=2", "test.c", &setting
-            ASSERT_EQUALS("\nFred & Wilma\n\n", actual);
-        }
-    }
-
-    void predefine3() {
-        // #2871 - define in source is not used if -D is used
-        const char code[] = "#define X 1\n"
-                            "#define Y X\n"
-                            "#if (X == Y)\n"
-                            "Fred & Wilma\n"
-                            "#endif\n";
-        const Settings settings;
-        const std::string actual = Preprocessor::getcode(code, "TEST", "test.c", &settings, this);
-        ASSERT_EQUALS("\n\n\nFred & Wilma\n\n", actual);
-    }
-
-    void simplifyCondition() {
-        // Ticket #2794
-        std::map<std::string, std::string> cfg;
-        cfg["C"] = "";
-        std::string condition("defined(A) || defined(B) || defined(C)");
-        Preprocessor::simplifyCondition(cfg, condition, true);
-        ASSERT_EQUALS("1", condition);
-    }
-
-    void invalidElIf() {
-        // #2942 - segfault
-        const char code[] = "#elif (){\n";
-        const Settings settings;
-        const std::string actual = Preprocessor::getcode(code, "TEST", "test.c", &settings, this);
-        ASSERT_EQUALS("\n", actual);
-    }
-
-    void def_handleIncludes() {
-        const std::string filePath("test.c");
-        const std::list<std::string> includePaths;
-        std::map<std::string,std::string> defs;
-        Preprocessor preprocessor(NULL, this);
-
-        // ifdef
-        {
-            defs.clear();
-            defs["A"] = "";
-            {
-                const std::string code("#ifdef A\n123\n#endif\n");
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n123\n\n", actual);
-            }{
-                const std::string code("#ifdef B\n123\n#endif\n");
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n\n\n", actual);
-            }
-        }
-
-        // ifndef
-        {
-            defs.clear();
-            defs["A"] = "";
-            {
-                const std::string code("#ifndef A\n123\n#endif\n");
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n\n\n", actual);
-            }{
-                const std::string code("#ifndef B\n123\n#endif\n");
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n123\n\n", actual);
-            }
-        }
-
-        // define - ifndef
-        {
-            defs.clear();
-            const std::string code("#ifndef X\n#define X\n123\n#endif\n"
-                                   "#ifndef X\n#define X\n123\n#endif\n");
-            const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-            ASSERT_EQUALS("\n#define X\n123\n\n" "\n\n\n\n", actual);
-        }
-
-        // #define => #if
-        {
-            defs.clear();
-            const std::string code("#define X 123\n"
-                                   "#if X==123\n"
-                                   "456\n"
-                                   "#endif\n");
-            const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-            ASSERT_EQUALS("#define X 123\n\n456\n\n", actual);
-        }
-
-        // #elif
-        {
-            const std::string code("#if defined(A)\n"
-                                   "1\n"
-                                   "#elif defined(B)\n"
-                                   "2\n"
-                                   "#elif defined(C)\n"
-                                   "3\n"
-                                   "#else\n"
-                                   "4\n"
-                                   "#endif");
-            {
-                defs.clear();
-                defs["A"] = "";
-                defs["C"] = "";
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n1\n\n\n\n\n\n\n\n", actual);
-            }
-
-            {
-                defs.clear();
-                defs["B"] = "";
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n\n\n2\n\n\n\n\n\n", actual);
-            }
-
-            {
-                defs.clear();
-                const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-                ASSERT_EQUALS("\n\n\n\n\n\n\n4\n\n", actual);
-            }
-        }
-
-        // #endif
-        {
-            // see also endifsemicolon
-            const std::string code("{\n#ifdef X\n#endif;\n}");
-            defs.clear();
-            defs["Z"] = "";
-            const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-            ASSERT_EQUALS("{\n\n\n}\n", actual);
-        }
-
-        // #undef
-        {
-            const std::string code("#ifndef X\n"
-                                   "#define X\n"
-                                   "123\n"
-                                   "#endif\n");
-
-            defs.clear();
-            const std::string actual1(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-
-            defs.clear();
-            const std::string actual(preprocessor.handleIncludes(code + "#undef X\n" + code, filePath, includePaths, defs));
-
-            ASSERT_EQUALS(actual1 + "#undef X\n" + actual1, actual);
-        }
-
-        // #error
-        {
-            errout.str("");
-            defs.clear();
-            const std::string code("#ifndef X\n#error abc\n#endif");
-            const std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-            ASSERT_EQUALS("\n#error abc\n\n", actual);
-            ASSERT_EQUALS("[test.c:2]: (error) abc\n", errout.str());
-        }
-    }
-
-    void def_missingInclude() {
-        const std::list<std::string> includePaths;
-        std::map<std::string,std::string> defs;
-        defs["AA"] = "";
-        Settings settings;
-        Preprocessor preprocessor(&settings,this);
-
-        // missing local include
-        {
-            const std::string code("#include \"missing-include!!.h\"\n");
-
-            errout.str("");
-            preprocessor.handleIncludes(code,"test.c",includePaths,defs);
-            ASSERT_EQUALS("[test.c:1]: (information) Include file: \"missing-include!!.h\" not found.\n", errout.str());
-
-            errout.str("");
-            settings.nomsg.addSuppression("missingInclude");
-            preprocessor.handleIncludes(code,"test.c",includePaths,defs);
-            ASSERT_EQUALS("", errout.str());
-        }
-
-        // missing system header
-        {
-            const std::string code("#include <missing-include!!.h>\n");
-
-            errout.str("");
-            settings = Settings();
-            preprocessor.handleIncludes(code,"test.c",includePaths,defs);
-            ASSERT_EQUALS("", errout.str());
-
-            errout.str("");
-            settings.debugwarnings = true;
-            preprocessor.handleIncludes(code,"test.c",includePaths,defs);
-            ASSERT_EQUALS("[test.c:1]: (debug) Include file: \"missing-include!!.h\" not found.\n", errout.str());
-
-            errout.str("");
-            settings.nomsg.addSuppression("missingInclude");
-            preprocessor.handleIncludes(code,"test.c",includePaths,defs);
-            ASSERT_EQUALS("", errout.str());
-        }
-
-        // #3285 - #elif
-        {
-            const std::string code("#ifdef GNU\n"
-                                   "#elif defined(WIN32)\n"
-                                   "#include \"missing-include!!.h\"\n"
-                                   "#endif");
-            defs.clear();
-            defs["GNU"] = "";
-
-            errout.str("");
-            settings = Settings();
-            preprocessor.handleIncludes(code,"test.c",includePaths,defs);
-            ASSERT_EQUALS("", errout.str());
-        }
-    }
-
-    void def_handleIncludes_ifelse() {
-        const std::string filePath("test.c");
-        const std::list<std::string> includePaths;
-        std::map<std::string,std::string> defs;
-        Preprocessor preprocessor(NULL, this);
-
-        // #3405
-        {
-            defs.clear();
-            defs["A"] = "";
-            const std::string code("\n#ifndef PAL_UTIL_UTILS_H_\n"
-                                   "#define PAL_UTIL_UTILS_H_\n"
-                                   "1\n"
-                                   "#ifndef USE_BOOST\n"
-                                   "2\n"
-                                   "#else\n"
-                                   "3\n"
-                                   "#endif\n"
-                                   "4\n"
-                                   "#endif\n"
-                                   "\n"
-                                   "#ifndef PAL_UTIL_UTILS_H_\n"
-                                   "#define PAL_UTIL_UTILS_H_\n"
-                                   "5\n"
-                                   "#ifndef USE_BOOST\n"
-                                   "6\n"
-                                   "#else\n"
-                                   "7\n"
-                                   "#endif\n"
-                                   "8\n"
-                                   "#endif\n"
-                                   "\n");
-            std::string actual(preprocessor.handleIncludes(code,filePath,includePaths,defs));
-
-            // the 1,2,4 should be in the result
-            actual.erase(0, actual.find("1"));
-            while (actual.find("\n") != std::string::npos)
-                actual.erase(actual.find("\n"),1);
-            ASSERT_EQUALS("124", actual);
-        }
-
-        // #3418
-        {
-            defs.clear();
-            const char code[] = "#define A 1\n"
-                                "#define B A\n"
-                                "#if A == B\n"
-                                "123\n"
-                                "#endif\n";
-
-            std::string actual(preprocessor.handleIncludes(code, filePath, includePaths, defs));
-            ASSERT_EQUALS("#define A 1\n#define B A\n\n123\n\n", actual);
-        }
-    }
-
-    void undef1() {
-        Settings settings;
-
-        const char filedata[] = "#ifdef X\n"
-                                "Fred & Wilma\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X");
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\n\n\n", actual[""]);
-    }
-
-    void undef2() {
-        Settings settings;
-
-        const char filedata[] = "#ifndef X\n"
-                                "Fred & Wilma\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X");
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\nFred & Wilma\n\n", actual[""]);
-    }
-
-    void undef3() {
-        Settings settings;
-
-        const char filedata[] = "#define X\n"
-                                "#ifdef X\n"
-                                "Fred & Wilma\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\n\n\n\n", actual[""]);
-    }
-
-    void undef4() {
-        Settings settings;
-
-        const char filedata[] = "#define X Y\n"
-                                "#ifdef X\n"
-                                "Fred & Wilma\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\n\n\n\n", actual[""]);
-    }
-
-    void undef5() {
-        Settings settings;
-
-        const char filedata[] = "#define X() Y\n"
-                                "#ifdef X\n"
-                                "Fred & Wilma\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\n\n\n\n", actual[""]);
-    }
-
-    void undef6() {
-        Settings settings;
-
-        const char filedata[] = "#define X Y\n"
-                                "#ifdef X\n"
-                                "Fred & Wilma\n"
-                                "#else\n"
-                                "Barney & Betty\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\n\n\n\nBarney & Betty\n\n", actual[""]);
-    }
-
-    void undef7() {
-        Settings settings;
-
-        const char filedata[] = "#define X XDefined\n"
-                                "X;\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        TODO_ASSERT_EQUALS("\n;\n","\n$XDefined;\n", actual[""]);
-    }
-
-    void undef8() {
-        Settings settings;
-
-        const char filedata[] = "#ifdef HAVE_CONFIG_H\n"
-                                "#include \"config.h\"\n"
-                                "#endif\n"
-                                "\n"
-                                "void foo();\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-        settings.checkConfiguration = true;
-        errout.str("");
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS("", errout.str());
-        ASSERT_EQUALS("\n\n\n\nvoid foo();\n", actual[""]);
-    }
-
-    void undef9() {
-        Settings settings;
-
-        const char filedata[] = "#define X Y\n"
-                                "#ifndef X\n"
-                                "Fred & Wilma\n"
-                                "#else\n"
-                                "Barney & Betty\n"
-                                "#endif\n";
-
-        // Preprocess => actual result..
-        std::istringstream istr(filedata);
-        std::map<std::string, std::string> actual;
-        settings.userUndefs.insert("X"); // User undefs should override internal defines
-
-        Preprocessor preprocessor(&settings, this);
-        preprocessor.preprocess(istr, actual, "file.c");
-
-        // Compare results..
-        ASSERT_EQUALS(1U, actual.size());
-        ASSERT_EQUALS("\n\nFred & Wilma\n\n\n\n", actual[""]);
-    }
-};
-
-REGISTER_TEST(TestPreprocessor)