Fixed #2559 (Refactoring Preprocessor::read)

lib/preprocessor.cpp

@@ -109,39 +109,19 @@ static std::string join(const std::list<std::string> &list, char separator)
 /** Just read the code into a string. Perform simple cleanup of the code */
 std::string Preprocessor::read(std::istream &istr, const std::string &filename, Settings *settings)
 {
-    // Get filedata from stream..
+    // ------------------------------------------------------------------------------------------
-    bool ignoreSpace = true;
+    //
-
-    // need space.. #if( => #if (
-    bool needSpace = false;
-
     // handling <backspace><newline>
     // when this is encountered the <backspace><newline> will be "skipped".
     // on the next <newline>, extra newlines will be added
-    unsigned int newlines = 0;
-
     std::ostringstream code;
+    unsigned int newlines = 0;
     for (unsigned char ch = readChar(istr); istr.good(); ch = readChar(istr))
     {
         // Replace assorted special chars with spaces..
         if (((ch & 0x80) == 0) && (ch != '\n') && (std::isspace(ch) || std::iscntrl(ch)))
             ch = ' ';
 
-        // Skip spaces after ' ' and after '#'
-        if (ch == ' ' && ignoreSpace)
-            continue;
-        ignoreSpace = bool(ch == ' ' || ch == '#' || ch == '\n');
-
-        if (needSpace)
-        {
-            if (ch == '(' || ch == '!')
-                code << " ";
-            else if (!std::isalpha(ch))
-                needSpace = false;
-        }
-        if (ch == '#')
-            needSpace = true;
-
         // <backspace><newline>..
         // for gcc-compatibility the trailing spaces should be ignored
         // for vs-compatibility the trailing spaces should be kept
@@ -178,8 +158,6 @@ std::string Preprocessor::read(std::istream &istr, const std::string &filename,
             else
                 code << "\\";
         }
-
-        // Just some code..
         else
         {
             code << char(ch);
@@ -192,8 +170,117 @@ std::string Preprocessor::read(std::istream &istr, const std::string &filename,
             }
         }
     }
+    std::string result = code.str();
+    code.str("");
 
-    return removeParantheses(removeComments(code.str(), filename, settings));
+    // ------------------------------------------------------------------------------------------
+    //
+    // Remove all comments..
+    result = removeComments(result, filename, settings);
+
+    // ------------------------------------------------------------------------------------------
+    //
+    // Clean up all preprocessor statements
+    result = preprocessCleanupDirectives(result);
+
+    // ------------------------------------------------------------------------------------------
+    //
+    // Clean up preprocessor #if statements with Parantheses
+    result = removeParantheses(result);
+
+    return result;
+}
+
+std::string Preprocessor::preprocessCleanupDirectives(const std::string &processedFile) const
+{
+    std::ostringstream code;
+    std::istringstream sstr(processedFile);
+
+    std::string line;
+    while (std::getline(sstr, line))
+    {
+        // Trim lines..
+        if (!line.empty() && line[0] == ' ')
+            line.erase(0, line.find_first_not_of(" "));
+        if (!line.empty() && line[line.size()-1] == ' ')
+            line.erase(line.find_last_not_of(" ") + 1);
+
+        // Preprocessor
+        if (!line.empty() && line[0] == '#')
+        {
+            enum {
+                ESC_NONE,
+                ESC_SINGLE,
+                ESC_DOUBLE
+            } escapeStatus = ESC_NONE;
+
+            char prev = ' '; // hack to make it skip spaces between # and the directive
+            code << "#";
+            std::string::const_iterator i = line.begin();
+            i++;
+
+            // need space.. #if( => #if (
+            bool needSpace = true;
+            while(i != line.end())
+            {
+                // disable esc-mode
+                if (escapeStatus != ESC_NONE)
+                {
+                    if (prev != '\\' && escapeStatus == ESC_SINGLE && *i == '\'')
+                    {
+                        escapeStatus = ESC_NONE;
+                    }
+                    if (prev != '\\' && escapeStatus == ESC_DOUBLE && *i == '"')
+                    {
+                        escapeStatus = ESC_NONE;
+                    }
+                } else {
+                    // enable esc-mode
+                    if (escapeStatus == ESC_NONE && *i == '"')
+                        escapeStatus = ESC_DOUBLE;
+                    if (escapeStatus == ESC_NONE && *i == '\'')
+                        escapeStatus = ESC_SINGLE;
+                }
+                // skip double whitespace between arguments
+                if (escapeStatus == ESC_NONE && prev == ' ' && *i == ' ')
+                {
+                    i++;
+                    continue;
+                }
+                // Convert #if( to "#if ("
+                if (escapeStatus == ESC_NONE)
+                {
+                    if (needSpace)
+                    {
+                        if (*i == '(' || *i == '!')
+                            code << " ";
+                        else if (!std::isalpha(*i))
+                            needSpace = false;
+                    }
+                    if (*i == '#')
+                        needSpace = true;
+                }
+                code << *i;
+                if (escapeStatus != ESC_NONE && prev == '\\' && *i == '\\')
+                {
+                    prev = ' ';
+                } else {
+                    prev = *i;
+                }
+                i++;
+            }
+            if (escapeStatus != ESC_NONE)
+            {
+                // unmatched quotes.. compiler should probably complain about this..
+            }
+        } else {
+            // Do not mess with regular code..
+            code << line;
+        }
+        code << (sstr.eof()?"":"\n");
+    }
+
+    return code.str();
 }
 
 static bool hasbom(const std::string &str)
@@ -668,9 +755,6 @@ void Preprocessor::preprocess(std::istream &srcCodeStream, std::string &processe
 
     processedFile = read(srcCodeStream, filename, _settings);
 
-    // normalize the whitespaces of the file
-    preprocessWhitespaces(processedFile);
-
     // Remove asm(...)
     removeAsm(processedFile);
 
@@ -1635,15 +1719,7 @@ void Preprocessor::handleIncludes(std::string &code, const std::string &filePath
 
         if (!processedFile.empty())
         {
-            // Replace all tabs with spaces..
-            std::replace(processedFile.begin(), processedFile.end(), '\t', ' ');
-
-            // Remove all indentation..
-            if (!processedFile.empty() && processedFile[0] == ' ')
-                processedFile.erase(0, processedFile.find_first_not_of(" "));
-
             // Remove space characters that are after or before new line character
-            processedFile = removeSpaceNearNL(processedFile);
             processedFile = "#file \"" + filename + "\"\n" + processedFile + "\n#endfile";
             code.insert(pos, processedFile);
 

lib/preprocessor.h

@@ -106,7 +106,6 @@ public:
      * @param processedFile The data to be processed
      */
     static void preprocessWhitespaces(std::string &processedFile);
-
 protected:
 
     /**
@@ -152,6 +151,12 @@ protected:
      */
     static std::string removeParantheses(const std::string &str);
 
+    /**
+     * clean up #-preprocessor lines (only) 
+     * @param processedFile The data to be processed
+     */
+    std::string preprocessCleanupDirectives(const std::string &processedFile) const;
+
     /**
      * Returns the string between double quote characters or \< \> characters.
      * @param str e.g. \code#include "menu.h"\endcode or \code#include <menu.h>\endcode

test/testpreprocessor.cpp

@@ -148,6 +148,7 @@ private:
         TEST_CASE(macro_simple11);
         TEST_CASE(macro_simple12);
         TEST_CASE(macro_simple13);
+        TEST_CASE(macro_simple14);
         TEST_CASE(macroInMacro);
         TEST_CASE(macro_mismatch);
         TEST_CASE(macro_linenumbers);
@@ -198,6 +199,11 @@ private:
         TEST_CASE(endfile);
 
         TEST_CASE(redundant_config);
+
+        TEST_CASE(testPreprocessorRead1);
+        TEST_CASE(testPreprocessorRead2);
+        TEST_CASE(testPreprocessorRead3);
+        TEST_CASE(testPreprocessorRead4);
     }
 
 
@@ -209,7 +215,7 @@ private:
         Preprocessor preprocessor(&settings, this);
         std::istringstream istr(code);
         std::string codestr(preprocessor.read(istr,"test.c",0));
-        ASSERT_EQUALS("a \n#aa b \n", codestr);
+        ASSERT_EQUALS("a\n#aa b\n", codestr);
     }
 
     void readCode2()
@@ -323,7 +329,7 @@ private:
         preprocessor.preprocess(istr, actual, "file.c");
 
         // Compare results..
-        ASSERT_EQUALS("\n\" #ifdef WIN32\"\n\n\n\n", actual[""]);
+        ASSERT_EQUALS("\n\" # ifdef WIN32\"\n\n\n\n", actual[""]);
         ASSERT_EQUALS("\n\n\nqwerty\n\n", actual["WIN32"]);
         ASSERT_EQUALS(2, static_cast<unsigned int>(actual.size()));
     }
@@ -1363,7 +1369,7 @@ private:
         std::istringstream istr(filedata);
         Settings settings;
         Preprocessor preprocessor(&settings, this);
-        ASSERT_EQUALS("#define str \"abc\" \"def\" \n\nabcdef = str;\n", preprocessor.read(istr, "test.c", 0));
+        ASSERT_EQUALS("#define str \"abc\" \"def\"\n\nabcdef = str;\n", preprocessor.read(istr, "test.c", 0));
     }
 
     void multiline2()
@@ -1576,6 +1582,13 @@ private:
         ASSERT_EQUALS("\n\n", OurPreprocessor::expandMacros(filedata));
     }
 
+    void macro_simple14()
+    {
+        const char filedata[] = "#define A \"  a  \"\n"
+                                "printf(A);\n";
+        ASSERT_EQUALS("\nprintf(\"  a  \");\n", OurPreprocessor::expandMacros(filedata));
+    }
+
     void macroInMacro()
     {
         {
@@ -2591,6 +2604,63 @@ private:
             ASSERT_EQUALS("A;A;B is NOT checked", "A;A;B is NOT checked");
         }
     }
+
+    void testPreprocessorRead1()
+    {
+        const std::string filedata("/*\n*/ # /*\n*/ defi\\\nne FO\\\nO 10\\\n20");
+        std::istringstream istr(filedata.c_str());
+        Settings settings;
+        Preprocessor preprocessor(&settings, this);
+        ASSERT_EQUALS("#define FOO 1020", preprocessor.read(istr, "test.cpp", 0));
+    }
+
+    void testPreprocessorRead2()
+    {
+        const std::string filedata("\"foo\\\\\nbar\"");
+        std::istringstream istr(filedata.c_str());
+        Settings settings;
+        Preprocessor preprocessor(&settings, this);
+        ASSERT_EQUALS("\"foo\\bar\"", preprocessor.read(istr, "test.cpp", 0));
+    }
+
+    void testPreprocessorRead3()
+    {
+        const std::string filedata("#define A \" a  \"\n\" b\"");
+        std::istringstream istr(filedata.c_str());
+        Settings settings;
+        Preprocessor preprocessor(&settings, this);
+        ASSERT_EQUALS(filedata, preprocessor.read(istr, "test.cpp", 0));
+    }
+
+    void testPreprocessorRead4()
+    {
+        {
+            // test < \\> < > (unescaped)
+            const std::string filedata("#define A \" \\\\\"/*space*/  \" \"");
+            std::istringstream istr(filedata.c_str());
+            Settings settings;
+            Preprocessor preprocessor(&settings, this);
+            ASSERT_EQUALS("#define A \" \\\\\" \" \"", preprocessor.read(istr, "test.cpp", 0));
+        }
+
+        {
+            // test <" \\\"  "> (unescaped)
+            const std::string filedata("#define A \" \\\\\\\"  \"");
+            std::istringstream istr(filedata.c_str());
+            Settings settings;
+            Preprocessor preprocessor(&settings, this);
+            ASSERT_EQUALS("#define A \" \\\\\\\"  \"", preprocessor.read(istr, "test.cpp", 0));
+        }
+
+        {
+            // test <" \\\\">  <" "> (unescaped)
+            const std::string filedata("#define A \" \\\\\\\\\"/*space*/  \" \"");
+            std::istringstream istr(filedata.c_str());
+            Settings settings;
+            Preprocessor preprocessor(&settings, this);
+            ASSERT_EQUALS("#define A \" \\\\\\\\\" \" \"", preprocessor.read(istr, "test.cpp", 0));
+        }
+    }
 };
 
 REGISTER_TEST(TestPreprocessor)