Preprocessor: Reverted refactoring

lib/preprocessor.cpp

@@ -31,24 +31,203 @@
 #include <cstring>
 #include <vector>
 #include <set>
-#include <stack>
 
 Preprocessor::Preprocessor(Settings *settings, ErrorLogger *errorLogger) : _settings(settings), _errorLogger(errorLogger)
 {
 
 }
 
-void Preprocessor::writeError(const std::string &fileName, const int linenr, ErrorLogger *errorLogger, const std::string &errorType, const std::string &errorText)
+void Preprocessor::writeError(const std::string &fileName, const std::string &code, size_t endPos, ErrorLogger *errorLogger, const std::string &errorType, const std::string &errorText)
 {
     if (!errorLogger)
+    {
         return;
+    }
+
+    // line number in parsed code
+    unsigned int lineno = 1;
+
+    // The current token being parsed
+    std::string CurrentToken;
+
+    // lineNumbers holds line numbers for files in fileIndexes
+    // every time an include file is complitely parsed, last item in the vector
+    // is removed and lineno is set to point to that value.
+    std::vector<unsigned int> lineNumbers;
+
+    // fileIndexes holds index for _files vector about currently parsed files
+    // every time an include file is complitely parsed, last item in the vector
+    // is removed and FileIndex is set to point to that value.
+    std::vector<std::string> fileIndexes;
+
+    // FileIndex. What file in the _files vector is read now?
+    std::string FileIndex = fileName;
+
+    // Read one byte at a time from code and create tokens
+    if (endPos > code.length())
+        endPos = code.length();
+
+    for (size_t codePos = 0; codePos < endPos; ++codePos)
+    {
+
+        char ch = code[codePos];
+
+        // UTF / extended ASCII => The output from the preprocessor should only be standard ASCII
+        if (ch < 0)
+        {
+            /**
+             * @todo handle utf better:
+             * - remove characters that are redundant
+             * - convert needed characters to standard ASCII
+             */
+
+            // Not sure how to handle this character. Bailing out.
+            if (ch < 0)
+                continue;
+        }
+
+        // char/string..
+        if (ch == '\'' || ch == '\"')
+        {
+            std::string line;
+
+            // read char
+            bool special = false;
+            char c = ch;
+            do
+            {
+                // Append token..
+                line += c;
+
+                if (c == '\n')
+                    ++lineno;
+
+                // Special sequence '\.'
+                if (special)
+                    special = false;
+                else
+                    special = (c == '\\');
+
+                // Get next character
+                ++codePos;
+                c = code[codePos];
+            }
+            while (codePos < endPos && (special || c != ch));
+            line += ch;
+
+            // Handle #file "file.h"
+            if (CurrentToken == "#file")
+            {
+                // Extract the filename
+                line = line.substr(1, line.length() - 2);
+
+                // Has this file been tokenized already?
+                ++lineno;
+                fileIndexes.push_back(FileIndex);
+                FileIndex = FileLister::simplifyPath(line.c_str());
+                lineNumbers.push_back(lineno);
+                lineno = 0;
+            }
+
+            CurrentToken.clear();
+
+            continue;
+        }
+
+        if (strchr("+-*/%&|^?!=<>[](){};:,.~\n ", ch))
+        {
+            if (ch == '.' &&
+                CurrentToken.length() > 0 &&
+                std::isdigit(CurrentToken[0]))
+            {
+                // Don't separate doubles "5.4"
+            }
+            else if (strchr("+-", ch) &&
+                     CurrentToken.length() > 0 &&
+                     std::isdigit(CurrentToken[0]) &&
+                     CurrentToken[CurrentToken.length()-1] == 'e')
+            {
+                // Don't separate doubles "4.2e+10"
+            }
+            else
+            {
+                if (CurrentToken == "#file")
+                {
+                    // Handle this where strings are handled
+                    continue;
+                }
+                else if (CurrentToken == "#endfile")
+                {
+                    if (lineNumbers.empty() || fileIndexes.empty())
+                    {
+                        std::ostringstream line;
+                        line << __LINE__;
+
+                        ErrorLogger::ErrorMessage errmsg;
+                        errmsg._severity = "error";
+                        errmsg._msg = "####### Preprocessor bug! #######";
+                        errmsg._id  = "preprocessor" + line.str();
+                        errorLogger->reportErr(errmsg);
+
+                        std::exit(0);
+                    }
+
+                    lineno = lineNumbers.back();
+                    lineNumbers.pop_back();
+                    FileIndex = fileIndexes.back();
+                    fileIndexes.pop_back();
+                    CurrentToken.clear();
+                    continue;
+                }
+
+                // If token contains # characters, split it up
+                std::string temp;
+                for (std::string::size_type i = 0; i < CurrentToken.length(); ++i)
+                {
+                    if (CurrentToken[i] == '#' && CurrentToken.length() + 1 > i && CurrentToken[i+1] == '#')
+                    {
+                        temp.clear();
+                        ++i;
+                    }
+                    else
+                        temp += CurrentToken[i];
+                }
+
+                CurrentToken.clear();
+
+                if (ch == '\n')
+                {
+                    ++lineno;
+                    continue;
+                }
+                else if (ch == ' ')
+                {
+                    continue;
+                }
+
+                CurrentToken += ch;
+                // Add "++", "--" or ">>" token
+                if ((ch == '+' || ch == '-' || ch == '>') && (code[codePos+1] == ch))
+                {
+                    ++codePos;
+                    CurrentToken += code[codePos];
+                }
+                CurrentToken.clear();
+                continue;
+            }
+        }
+
+        CurrentToken += ch;
+    }
+
 
     std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
     ErrorLogger::ErrorMessage::FileLocation loc;
-    loc.line = linenr;
+    loc.line = lineno;
-    loc.file = fileName;
+    loc.file = FileIndex;
     locationList.push_back(loc);
-    errorLogger->reportErr(ErrorLogger::ErrorMessage(locationList,
+    errorLogger->reportErr(
+        ErrorLogger::ErrorMessage(locationList,
                                   "error",
                                   errorText,
                                   errorType));
@@ -1410,7 +1589,7 @@ public:
         return tokenizer.tokens();
     }
 
-    const std::vector<std::string> &params() const
+    const std::vector<std::string> params() const
     {
         return _params;
     }
@@ -1569,217 +1748,224 @@ public:
     }
 };
 
-
+std::string Preprocessor::expandMacros(std::string code, const std::string &filename, ErrorLogger *errorLogger)
-static void skipstring(const std::string &line, std::string::size_type &pos)
-{
-    const char ch = line[pos];
-
-    ++pos;
-    while (pos < line.size() && line[pos] != ch)
-    {
-        if (line[pos] == '\\')
-            ++pos;
-        ++pos;
-    }
-}
-
-
-static bool getlines(std::istream &istr, std::string &line)
-{
-    if (!istr.good())
-        return false;
-    line = "";
-    for (char ch = (char)istr.get(); istr.good(); ch = (char)istr.get())
-    {
-        if (ch == '\'' || ch == '\"')
-        {
-            line += ch;
-            char c = 0;
-            while (istr.good() && c != ch)
-            {
-                if (c == '\\')
-                {
-                    c = (char)istr.get();
-                    if (!istr.good())
-                        return true;
-                    line += c;
-                }
-
-                c = (char)istr.get();
-                if (!istr.good())
-                    return true;
-                if (c == '\n' && line.compare(0, 1, "#") == 0)
-                    return true;
-                line += c;
-            }
-            continue;
-        }
-        if (ch == '\n')
-        {
-            if (line.compare(0, 1, "#") == 0)
-                return true;
-
-            if ((char)istr.peek() == '#')
-            {
-                line += ch;
-                return true;
-            }
-        }
-        else if (line.compare(0, 1, "#") != 0 && ch == ';')
-        {
-            line += ";";
-            return true;
-        }
-
-        line += ch;
-    }
-    return true;
-}
-
-std::string Preprocessor::expandMacros(const std::string &code, std::string filename, ErrorLogger *errorLogger)
 {
     // Search for macros and expand them..
 
-    std::map<std::string, PreprocessorMacro *> macros;
+    // First item is location where macro is located and
+    // the second is its length. 0 length means that values
+    // are not set.
+    size_t macroLoc[2] = {0, 0};
 
-    unsigned int linenr = 1;
+    std::string::size_type defpos = 0;
-
+    bool loopAgain = false;
-    // linenr, filename
+    int counter = 0;
-    std::stack< std::pair<unsigned int, std::string> > fileinfo;
+    for (;;)
-
-    std::ostringstream ostr;
-    std::istringstream istr(code.c_str());
-    std::string line;
-    while (getlines(istr, line))
     {
-        if (line.compare(0, 8, "#define ") == 0)
+        if (macroLoc[1] > 0)
         {
-            PreprocessorMacro *macro = new PreprocessorMacro(line.substr(8));
+            // Erase previously handled "#define A foo"
-            if (macro->name().empty())
+            code.erase(macroLoc[0], macroLoc[1]);
-                delete macro;
+            defpos -= macroLoc[1];
-            else
+            macroLoc[1] = 0;
-                macros[macro->name()] = macro;
-            line = "\n";
         }
 
-        else if (line.compare(0, 7, "#undef ") == 0)
+        defpos = code.find("#define ", defpos);
+        if (defpos == std::string::npos)
         {
-            std::map<std::string, PreprocessorMacro *>::iterator it;
+            if (loopAgain)
-            it = macros.find(line.substr(7));
-            if (it != macros.end())
             {
-                delete it->second;
+                loopAgain = false;
-                macros.erase(it);
+                counter++;
-            }
+                if (counter < 100)
-            line = "\n";
-        }
-
-        else if (line.compare(0, 7, "#file \"") == 0)
                 {
-            fileinfo.push(std::pair<unsigned int, std::string>(linenr, filename));
+                    defpos = 0;
-            filename = line.substr(7, line.length() - 8);
+                    continue;
-            linenr = 0;
-            line += "\n";
                 }
-
-        else if (line == "#endfile")
-        {
-            if (fileinfo.size())
-            {
-                linenr = fileinfo.top().first;
-                filename = fileinfo.top().second;
-                fileinfo.pop();
-            }
-            line += "\n";
-        }
-
-        else if (line.compare(0, 1, "#") == 0)
-        {
-            line += "\n";
-        }
-
-        // expand macros..
                 else
                 {
-            std::string::size_type pos = 0;
+                    std::ostringstream line;
-            while (pos < line.size())
+                    line << __LINE__;
-            {
-                if (line[pos] == '\n')
-                    ++linenr;
 
-                // skip strings..
+                    ErrorLogger::ErrorMessage errmsg;
-                if (line[pos] == '\"' || line[pos] == '\'')
+                    errmsg._severity = "error";
-                {
+                    errmsg._msg = "### Preprocessor::expandMacros() loop limit exceeded.";
-                    const char ch = line[pos];
+                    errmsg._id  = "preprocessor" + line.str();
+                    errorLogger->reportErr(errmsg);
+                }
+            }
 
-                    skipstring(line, pos);
+            break;
-                    ++pos;
+        }
 
-                    if (pos >= line.size())
+        if (defpos > 0 && code[defpos-1] != '\n')
         {
+            defpos++;
+            continue;
+        }
+
+        // Get macro..
+        std::string::size_type endpos = code.find("\n", defpos + 6);
+        if (endpos == std::string::npos)
+        {
+            code.erase(defpos);
+            break;
+        }
+
+        // Extract the whole macro into a separate variable "macro" and then erase it from "code"
+        const PreprocessorMacro macro(code.substr(defpos + 8, endpos - defpos - 7));
+        //code.erase(defpos, endpos - defpos);
+        macroLoc[0] = defpos;
+        macroLoc[1] = endpos - defpos;
+        defpos += (endpos - defpos);
+
+        // No macro name => continue
+        if (macro.name() == "")
+            continue;
+
+        // Expand all macros in the code..
+        char pattern[5] = "\"'# ";
+        pattern[3] = macro.name().at(0);
+        std::string::size_type pos1 = defpos;
+        while ((pos1 = code.find_first_of(pattern, pos1 + 1)) != std::string::npos)
+        {
+            char ch = code[pos1];
+
+            // #undef => break
+            if (code[pos1] == '#')
+            {
+                // Are we at a #undef or #define?
+                if (code.compare(pos1, 7, "#undef ") == 0)
+                    pos1 += 7;
+                else if (code.compare(pos1, 8, "#define ") == 0)
+                    pos1 += 8;
+                else
+                    continue;
+
+                // Compare the macroname with the macroname we're currently parsing (macro.name())
+                // If it's the same macroname.. break.
+                std::string::size_type pos = pos1 + macro.name().length();
+                if (pos < code.length()
+                    && code.compare(pos1, macro.name().length(), macro.name()) == 0
+                    && !std::isalnum(code[pos]) && code[pos] != '_')
+                    break;
+
+
+                continue;
+            }
+
+            // String or char..
+            if (code[pos1] == '\"' || code[pos1] == '\'')
+            {
+                // Find the end of the string/char..
+                ++pos1;
+                while (pos1 < code.size() && code[pos1] != ch && code[pos1] != '\n')
+                {
+                    if (code[pos1] == '\\')
+                        ++pos1;
+                    ++pos1;
+                }
+
+                // End of line/file was reached without finding pair
+                if (pos1 >= code.size() || code[pos1] == '\n')
+                {
+                    std::string::size_type lineStart = code.rfind('\n', pos1 - 1);
+                    if (lineStart != std::string::npos)
+                    {
+                        if (code.compare(lineStart + 1, 7, "#define") == 0)
+                        {
+                            // There is nothing wrong #define containing quote without
+                            // a pair.
+                            continue;
+                        }
+                    }
+
                     writeError(filename,
-                                   linenr,
+                               code,
+                               pos1,
                                errorLogger,
                                "noQuoteCharPair",
                                std::string("No pair for character (") + ch + "). Can't process file. File is either invalid or unicode, which is currently not supported.");
+
                     return "";
                 }
 
                 continue;
             }
 
-                if (!std::isalpha(line[pos]) && line[pos] != '_')
+            // Matching the macroname?
-                    ++pos;
+            if (code.compare(pos1, macro.name().length(), macro.name()) != 0)
+                continue;
 
-                // found an identifier..
+            // Previous char must not be alphanumeric nor '_'
-                unsigned int count = 0;
+            if (pos1 != 0 && (std::isalnum(code[pos1-1]) || code[pos1-1] == '_'))
-                while (pos < line.length() && (std::isalpha(line[pos]) || line[pos] == '_'))
+                continue;
+
+            // The char after the macroname must not be alphanumeric nor '_'
+            if (pos1 + macro.name().length() < code.length())
             {
-                    const std::string::size_type pos1 = pos++;
+                std::string::size_type pos2 = pos1 + macro.name().length();
-                    while (pos < line.size() && (std::isalnum(line[pos]) || line[pos] == '_'))
+                if (std::isalnum(code[pos2]) || code[pos2] == '_')
-                        ++pos;
+                    continue;
-
+            }
-                    ++count;
-                    if (count > 10)
-                        break;
-
-                    const std::string id = line.substr(pos1, pos - pos1);
-
-                    // is there a macro with this name?
-                    std::map<std::string, PreprocessorMacro *>::const_iterator it;
-                    it = macros.find(id);
-                    if (it == macros.end())
-                        break;
-
-                    const PreprocessorMacro * const macro = it->second;
 
             std::vector<std::string> params;
-                    std::string::size_type pos2 = pos;
+            std::string::size_type pos2 = pos1 + macro.name().length();
-                    if (macro->params().size() && pos2 >= line.length())
+            if (macro.params().size() && pos2 >= code.length())
-                        break;
+                continue;
+
+            // Check are we in #define
+            std::string::size_type startOfLine = code.rfind("\n", pos1);
+            ++startOfLine;
+
+            bool insideDefine = false;
+            if (code.compare(startOfLine, 8, "#define ") == 0)
+            {
+                // We are inside a define, make sure we don't have name collision
+                // by e.g. replacing the following code:
+                // #define B(a) A(a)
+                // With this:
+                // #define B(2a) A(2a)
+                std::string::size_type endOfLine = code.find("\n", pos1);
+                startOfLine += 8;
+
+                PreprocessorMacro tempMacro(code.substr(startOfLine, endOfLine - startOfLine));
+                std::string tempMacroCode;
+                if (tempMacro.renameMacroVariables(tempMacroCode, macro))
+                {
+                    // Change the macro and then start again from the start
+                    // of the line, as code has changed.
+                    code.erase(startOfLine, endOfLine - startOfLine);
+                    code.insert(startOfLine, tempMacroCode);
+                    pos1 = startOfLine;
+                    continue;
+                }
+
+                insideDefine = true;
+            }
 
             unsigned int numberOfNewlines = 0;
 
-                    if (macro->variadic() || macro->nopar() || macro->params().size())
+            if (macro.variadic() || macro.nopar() || macro.params().size())
             {
-                        if (line[pos2] == ' ')
+                if (code[pos2] == ' ')
                     pos2++;
 
-                        if (line[pos2] != '(')
+                if (code[pos2] != '(')
-                            break;
+                    continue;
 
                 int parlevel = 0;
                 std::string par;
                 bool endFound = false;
-                        for (; pos2 < line.length(); ++pos2)
+                for (; pos2 < code.length(); ++pos2)
                 {
-                            if (line[pos2] == '(')
+                    if (code[pos2] == '(')
                     {
                         ++parlevel;
                         if (parlevel == 1)
                             continue;
                     }
-                            else if (line[pos2] == ')')
+                    else if (code[pos2] == ')')
                     {
                         --parlevel;
                         if (parlevel <= 0)
@@ -1789,27 +1975,50 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
                             break;
                         }
                     }
-                            else if (line[pos2] == '\"' || line[pos2] == '\'')
+                    else if (code[pos2] == '\"' || code[pos2] == '\'')
                     {
-                                const std::string::size_type p = pos2;
+                        par += code[pos2];
-                                skipstring(line, pos2);
+                        char ch = code[pos2];
-                                if (pos2 == line.length())
+                        ++pos2;
+                        while (pos2 < code.length() && code[pos2] != ch)
+                        {
+                            if (code[pos2] == '\\')
+                            {
+                                par += code[pos2];
+                                ++pos2;
+                            }
+
+                            par += code[pos2];
+                            ++pos2;
+                        }
+                        if (pos2 == code.length())
                             break;
-                                par += line.substr(p, pos2 + 1 - p);
+                        par += code[pos2];
                         continue;
                     }
-                            else if (line[pos2] == '\n')
+                    else if (code[pos2] == '\n')
                     {
+                        if (insideDefine)
+                        {
+                            // We have code like this "#define B A("
+                            // so we shouldn't modify the A
+
+                            // Don't delete the #define
+                            macroLoc[1] = 0;
+                            loopAgain = true;
+                            break;
+                        }
+
                         ++numberOfNewlines;
                         continue;
                     }
 
-                            if (parlevel == 1 && line[pos2] == ',')
+                    if (parlevel == 1 && code[pos2] == ',')
                     {
                         params.push_back(par);
                         par = "";
                     }
-                            else if (line[pos2] == ' ')
+                    else if (code[pos2] == ' ')
                     {
                         // Add space only if it is needed
                         if (par.size() && std::isalnum(par[par.length()-1]))
@@ -1819,36 +2028,35 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
                     }
                     else if (parlevel >= 1)
                     {
-                                par.append(1, line[pos2]);
+                        par.append(1, code[pos2]);
                     }
                 }
 
                 if (!endFound)
-                            break;
+                    continue;
             }
 
             if (params.size() == 1 && params[0] == "")
                 params.clear();
 
             // Same number of parameters..
-                    if (!macro->variadic() && params.size() != macro->params().size())
+            if (!macro.variadic() && params.size() != macro.params().size())
-                        break;
+                continue;
 
             // Create macro code..
             std::string tempMacro;
-                    if (!macro->code(params, tempMacro))
+            if (!macro.code(params, tempMacro))
             {
                 // Syntax error in code
+
+
                 writeError(filename,
-                                   linenr,
+                           code,
+                           pos1,
                            errorLogger,
                            "syntaxError",
-                                   std::string("Syntax error. Not enough parameters for macro '") + macro->name() + "'.");
+                           std::string("Syntax error. Not enough parameters for macro '") + macro.name() + "'.");
-                        {
+
-                            std::map<std::string, PreprocessorMacro *>::iterator it;
-                            for (it = macros.begin(); it != macros.end(); ++it)
-                                delete it->second;
-                        }
 
                 return "";
             }
@@ -1856,30 +2064,24 @@ std::string Preprocessor::expandMacros(const std::string &code, std::string file
             const std::string macrocode(std::string(numberOfNewlines, '\n') + tempMacro);
 
             // Insert macro code..
-                    if (macro->variadic() || macro->nopar() || !macro->params().empty())
+            if (macro.variadic() || macro.nopar() || !macro.params().empty())
                 ++pos2;
 
-                    line.erase(pos1, pos2 - pos1);
+            code.erase(pos1, pos2 - pos1);
-                    line.insert(pos1, macrocode);
+            code.insert(pos1, macrocode);
-                    pos = pos1;
+            pos1 += macrocode.length() - 1;
-                }
         }
     }
 
-        ostr << line;
+    // Remove all #undef..
-        for (std::string::size_type p = 0; p < line.length(); ++p)
+    defpos = 0;
+    while ((defpos = code.find("\n#undef ", defpos)) != std::string::npos)
     {
-            if (line[p] == '\n')
+        ++defpos;
-                ++linenr;
+        std::string::size_type pos2 = code.find("\n", defpos);
-        }
+        code.erase(defpos, pos2 - defpos);
     }
 
-    {
+    return code;
-        std::map<std::string, PreprocessorMacro *>::iterator it;
-        for (it = macros.begin(); it != macros.end(); ++it)
-            delete it->second;
-    }
-
-    return ostr.str();
 }
 

lib/preprocessor.h

@@ -92,7 +92,7 @@ public:
 
 protected:
 
-    static void writeError(const std::string &fileName, const int linenr, ErrorLogger *errorLogger, const std::string &errorType, const std::string &errorText);
+    static void writeError(const std::string &fileName, const std::string &code, size_t pos, ErrorLogger *errorLogger, const std::string &errorType, const std::string &errorText);
 
     /**
      * Replace "#if defined" with "#ifdef" where possible
@@ -102,7 +102,7 @@ protected:
      */
     static std::string replaceIfDefined(const std::string &str);
 
-    static std::string expandMacros(const std::string &code, std::string filename, ErrorLogger *errorLogger);
+    static std::string expandMacros(std::string code, const std::string &filename, ErrorLogger *errorLogger);
 
     /**
      * Remove comments from code. This should only be called from read().

test/testpreprocessor.cpp

@@ -130,7 +130,6 @@ private:
         TEST_CASE(macro_nopar);
         TEST_CASE(string1);
         TEST_CASE(string2);
-        TEST_CASE(string3);
         TEST_CASE(preprocessor_undef);
         TEST_CASE(defdef);  // Defined multiple times
         TEST_CASE(preprocessor_doublesharp);
@@ -980,45 +979,45 @@ private:
     void macro_simple6()
     {
         const char filedata[] = "#define ABC (a+b+c)\n"
-                                "ABC\n";
+                                "ABC";
-        ASSERT_EQUALS("\n(a+b+c)\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\n(a+b+c)", OurPreprocessor::expandMacros(filedata));
     }
 
     void macro_simple7()
     {
         const char filedata[] = "#define ABC(str) str\n"
-                                "ABC(\"(\")\n";
+                                "ABC(\"(\")";
-        ASSERT_EQUALS("\n\"(\"\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\n\"(\"", OurPreprocessor::expandMacros(filedata));
     }
 
     void macro_simple8()
     {
         const char filedata[] = "#define ABC 123\n"
                                 "#define ABCD 1234\n"
-                                "ABC ABCD\n";
+                                "ABC ABCD";
-        ASSERT_EQUALS("\n\n123 1234\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\n\n123 1234", OurPreprocessor::expandMacros(filedata));
     }
 
     void macro_simple9()
     {
         const char filedata[] = "#define ABC(a) f(a)\n"
                                 "ABC( \"\\\"\" );\n"
-                                "ABC( \"g\" );\n";
+                                "ABC( \"g\" );";
-        ASSERT_EQUALS("\nf(\"\\\"\");\nf(\"g\");\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\nf(\"\\\"\");\nf(\"g\");", OurPreprocessor::expandMacros(filedata));
     }
 
     void macro_simple10()
     {
         const char filedata[] = "#define ABC(t) t x\n"
-                                "ABC(unsigned long);\n";
+                                "ABC(unsigned long);";
-        ASSERT_EQUALS("\nunsigned long x;\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\nunsigned long x;", OurPreprocessor::expandMacros(filedata));
     }
 
     void macro_simple11()
     {
         const char filedata[] = "#define ABC(x) delete x\n"
-                                "ABC(a);\n";
+                                "ABC(a);";
-        ASSERT_EQUALS("\ndelete a;\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\ndelete a;", OurPreprocessor::expandMacros(filedata));
     }
 
     void macroInMacro()
@@ -1026,36 +1025,36 @@ private:
         {
             const char filedata[] = "#define A(m) long n = m; n++;\n"
                                     "#define B(n) A(n)\n"
-                                    "B(0)\n";
+                                    "B(0)";
-            ASSERT_EQUALS("\n\nlong n=0;n++;\n", OurPreprocessor::expandMacros(filedata));
+            ASSERT_EQUALS("\n\nlong n=0;n++;", OurPreprocessor::expandMacros(filedata));
         }
 
         {
             const char filedata[] = "#define A B\n"
                                     "#define B 3\n"
-                                    "A\n";
+                                    "A";
-            ASSERT_EQUALS("\n\n3\n", OurPreprocessor::expandMacros(filedata));
+            ASSERT_EQUALS("\n\n3", OurPreprocessor::expandMacros(filedata));
         }
 
         {
             const char filedata[] = "#define DBG(fmt, args...) printf(fmt, ## args)\n"
                                     "#define D(fmt, args...) DBG(fmt, ## args)\n"
-                                    "DBG(\"hello\");\n";
+                                    "DBG(\"hello\");";
-            ASSERT_EQUALS("\n\nprintf(\"hello\");\n", OurPreprocessor::expandMacros(filedata));
+            ASSERT_EQUALS("\n\nprintf(\"hello\");", OurPreprocessor::expandMacros(filedata));
         }
 
         {
             const char filedata[] = "#define DBG(fmt, args...) printf(fmt, ## args)\n"
                                     "#define D(fmt, args...) DBG(fmt, ## args)\n"
-                                    "DBG(\"hello: %d\",3);\n";
+                                    "DBG(\"hello: %d\",3);";
-            ASSERT_EQUALS("\n\nprintf(\"hello: %d\",3);\n", OurPreprocessor::expandMacros(filedata));
+            ASSERT_EQUALS("\n\nprintf(\"hello: %d\",3);", OurPreprocessor::expandMacros(filedata));
         }
 
         {
             const char filedata[] = "#define A 4\n"
                                     "#define B(a) a,A\n"
-                                    "B(2);\n";
+                                    "B(2);";
-            ASSERT_EQUALS("\n\n2,4;\n", OurPreprocessor::expandMacros(filedata));
+            ASSERT_EQUALS("\n\n2,4;", OurPreprocessor::expandMacros(filedata));
         }
 
         {
@@ -1104,8 +1103,8 @@ private:
     void macro_nopar()
     {
         const char filedata[] = "#define AAA( ) { NULL }\n"
-                                "AAA()\n";
+                                "AAA()";
-        ASSERT_EQUALS("\n{ NULL }\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\n{ NULL }", OurPreprocessor::expandMacros(filedata));
     }
 
 
@@ -1114,7 +1113,7 @@ private:
         const char filedata[] = "int main()"
                                 "{"
                                 "    const char *a = \"#define A\n\";"
-                                "}\n";
+                                "}";
 
         // Preprocess => actual result..
         std::istringstream istr(filedata);
@@ -1130,18 +1129,10 @@ private:
     void string2()
     {
         const char filedata[] = "#define AAA 123\n"
-                                "str = \"AAA\"\n";
+                                "str = \"AAA\"";
 
         // Compare results..
-        ASSERT_EQUALS("\nstr = \"AAA\"\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\nstr = \"AAA\"", OurPreprocessor::expandMacros(filedata));
-    }
-
-    void string3()
-    {
-        const char filedata[] = "str(\";\");\n";
-
-        // Compare results..
-        ASSERT_EQUALS("str(\";\");\n", OurPreprocessor::expandMacros(filedata));
     }
 
 
@@ -1221,30 +1212,30 @@ private:
     void fmt1()
     {
         const char filedata[] = "#define DBG(fmt...) printf(fmt)\n"
-                                "DBG(\"[0x%lx-0x%lx)\", pstart, pend);\n";
+                                "DBG(\"[0x%lx-0x%lx)\", pstart, pend);";
 
         // Preprocess..
         std::string actual = OurPreprocessor::expandMacros(filedata);
 
-        ASSERT_EQUALS("\nprintf(\"[0x%lx-0x%lx)\",pstart,pend);\n", actual);
+        ASSERT_EQUALS("\nprintf(\"[0x%lx-0x%lx)\",pstart,pend);", actual);
     }
 
     void fmt2()
     {
         const char filedata[] = "#define DBG(fmt, args...) printf(fmt, ## args)\n"
-                                "DBG(\"hello\");\n";
+                                "DBG(\"hello\");";
 
         // Preprocess..
         std::string actual = OurPreprocessor::expandMacros(filedata);
 
-        ASSERT_EQUALS("\nprintf(\"hello\");\n", actual);
+        ASSERT_EQUALS("\nprintf(\"hello\");", actual);
     }
 
     void fmt3()
     {
         const char filedata[] = "#define FRED(...) { fred(__VA_ARGS__); }\n"
-                                "FRED(123)\n";
+                                "FRED(123)";
-        ASSERT_EQUALS("\n{ fred(123); }\n", OurPreprocessor::expandMacros(filedata));
+        ASSERT_EQUALS("\n{ fred(123); }", OurPreprocessor::expandMacros(filedata));
     }
 
 
@@ -1273,34 +1264,34 @@ private:
     void stringify()
     {
         const char filedata[] = "#define STRINGIFY(x) #x\n"
-                                "STRINGIFY(abc)\n";
+                                "STRINGIFY(abc)";
 
         // expand macros..
         std::string actual = OurPreprocessor::expandMacros(filedata);
 
-        ASSERT_EQUALS("\n\"abc\"\n", actual);
+        ASSERT_EQUALS("\n\"abc\"", actual);
     }
 
     void stringify2()
     {
         const char filedata[] = "#define A(x) g(#x)\n"
-                                "A(abc);\n";
+                                "A(abc);";
 
         // expand macros..
         std::string actual = OurPreprocessor::expandMacros(filedata);
 
-        ASSERT_EQUALS("\ng(\"abc\");\n", actual);
+        ASSERT_EQUALS("\ng(\"abc\");", actual);
     }
 
     void stringify3()
     {
         const char filedata[] = "#define A(x) g(#x)\n"
-                                "A( abc);\n";
+                                "A( abc);";
 
         // expand macros..
         std::string actual = OurPreprocessor::expandMacros(filedata);
 
-        ASSERT_EQUALS("\ng(\"abc\");\n", actual);
+        ASSERT_EQUALS("\ng(\"abc\");", actual);
     }
 
     void stringify4()
@@ -1308,12 +1299,12 @@ private:
         const char filedata[] = "#define A(x) #x\n"
                                 "1 A(\n"
                                 "abc\n"
-                                ") 2\n";
+                                ") 2";
 
         // expand macros..
         std::string actual = OurPreprocessor::expandMacros(filedata);
 
-        ASSERT_EQUALS("\n1 \n\n\"abc\" 2\n", actual);
+        ASSERT_EQUALS("\n1 \n\n\"abc\" 2", actual);
     }
 
     void stringify5()