Revert "Preprocessor: Start replacing our Preprocessor code with simplecpp"

This reverts commit 9820783b60.

There was Travis errors I'll need to look more at.

Makefile

@@ -107,7 +107,7 @@ ifndef PREFIX
 endif
 
 ifndef INCLUDE_FOR_LIB
-    INCLUDE_FOR_LIB=-Ilib -Iexternals/simplecpp -Iexternals/tinyxml
+    INCLUDE_FOR_LIB=-Ilib -Iexternals/tinyxml
 endif
 
 ifndef INCLUDE_FOR_CLI
@@ -115,7 +115,7 @@ ifndef INCLUDE_FOR_CLI
 endif
 
 ifndef INCLUDE_FOR_TEST
-    INCLUDE_FOR_TEST=-Ilib -Icli -Iexternals/simplecpp -Iexternals/tinyxml
+    INCLUDE_FOR_TEST=-Ilib -Icli -Iexternals/tinyxml
 endif
 
 BIN=$(DESTDIR)$(PREFIX)/bin
@@ -236,17 +236,11 @@ TESTOBJ =     test/options.o \
               test/testvalueflow.o \
               test/testvarid.o
 
-ifndef SIMPLECPP
-    SIMPLECPP = externals/simplecpp/simplecpp.o
-endif
-
-
 ifndef TINYXML
     TINYXML = externals/tinyxml/tinyxml2.o
 endif
 
 
-EXTOBJ += $(SIMPLECPP)
 EXTOBJ += $(TINYXML)
 .PHONY: run-dmake
 
@@ -620,9 +614,6 @@ test/testvalueflow.o: test/testvalueflow.cpp lib/cxx11emu.h test/testsuite.h lib
 test/testvarid.o: test/testvarid.cpp lib/cxx11emu.h test/testsuite.h lib/errorlogger.h lib/config.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/token.h lib/valueflow.h lib/mathlib.h lib/settings.h lib/library.h lib/standards.h lib/timer.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CFG) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o test/testvarid.o test/testvarid.cpp
 
-externals/simplecpp/simplecpp.o: externals/simplecpp/simplecpp.cpp lib/cxx11emu.h externals/simplecpp/simplecpp.h
-	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CFG) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o externals/simplecpp/simplecpp.o externals/simplecpp/simplecpp.cpp
-
 externals/tinyxml/tinyxml2.o: externals/tinyxml/tinyxml2.cpp lib/cxx11emu.h externals/tinyxml/tinyxml2.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CFG) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o externals/tinyxml/tinyxml2.o externals/tinyxml/tinyxml2.cpp
 

externals/simplecpp/LICENSE

@@ -1,166 +0,0 @@
-                   GNU LESSER GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-
-  This version of the GNU Lesser General Public License incorporates
-the terms and conditions of version 3 of the GNU General Public
-License, supplemented by the additional permissions listed below.
-
-  0. Additional Definitions.
-
-  As used herein, "this License" refers to version 3 of the GNU Lesser
-General Public License, and the "GNU GPL" refers to version 3 of the GNU
-General Public License.
-
-  "The Library" refers to a covered work governed by this License,
-other than an Application or a Combined Work as defined below.
-
-  An "Application" is any work that makes use of an interface provided
-by the Library, but which is not otherwise based on the Library.
-Defining a subclass of a class defined by the Library is deemed a mode
-of using an interface provided by the Library.
-
-  A "Combined Work" is a work produced by combining or linking an
-Application with the Library.  The particular version of the Library
-with which the Combined Work was made is also called the "Linked
-Version".
-
-  The "Minimal Corresponding Source" for a Combined Work means the
-Corresponding Source for the Combined Work, excluding any source code
-for portions of the Combined Work that, considered in isolation, are
-based on the Application, and not on the Linked Version.
-
-  The "Corresponding Application Code" for a Combined Work means the
-object code and/or source code for the Application, including any data
-and utility programs needed for reproducing the Combined Work from the
-Application, but excluding the System Libraries of the Combined Work.
-
-  1. Exception to Section 3 of the GNU GPL.
-
-  You may convey a covered work under sections 3 and 4 of this License
-without being bound by section 3 of the GNU GPL.
-
-  2. Conveying Modified Versions.
-
-  If you modify a copy of the Library, and, in your modifications, a
-facility refers to a function or data to be supplied by an Application
-that uses the facility (other than as an argument passed when the
-facility is invoked), then you may convey a copy of the modified
-version:
-
-   a) under this License, provided that you make a good faith effort to
-   ensure that, in the event an Application does not supply the
-   function or data, the facility still operates, and performs
-   whatever part of its purpose remains meaningful, or
-
-   b) under the GNU GPL, with none of the additional permissions of
-   this License applicable to that copy.
-
-  3. Object Code Incorporating Material from Library Header Files.
-
-  The object code form of an Application may incorporate material from
-a header file that is part of the Library.  You may convey such object
-code under terms of your choice, provided that, if the incorporated
-material is not limited to numerical parameters, data structure
-layouts and accessors, or small macros, inline functions and templates
-(ten or fewer lines in length), you do both of the following:
-
-   a) Give prominent notice with each copy of the object code that the
-   Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the object code with a copy of the GNU GPL and this license
-   document.
-
-  4. Combined Works.
-
-  You may convey a Combined Work under terms of your choice that,
-taken together, effectively do not restrict modification of the
-portions of the Library contained in the Combined Work and reverse
-engineering for debugging such modifications, if you also do each of
-the following:
-
-   a) Give prominent notice with each copy of the Combined Work that
-   the Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the Combined Work with a copy of the GNU GPL and this license
-   document.
-
-   c) For a Combined Work that displays copyright notices during
-   execution, include the copyright notice for the Library among
-   these notices, as well as a reference directing the user to the
-   copies of the GNU GPL and this license document.
-
-   d) Do one of the following:
-
-       0) Convey the Minimal Corresponding Source under the terms of this
-       License, and the Corresponding Application Code in a form
-       suitable for, and under terms that permit, the user to
-       recombine or relink the Application with a modified version of
-       the Linked Version to produce a modified Combined Work, in the
-       manner specified by section 6 of the GNU GPL for conveying
-       Corresponding Source.
-
-       1) Use a suitable shared library mechanism for linking with the
-       Library.  A suitable mechanism is one that (a) uses at run time
-       a copy of the Library already present on the user's computer
-       system, and (b) will operate properly with a modified version
-       of the Library that is interface-compatible with the Linked
-       Version.
-
-   e) Provide Installation Information, but only if you would otherwise
-   be required to provide such information under section 6 of the
-   GNU GPL, and only to the extent that such information is
-   necessary to install and execute a modified version of the
-   Combined Work produced by recombining or relinking the
-   Application with a modified version of the Linked Version. (If
-   you use option 4d0, the Installation Information must accompany
-   the Minimal Corresponding Source and Corresponding Application
-   Code. If you use option 4d1, you must provide the Installation
-   Information in the manner specified by section 6 of the GNU GPL
-   for conveying Corresponding Source.)
-
-  5. Combined Libraries.
-
-  You may place library facilities that are a work based on the
-Library side by side in a single library together with other library
-facilities that are not Applications and are not covered by this
-License, and convey such a combined library under terms of your
-choice, if you do both of the following:
-
-   a) Accompany the combined library with a copy of the same work based
-   on the Library, uncombined with any other library facilities,
-   conveyed under the terms of this License.
-
-   b) Give prominent notice with the combined library that part of it
-   is a work based on the Library, and explaining where to find the
-   accompanying uncombined form of the same work.
-
-  6. Revised Versions of the GNU Lesser General Public License.
-
-  The Free Software Foundation may publish revised and/or new versions
-of the GNU Lesser General Public License from time to time. Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.
-
-  Each version is given a distinguishing version number. If the
-Library as you received it specifies that a certain numbered version
-of the GNU Lesser General Public License "or any later version"
-applies to it, you have the option of following the terms and
-conditions either of that published version or of any later version
-published by the Free Software Foundation. If the Library as you
-received it does not specify a version number of the GNU Lesser
-General Public License, you may choose any version of the GNU Lesser
-General Public License ever published by the Free Software Foundation.
-
-  If the Library as you received it specifies that a proxy can decide
-whether future versions of the GNU Lesser General Public License shall
-apply, that proxy's public statement of acceptance of any version is
-permanent authorization for you to choose that version for the
-Library.
-

externals/simplecpp/simplecpp.h

@@ -1,219 +0,0 @@
-/*
- * simplecpp - A simple and high-fidelity C/C++ preprocessor library
- * Copyright (C) 2016 Daniel Marjamäki.
- *
- * This library is free software: you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation, either
- * version 3 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef simplecppH
-#define simplecppH
-
-#include <istream>
-#include <string>
-#include <map>
-#include <cctype>
-#include <list>
-
-namespace simplecpp {
-
-typedef std::string TokenString;
-
-/**
- * Location in source code
- */
-class Location {
-public:
-    Location() : line(1U), col(0U) {}
-
-    std::string file;
-    unsigned int line;
-    unsigned int col;
-
-    Location &operator=(const Location &other) {
-        if (this != &other) {
-            file = other.file;
-            line = other.line;
-            col  = other.col;
-        }
-        return *this;
-    }
-
-    /** increment this location by string */
-    void adjust(const std::string &str);
-
-    bool operator<(const Location &rhs) const {
-        if (file != rhs.file)
-            return file < rhs.file;
-        if (line != rhs.line)
-            return line < rhs.line;
-        return col < rhs.col;
-    }
-};
-
-/**
- * token class.
- * @todo don't use std::string representation - for both memory and performance reasons
- */
-class Token {
-public:
-    Token(const TokenString &s, const Location &location) :
-        str(string), location(location), previous(nullptr), next(nullptr), string(s)
-    {
-        flags();
-    }
-
-    Token(const Token &tok) :
-        str(string), macro(tok.macro), location(tok.location), previous(nullptr), next(nullptr), string(tok.str)
-    {
-        flags();
-    }
-
-    void flags() {
-        name = (str[0] == '_' || std::isalpha(str[0]));
-        comment = (str.compare(0, 2, "//") == 0 || str.compare(0, 2, "/*") == 0);
-        number = std::isdigit(str[0]) || (str.size() > 1U && str[0] == '-' && std::isdigit(str[1]));
-        op = (str.size() == 1U) ? str[0] : '\0';
-    }
-
-    void setstr(const std::string &s) {
-        string = s;
-        flags();
-    }
-
-    bool isOneOf(const char ops[]) const;
-    bool startsWithOneOf(const char c[]) const;
-    bool endsWithOneOf(const char c[]) const;
-
-    char op;
-    const TokenString &str;
-    TokenString macro;
-    bool comment;
-    bool name;
-    bool number;
-    Location location;
-    Token *previous;
-    Token *next;
-private:
-    TokenString string;
-};
-
-/** Output from preprocessor */
-struct Output {
-    enum Type {
-        ERROR, /* error */
-        WARNING /* warning */
-    } type;
-    Location location;
-    std::string msg;
-};
-
-typedef std::list<struct Output> OutputList;
-
-/** List of tokens. */
-class TokenList {
-public:
-    TokenList();
-    TokenList(std::istringstream &istr, const std::string &filename=std::string(), OutputList *outputList = 0);
-    TokenList(const TokenList &other);
-    ~TokenList();
-    void operator=(const TokenList &other);
-
-    void clear();
-    bool empty() const {
-        return !cbegin();
-    }
-    void push_back(Token *token);
-
-    void dump() const;
-    std::string stringify() const;
-
-    void readfile(std::istream &istr, const std::string &filename=std::string(), OutputList *outputList = 0);
-    void constFold();
-
-    Token *begin() {
-        return first;
-    }
-
-    const Token *cbegin() const {
-        return first;
-    }
-
-    Token *end() {
-        return last;
-    }
-
-    const Token *cend() const {
-        return last;
-    }
-
-    void deleteToken(Token *tok) {
-        if (!tok)
-            return;
-        Token *prev = tok->previous;
-        Token *next = tok->next;
-        if (prev)
-            prev->next = next;
-        if (next)
-            next->previous = prev;
-        if (first == tok)
-            first = next;
-        if (last == tok)
-            last = prev;
-        delete tok;
-    }
-
-private:
-    void combineOperators();
-
-    void constFoldUnaryNotPosNeg(Token *tok);
-    void constFoldMulDivRem(Token *tok);
-    void constFoldAddSub(Token *tok);
-    void constFoldComparison(Token *tok);
-    void constFoldBitwise(Token *tok);
-    void constFoldLogicalOp(Token *tok);
-    void constFoldQuestionOp(Token **tok);
-
-    std::string lastLine() const;
-
-    Token *first;
-    Token *last;
-};
-
-/** Tracking how macros are used */
-struct MacroUsage {
-    std::string macroName;
-    Location    macroLocation;
-    Location    useLocation;
-};
-
-typedef std::map<std::string, std::string> Defines;
-
-/**
- * Preprocess
- *
- * Preprocessing is done in two steps currently:
- *   const simplecpp::TokenList tokens1 = simplecpp::TokenList(f);
- *   const simplecpp::TokenList tokens2 = simplecpp::preprocess(tokens1, defines);
- *
- * The "tokens1" will contain tokens for comments and for preprocessor directives. And there is no preprocessing done.
- * This "tokens1" can be used if you need to see what comments/directives there are. Or what code is hidden in #if.
- *
- * The "tokens2" will have normal preprocessor output. No comments nor directives are seen.
- *
- * @todo simplify interface
- */
-TokenList preprocess(const TokenList &rawtokens, const Defines &defines, OutputList *outputList = 0, std::list<struct MacroUsage> *macroUsage = 0);
-}
-
-#endif

lib/preprocessor.cpp

@@ -23,7 +23,6 @@
 #include "path.h"
 #include "errorlogger.h"
 #include "settings.h"
-#include "simplecpp.h"
 
 #include <algorithm>
 #include <sstream>
@@ -1020,6 +1019,7 @@ void Preprocessor::preprocess(std::istream &srcCodeStream, std::string &processe
 
         forcedIncludes +=
             "#file \"" + cur + "\"\n" +
+            "#line 1\n" +
             fileData + "\n" +
             "#endfile\n"
             ;
@@ -1035,6 +1035,7 @@ void Preprocessor::preprocess(std::istream &srcCodeStream, std::string &processe
         processedFile =
             forcedIncludes +
             "#file \"" + filename + "\"\n" +
+            "#line 1\n" +
             processedFile +
             "#endfile\n"
             ;
@@ -1792,145 +1793,6 @@ bool Preprocessor::match_cfg_def(std::map<std::string, std::string> cfg, std::st
 
 std::string Preprocessor::getcode(const std::string &filedata, const std::string &cfg, const std::string &filename)
 {
-    if (_settings.userUndefs.empty()) {
-        // Create a map for the cfg for faster access to defines
-        const std::map<std::string, std::string> defines(getcfgmap(cfg, &_settings, Path::simplifyPath(filename)));
-
-        simplecpp::OutputList outputList;
-        std::list<simplecpp::MacroUsage> macroUsage;
-
-        std::istringstream istr(filedata);
-        const simplecpp::TokenList &tokens1 = simplecpp::TokenList(istr, Path::simplifyPath(filename), &outputList);
-        const simplecpp::TokenList &tokens2 = simplecpp::preprocess(tokens1, defines, &outputList, &macroUsage);
-
-        if (!_settings.force) {
-            for (simplecpp::OutputList::const_iterator it = outputList.begin(); it != outputList.end(); ++it) {
-                if (it->type == simplecpp::Output::ERROR) {
-                    error(it->location.file, it->location.line, it->msg);
-                    return "";
-                }
-            }
-        }
-
-        // directive list..
-        directives.clear();
-        for (const simplecpp::Token *tok = tokens1.cbegin(); tok; tok = tok ? tok->next : nullptr) {
-            if ((tok->op != '#') || (tok->previous && tok->previous->location.line == tok->location.line))
-                continue;
-            if (tok->next && tok->next->str == "endfile")
-                continue;
-            Directive directive(tok->location.file, tok->location.line, "");
-            for (const simplecpp::Token *tok2 = tok; tok2 && tok2->location.line == directive.linenr; tok2 = tok2->next) {
-                if (tok2->comment)
-                    continue;
-                if (!directive.str.empty() && (tok2->location.col > tok2->previous->location.col + tok2->previous->str.size()))
-                    directive.str += ' ';
-                if (directive.str == "#" && tok2->str == "file")
-                    directive.str += "include";
-                else
-                    directive.str += tok2->str;
-            }
-            directives.push_back(directive);
-        }
-
-        // ensure that define macros are not used in the code
-        for (std::map<std::string, std::string>::const_iterator defineIt = defines.begin(); defineIt != defines.end(); ++defineIt) {
-            const std::string macroName = defineIt->first;
-            if (macroName.find("(") != std::string::npos)
-                continue;
-            const std::string macroValue = defineIt->second;
-            if (!defineIt->second.empty())
-                continue;
-            for (std::list<simplecpp::MacroUsage>::const_iterator usageIt = macroUsage.begin(); usageIt != macroUsage.end(); ++usageIt) {
-                const simplecpp::MacroUsage &mu = *usageIt;
-                if (mu.macroName != macroName)
-                    continue;
-                bool directiveLocation = false;
-                for (std::list<Directive>::const_iterator dirIt = directives.begin(); dirIt != directives.end(); ++dirIt) {
-                    if (mu.useLocation.file == dirIt->file && mu.useLocation.line == dirIt->linenr) {
-                        directiveLocation = true;
-                        break;
-                    }
-                }
-                if (!directiveLocation) {
-                    if (_settings.isEnabled("information"))
-                        validateCfgError(cfg, macroName);
-                    return "";
-                }
-            }
-        }
-
-        // assembler code locations..
-        std::set<simplecpp::Location> assemblerLocations;
-        for (std::list<Directive>::const_iterator dirIt = directives.begin(); dirIt != directives.end(); ++dirIt) {
-            const Directive &d1 = *dirIt;
-            if (d1.str.compare(0, 11, "#pragma asm") != 0)
-                continue;
-            std::list<Directive>::const_iterator dirIt2 = dirIt;
-            ++dirIt2;
-            if (dirIt2 == directives.end())
-                continue;
-
-            const Directive &d2 = *dirIt2;
-            if (d2.str.compare(0,14,"#pragma endasm") != 0 || d1.file != d2.file)
-                continue;
-
-            simplecpp::Location loc;
-            loc.file = d1.file;
-            loc.col = 0U;
-
-            for (unsigned int linenr = d1.linenr + 1U; linenr < d2.linenr; linenr++) {
-                loc.line = linenr;
-                assemblerLocations.insert(loc);
-            }
-        }
-
-        const bool writeLocations = (filedata.find("\n#file \"") != std::string::npos);
-
-        std::string prevfile;
-        unsigned int line = 1;
-        std::ostringstream ret;
-        for (const simplecpp::Token *tok = tokens2.cbegin(); tok; tok = tok->next) {
-            if (writeLocations && tok->location.file != prevfile) {
-                ret << "\n#line " << tok->location.line << " \"" << tok->location.file << "\"\n";
-                prevfile = tok->location.file;
-                line = tok->location.line;
-            }
-
-            if (tok->previous && line == tok->location.line)
-                ret << ' ';
-            bool newline = false;
-            while (tok->location.line > line) {
-                ret << '\n';
-                line++;
-                newline = true;
-            }
-            if (newline) {
-                simplecpp::Location loc = tok->location;
-                loc.col = 0U;
-                if (assemblerLocations.find(loc) != assemblerLocations.end()) {
-                    ret << "asm();";
-                    while (assemblerLocations.find(loc) != assemblerLocations.end()) {
-                        loc.line++;
-                    }
-                    while (tok && tok->location.line < loc.line)
-                        tok = tok->next;
-                    if (!tok)
-                        break;
-                    while (line < tok->location.line) {
-                        ret << '\n';
-                        ++line;
-                    }
-                }
-            }
-            if (!tok->macro.empty())
-                ret << Preprocessor::macroChar;
-            ret << tok->str;
-        }
-
-        return ret.str();
-    }
-
     // For the error report and preprocessor dump:
     // line number relative to current (included) file
     // (may decrease when popping back from an included file)
@@ -1946,7 +1808,7 @@ std::string Preprocessor::getcode(const std::string &filedata, const std::string
     std::map<std::string, std::string> cfgmap(getcfgmap(cfg, &_settings, filename));
 
     std::stack<std::string> filenames;
-    filenames.push(Path::simplifyPath(filename));
+    filenames.push(filename);
     std::stack<unsigned int> lineNumbers;
     std::istringstream istr(filedata);
     std::string line;

tools/dmake.cpp

@@ -375,9 +375,9 @@ int main(int argc, char **argv)
          << "endif\n\n";
 
     makeConditionalVariable(fout, "PREFIX", "/usr");
-    makeConditionalVariable(fout, "INCLUDE_FOR_LIB", "-Ilib -Iexternals/simplecpp -Iexternals/tinyxml");
+    makeConditionalVariable(fout, "INCLUDE_FOR_LIB", "-Ilib -Iexternals/tinyxml");
     makeConditionalVariable(fout, "INCLUDE_FOR_CLI", "-Ilib -Iexternals/tinyxml");
-    makeConditionalVariable(fout, "INCLUDE_FOR_TEST", "-Ilib -Icli -Iexternals/simplecpp -Iexternals/tinyxml");
+    makeConditionalVariable(fout, "INCLUDE_FOR_TEST", "-Ilib -Icli -Iexternals/tinyxml");
 
     fout << "BIN=$(DESTDIR)$(PREFIX)/bin\n\n";
     fout << "# For 'make man': sudo apt-get install xsltproc docbook-xsl docbook-xml on Linux\n";