Preprocessor: Parse comments and then remove them

externals/simplecpp/simplecpp.cpp

@@ -1223,9 +1223,6 @@ std::map<std::string, simplecpp::TokenList*> simplecpp::load(const simplecpp::To
         ret[header2] = 0;
 
         TokenList *tokens = new TokenList(f, fileNumbers, header2);
-        tokens->removeComments();
-        if (!tokens->cbegin())
-            continue;
         ret[header2] = tokens;
         filelist.push_back(tokens->cbegin());
     }
@@ -1235,9 +1232,6 @@ std::map<std::string, simplecpp::TokenList*> simplecpp::load(const simplecpp::To
 
 simplecpp::TokenList simplecpp::preprocess(const simplecpp::TokenList &rawtokens, std::vector<std::string> &files, const std::map<std::string, simplecpp::TokenList *> &filedata, const struct simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list<struct simplecpp::MacroUsage> *macroUsage)
 {
-    simplecpp::TokenList rawtokens2(rawtokens);
-    rawtokens2.removeComments();
-
     std::map<std::string, std::size_t> sizeOfType(rawtokens.sizeOfType);
     sizeOfType.insert(std::pair<std::string, std::size_t>(std::string("char"), sizeof(char)));
     sizeOfType.insert(std::pair<std::string, std::size_t>(std::string("short"), sizeof(short)));
@@ -1285,7 +1279,7 @@ simplecpp::TokenList simplecpp::preprocess(const simplecpp::TokenList &rawtokens
     std::stack<const Token *> includetokenstack;
 
     TokenList output(files);
-    for (const Token *rawtok = rawtokens2.cbegin(); rawtok || !includetokenstack.empty();) {
+    for (const Token *rawtok = rawtokens.cbegin(); rawtok || !includetokenstack.empty();) {
         if (rawtok == nullptr) {
             rawtok = includetokenstack.top();
             includetokenstack.pop();
@@ -1333,7 +1327,8 @@ simplecpp::TokenList simplecpp::preprocess(const simplecpp::TokenList &rawtokens
                     const std::string header2 = getFileName(filedata, rawtok->location.file(), header, dui);
                     if (!header2.empty()) {
                         includetokenstack.push(gotoNextLine(rawtok));
-                        rawtok = filedata.find(header2)->second->cbegin();
+                        const TokenList *includetokens = filedata.find(header2)->second;
+                        rawtok = includetokens ? includetokens->cbegin() : 0;
                         continue;
                     } else {
                         // TODO: Write warning message

lib/cppcheck.cpp

@@ -104,20 +104,22 @@ unsigned int CppCheck::processFile(const std::string& filename, std::istream& fi
 
         simplecpp::OutputList outputList;
         std::vector<std::string> files;
-        const simplecpp::TokenList tokens1(fileStream, files, filename, &outputList);
+        simplecpp::TokenList tokens1(fileStream, files, filename, &outputList);
+        preprocessor.loadFiles(tokens1, files);
+
+        // Parse comments and then remove them
+        preprocessor.inlineSuppressions(tokens1);
+        tokens1.removeComments();
+        preprocessor.removeComments();
 
         // Get configurations..
         if (_settings.userDefines.empty() || _settings.force) {
             Timer t("Preprocessor::getConfigs", _settings.showtime, &S_timerResults);
-            preprocessor.loadFiles(tokens1, files);
             configurations = preprocessor.getConfigs(tokens1);
         } else {
             configurations.insert(_settings.userDefines);
-            preprocessor.loadFiles(tokens1, files);
         }
 
-        preprocessor.inlineSuppressions(tokens1);
-
         if (_settings.checkConfiguration) {
             return 0;
         }

lib/preprocessor.cpp

@@ -457,6 +457,14 @@ void Preprocessor::loadFiles(const simplecpp::TokenList &rawtokens, std::vector<
     tokenlists = simplecpp::load(rawtokens, files, dui, &outputList);
 }
 
+void Preprocessor::removeComments()
+{
+    for (std::map<std::string, simplecpp::TokenList*>::iterator it = tokenlists.begin(); it != tokenlists.end(); ++it) {
+        if (it->second)
+            it->second->removeComments();
+    }
+}
+
 std::string Preprocessor::getcode(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, const bool writeLocations)
 {
     const std::string filename(files[0]);

lib/preprocessor.h

@@ -92,6 +92,8 @@ public:
 
     void loadFiles(const simplecpp::TokenList &rawtokens, std::vector<std::string> &files);
 
+    void removeComments();
+
     /**
      * Extract the code for each configuration
      * @param istr The (file/string) stream to read from.

test/testpreprocessor.cpp

@@ -260,7 +260,8 @@ private:
         std::istringstream istr(code);
         simplecpp::OutputList outputList;
         std::vector<std::string> files;
-        const simplecpp::TokenList tokens(istr, files, filename, &outputList);
+        simplecpp::TokenList tokens(istr, files, filename, &outputList);
+        tokens.removeComments();
         const std::set<std::string> configs(preprocessor0.getConfigs(tokens));
         for (std::set<std::string>::const_iterator it = configs.begin(); it != configs.end(); ++it) {
             try {