some `TokenList` cleanups (#5848)

Makefile

@@ -491,7 +491,7 @@ $(libcppdir)/checkautovariables.o: lib/checkautovariables.cpp lib/addoninfo.h li
 $(libcppdir)/checkbool.o: lib/checkbool.cpp lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbool.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkbool.cpp
 
-$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
+$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/addoninfo.h lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkboost.cpp
 
 $(libcppdir)/checkbufferoverrun.o: lib/checkbufferoverrun.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
@@ -593,7 +593,7 @@ $(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes.
 $(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp
 
-$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
+$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp
 
 $(libcppdir)/mathlib.o: lib/mathlib.cpp externals/simplecpp/simplecpp.h lib/config.h lib/errortypes.h lib/mathlib.h lib/utils.h
@@ -626,7 +626,7 @@ $(libcppdir)/settings.o: lib/settings.cpp externals/picojson/picojson.h lib/addo
 $(libcppdir)/summaries.o: lib/summaries.cpp lib/addoninfo.h lib/analyzerinfo.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/summaries.cpp
 
-$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/mathlib.h lib/path.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
+$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp
 
 $(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h

lib/tokenize.cpp

@@ -3361,10 +3361,10 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
     if (mTimerResults) {
         Timer t("Tokenizer::simplifyTokens1::createAst", mSettings->showtime, mTimerResults);
         list.createAst();
-        list.validateAst();
+        list.validateAst(mSettings->debugnormal);
     } else {
         list.createAst();
-        list.validateAst();
+        list.validateAst(mSettings->debugnormal);
     }
 
     if (mTimerResults) {

lib/tokenlist.cpp

@@ -50,6 +50,9 @@ TokenList::TokenList(const Settings* settings) :
     mSettings(settings)
 {
     mTokensFrontBack.list = this;
+    if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) {
+        mLang = mSettings->enforcedLang;
+    }
 }
 
 TokenList::~TokenList()
@@ -80,12 +83,12 @@ void TokenList::deallocateTokens()
 
 void TokenList::determineCppC()
 {
-    if (!mSettings) {
+    // only try to determine it if it wasn't enforced
-        mIsC = Path::isC(getSourceFilePath());
+    if (mLang == Settings::Language::None) {
-        mIsCpp = Path::isCPP(getSourceFilePath());
+        if (Path::isC(getSourceFilePath()))
-    } else {
+            mLang = Settings::Language::C;
-        mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath()));
+        else if (Path::isCPP(getSourceFilePath()))
-        mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath()));
+            mLang = Settings::Language::CPP;
     }
 }
 
@@ -1722,10 +1725,10 @@ namespace {
     };
 }
 
-void TokenList::validateAst() const
+void TokenList::validateAst(bool print) const
 {
     OnException oe{[&] {
-            if (mSettings && mSettings->debugnormal)
+            if (print)
                 mTokensFrontBack.front->printOut();
         }};
     // Check for some known issues in AST to avoid crash/hang later on
@@ -1854,7 +1857,7 @@ void TokenList::simplifyPlatformTypes()
     if (!mSettings)
         return;
 
-    const bool isCPP11  = mSettings->standards.cpp >= Standards::CPP11;
+    const bool isCPP11 = isCPP() && (mSettings->standards.cpp >= Standards::CPP11);
 
     enum { isLongLong, isLong, isInt } type;
 
@@ -1996,7 +1999,7 @@ void TokenList::simplifyStdType()
             continue;
         }
 
-        if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mSettings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) {
+        if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok, "complex|_Complex"))) {
             bool isFloat= false;
             bool isSigned = false;
             bool isUnsigned = false;
@@ -2019,7 +2022,7 @@ void TokenList::simplifyStdType()
                 else if (Token::Match(tok2, "float|double")) {
                     isFloat = true;
                     typeSpec = tok2;
-                } else if (mSettings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex"))
+                } else if (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok2, "complex|_Complex"))
                     isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name
                 else if (Token::Match(tok2, "char|int")) {
                     if (!typeSpec)
@@ -2057,7 +2060,7 @@ void TokenList::simplifyStdType()
 
 bool TokenList::isKeyword(const std::string &str) const
 {
-    if (mIsCpp) {
+    if (isCPP()) {
         // TODO: integrate into keywords?
         // types and literals are not handled as keywords
         static const std::unordered_set<std::string> cpp_types = {"bool", "false", "true"};

lib/tokenlist.h

@@ -22,13 +22,13 @@
 //---------------------------------------------------------------------------
 
 #include "config.h"
+#include "settings.h"
 
 #include <cstddef>
 #include <iosfwd>
 #include <string>
 #include <vector>
 
-class Settings;
 class Token;
 class TokenList;
 
@@ -61,12 +61,12 @@ public:
 
     /** Is the code C. Used for bailouts */
     bool isC() const {
-        return mIsC;
+        return mLang == Settings::Language::C;
     }
 
     /** Is the code CPP. Used for bailouts */
     bool isCPP() const {
-        return mIsCpp;
+        return mLang == Settings::Language::CPP;
     }
 
     /**
@@ -171,7 +171,7 @@ public:
      * Check abstract syntax tree.
      * Throws InternalError on failure
      */
-    void validateAst() const;
+    void validateAst(bool print) const;
 
     /**
      * Verify that the given token is an element of the tokenlist.
@@ -214,8 +214,7 @@ private:
     const Settings* const mSettings{};
 
     /** File is known to be C/C++ code */
-    bool mIsC{};
+    Settings::Language mLang{Settings::Language::None};
-    bool mIsCpp{};
 };
 
 /// @}

test/testtokenize.cpp

@@ -2873,28 +2873,33 @@ private:
         }
         {
             const char code[] = "float complex x;";
-            const char expected[] = "_Complex float x ;";
+            const char expected[] = "float complex x ;";
             ASSERT_EQUALS(expected, tokenizeAndStringify(code));
         }
+        {
+            const char code[] = "float complex x;";
+            const char expected[] = "_Complex float x ;";
+            ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
+        }
         {
             const char code[] = "complex float x;";
             const char expected[] = "_Complex float x ;";
-            ASSERT_EQUALS(expected, tokenizeAndStringify(code));
+            ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
         }
         {
             const char code[] = "complex long double x;";
             const char expected[] = "_Complex long double x ;";
-            ASSERT_EQUALS(expected, tokenizeAndStringify(code));
+            ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
         }
         {
             const char code[] = "long double complex x;";
             const char expected[] = "_Complex long double x ;";
-            ASSERT_EQUALS(expected, tokenizeAndStringify(code));
+            ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
         }
         {
             const char code[] = "double complex;";
             const char expected[] = "double complex ;";
-            ASSERT_EQUALS(expected, tokenizeAndStringify(code));
+            ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
         }
     }
 
@@ -5975,7 +5980,7 @@ private:
         tokenList.prepareTernaryOpForAST();
         tokenList.list.createAst();
 
-        tokenList.list.validateAst();
+        tokenList.list.validateAst(false);
 
         // Basic AST validation
         for (const Token *tok = tokenList.list.front(); tok; tok = tok->next()) {