From 81700b481dc7e85290ec18de677087f1e332c514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 6 Jan 2024 23:49:25 +0100 Subject: [PATCH] some `TokenList` cleanups (#5848) --- Makefile | 6 +++--- lib/tokenize.cpp | 4 ++-- lib/tokenlist.cpp | 27 +++++++++++++++------------ lib/tokenlist.h | 11 +++++------ test/testtokenize.cpp | 17 +++++++++++------ 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 92608b760..60c16d87b 100644 --- a/Makefile +++ b/Makefile @@ -491,7 +491,7 @@ $(libcppdir)/checkautovariables.o: lib/checkautovariables.cpp lib/addoninfo.h li $(libcppdir)/checkbool.o: lib/checkbool.cpp lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbool.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkbool.cpp -$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h +$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/addoninfo.h lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkboost.cpp $(libcppdir)/checkbufferoverrun.o: lib/checkbufferoverrun.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h @@ -593,7 +593,7 @@ $(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes. $(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp -$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h +$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp $(libcppdir)/mathlib.o: lib/mathlib.cpp externals/simplecpp/simplecpp.h lib/config.h lib/errortypes.h lib/mathlib.h lib/utils.h @@ -626,7 +626,7 @@ $(libcppdir)/settings.o: lib/settings.cpp externals/picojson/picojson.h lib/addo $(libcppdir)/summaries.o: lib/summaries.cpp lib/addoninfo.h lib/analyzerinfo.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/summaries.cpp -$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/mathlib.h lib/path.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h +$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp $(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 7cab82b95..15c747b7f 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -3361,10 +3361,10 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration) if (mTimerResults) { Timer t("Tokenizer::simplifyTokens1::createAst", mSettings->showtime, mTimerResults); list.createAst(); - list.validateAst(); + list.validateAst(mSettings->debugnormal); } else { list.createAst(); - list.validateAst(); + list.validateAst(mSettings->debugnormal); } if (mTimerResults) { diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 1756d112b..5e45b8d57 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -50,6 +50,9 @@ TokenList::TokenList(const Settings* settings) : mSettings(settings) { mTokensFrontBack.list = this; + if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) { + mLang = mSettings->enforcedLang; + } } TokenList::~TokenList() @@ -80,12 +83,12 @@ void TokenList::deallocateTokens() void TokenList::determineCppC() { - if (!mSettings) { - mIsC = Path::isC(getSourceFilePath()); - mIsCpp = Path::isCPP(getSourceFilePath()); - } else { - mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath())); - mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath())); + // only try to determine it if it wasn't enforced + if (mLang == Settings::Language::None) { + if (Path::isC(getSourceFilePath())) + mLang = Settings::Language::C; + else if (Path::isCPP(getSourceFilePath())) + mLang = Settings::Language::CPP; } } @@ -1722,10 +1725,10 @@ namespace { }; } -void TokenList::validateAst() const +void TokenList::validateAst(bool print) const { OnException oe{[&] { - if (mSettings && mSettings->debugnormal) + if (print) mTokensFrontBack.front->printOut(); }}; // Check for some known issues in AST to avoid crash/hang later on @@ -1854,7 +1857,7 @@ void TokenList::simplifyPlatformTypes() if (!mSettings) return; - const bool isCPP11 = mSettings->standards.cpp >= Standards::CPP11; + const bool isCPP11 = isCPP() && (mSettings->standards.cpp >= Standards::CPP11); enum { isLongLong, isLong, isInt } type; @@ -1996,7 +1999,7 @@ void TokenList::simplifyStdType() continue; } - if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mSettings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) { + if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok, "complex|_Complex"))) { bool isFloat= false; bool isSigned = false; bool isUnsigned = false; @@ -2019,7 +2022,7 @@ void TokenList::simplifyStdType() else if (Token::Match(tok2, "float|double")) { isFloat = true; typeSpec = tok2; - } else if (mSettings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex")) + } else if (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok2, "complex|_Complex")) isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name else if (Token::Match(tok2, "char|int")) { if (!typeSpec) @@ -2057,7 +2060,7 @@ void TokenList::simplifyStdType() bool TokenList::isKeyword(const std::string &str) const { - if (mIsCpp) { + if (isCPP()) { // TODO: integrate into keywords? // types and literals are not handled as keywords static const std::unordered_set cpp_types = {"bool", "false", "true"}; diff --git a/lib/tokenlist.h b/lib/tokenlist.h index 19dfe51e2..39c58255f 100644 --- a/lib/tokenlist.h +++ b/lib/tokenlist.h @@ -22,13 +22,13 @@ //--------------------------------------------------------------------------- #include "config.h" +#include "settings.h" #include #include #include #include -class Settings; class Token; class TokenList; @@ -61,12 +61,12 @@ public: /** Is the code C. Used for bailouts */ bool isC() const { - return mIsC; + return mLang == Settings::Language::C; } /** Is the code CPP. Used for bailouts */ bool isCPP() const { - return mIsCpp; + return mLang == Settings::Language::CPP; } /** @@ -171,7 +171,7 @@ public: * Check abstract syntax tree. * Throws InternalError on failure */ - void validateAst() const; + void validateAst(bool print) const; /** * Verify that the given token is an element of the tokenlist. @@ -214,8 +214,7 @@ private: const Settings* const mSettings{}; /** File is known to be C/C++ code */ - bool mIsC{}; - bool mIsCpp{}; + Settings::Language mLang{Settings::Language::None}; }; /// @} diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 8dfaba260..43d534c38 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -2873,28 +2873,33 @@ private: } { const char code[] = "float complex x;"; - const char expected[] = "_Complex float x ;"; + const char expected[] = "float complex x ;"; ASSERT_EQUALS(expected, tokenizeAndStringify(code)); } + { + const char code[] = "float complex x;"; + const char expected[] = "_Complex float x ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); + } { const char code[] = "complex float x;"; const char expected[] = "_Complex float x ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } { const char code[] = "complex long double x;"; const char expected[] = "_Complex long double x ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } { const char code[] = "long double complex x;"; const char expected[] = "_Complex long double x ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } { const char code[] = "double complex;"; const char expected[] = "double complex ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } } @@ -5975,7 +5980,7 @@ private: tokenList.prepareTernaryOpForAST(); tokenList.list.createAst(); - tokenList.list.validateAst(); + tokenList.list.validateAst(false); // Basic AST validation for (const Token *tok = tokenList.list.front(); tok; tok = tok->next()) {