From d5951fa2b9a0796bdceba21556b01404b1ea5a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Tue, 9 May 2023 20:15:00 +0200 Subject: [PATCH] Tokenizer: Add attribute for exported symbols (#5043) --- lib/token.h | 87 +++++++++++++++++++---------------- lib/tokenize.cpp | 104 +++++++++++++++++++++++++----------------- lib/tokenize.h | 3 ++ test/testtokenize.cpp | 35 +++++++++++--- 4 files changed, 141 insertions(+), 88 deletions(-) diff --git a/lib/token.h b/lib/token.h index 633f59217..fdc45a6f7 100644 --- a/lib/token.h +++ b/lib/token.h @@ -538,6 +538,12 @@ public: void isAttributeNothrow(const bool value) { setFlag(fIsAttributeNothrow, value); } + bool isAttributeExport() const { + return getFlag(fIsAttributeExport); + } + void isAttributeExport(const bool value) { + setFlag(fIsAttributeExport, value); + } bool isAttributePacked() const { return getFlag(fIsAttributePacked); } @@ -1278,46 +1284,47 @@ private: Token *mLink; enum : uint64_t { - fIsUnsigned = (1 << 0), - fIsSigned = (1 << 1), - fIsPointerCompare = (1 << 2), - fIsLong = (1 << 3), - fIsStandardType = (1 << 4), - fIsExpandedMacro = (1 << 5), - fIsCast = (1 << 6), - fIsAttributeConstructor = (1 << 7), // __attribute__((constructor)) __attribute__((constructor(priority))) - fIsAttributeDestructor = (1 << 8), // __attribute__((destructor)) __attribute__((destructor(priority))) - fIsAttributeUnused = (1 << 9), // __attribute__((unused)) - fIsAttributePure = (1 << 10), // __attribute__((pure)) - fIsAttributeConst = (1 << 11), // __attribute__((const)) - fIsAttributeNoreturn = (1 << 12), // __attribute__((noreturn)), __declspec(noreturn) - fIsAttributeNothrow = (1 << 13), // __attribute__((nothrow)), __declspec(nothrow) - fIsAttributeUsed = (1 << 14), // __attribute__((used)) - fIsAttributePacked = (1 << 15), // __attribute__((packed)) - fIsAttributeMaybeUnused = (1 << 16), // [[maybe_unsed]] - fIsControlFlowKeyword = (1 << 17), // if/switch/while/... - fIsOperatorKeyword = (1 << 18), // operator=, etc - fIsComplex = (1 << 19), // complex/_Complex type - fIsEnumType = (1 << 20), // enumeration type - fIsName = (1 << 21), - fIsLiteral = (1 << 22), - fIsTemplateArg = (1 << 23), - fIsAttributeNodiscard = (1 << 24), // __attribute__ ((warn_unused_result)), [[nodiscard]] - fAtAddress = (1 << 25), // @ 0x4000 - fIncompleteVar = (1 << 26), - fConstexpr = (1 << 27), - fExternC = (1 << 28), - fIsSplitVarDeclComma = (1 << 29), // set to true when variable declarations are split up ('int a,b;' => 'int a; int b;') - fIsSplitVarDeclEq = (1 << 30), // set to true when variable declaration with initialization is split up ('int a=5;' => 'int a; a=5;') - fIsImplicitInt = (1U << 31), // Is "int" token implicitly added? - fIsInline = (1ULL << 32), // Is this a inline type - fIsTemplate = (1ULL << 33), - fIsSimplifedScope = (1ULL << 34), // scope added when simplifying e.g. if (int i = ...; ...) - fIsRemovedVoidParameter = (1ULL << 35), // A void function parameter has been removed - fIsIncompleteConstant = (1ULL << 36), - fIsRestrict = (1ULL << 37), // Is this a restrict pointer type - fIsSimplifiedTypedef = (1ULL << 38), - fIsFinalType = (1ULL << 39), // Is this a type with final specifier + fIsUnsigned = (1ULL << 0), + fIsSigned = (1ULL << 1), + fIsPointerCompare = (1ULL << 2), + fIsLong = (1ULL << 3), + fIsStandardType = (1ULL << 4), + fIsExpandedMacro = (1ULL << 5), + fIsCast = (1ULL << 6), + fIsAttributeConstructor = (1ULL << 7), // __attribute__((constructor)) __attribute__((constructor(priority))) + fIsAttributeDestructor = (1ULL << 8), // __attribute__((destructor)) __attribute__((destructor(priority))) + fIsAttributeUnused = (1ULL << 9), // __attribute__((unused)) + fIsAttributePure = (1ULL << 10), // __attribute__((pure)) + fIsAttributeConst = (1ULL << 11), // __attribute__((const)) + fIsAttributeNoreturn = (1ULL << 12), // __attribute__((noreturn)), __declspec(noreturn) + fIsAttributeNothrow = (1ULL << 13), // __attribute__((nothrow)), __declspec(nothrow) + fIsAttributeUsed = (1ULL << 14), // __attribute__((used)) + fIsAttributePacked = (1ULL << 15), // __attribute__((packed)) + fIsAttributeExport = (1ULL << 16), // __attribute__((__visibility__("default"))), __declspec(dllexport) + fIsAttributeMaybeUnused = (1ULL << 17), // [[maybe_unsed]] + fIsControlFlowKeyword = (1ULL << 18), // if/switch/while/... + fIsOperatorKeyword = (1ULL << 19), // operator=, etc + fIsComplex = (1ULL << 20), // complex/_Complex type + fIsEnumType = (1ULL << 21), // enumeration type + fIsName = (1ULL << 22), + fIsLiteral = (1ULL << 23), + fIsTemplateArg = (1ULL << 24), + fIsAttributeNodiscard = (1ULL << 25), // __attribute__ ((warn_unused_result)), [[nodiscard]] + fAtAddress = (1ULL << 26), // @ 0x4000 + fIncompleteVar = (1ULL << 27), + fConstexpr = (1ULL << 28), + fExternC = (1ULL << 29), + fIsSplitVarDeclComma = (1ULL << 30), // set to true when variable declarations are split up ('int a,b;' => 'int a; int b;') + fIsSplitVarDeclEq = (1ULL << 31), // set to true when variable declaration with initialization is split up ('int a=5;' => 'int a; a=5;') + fIsImplicitInt = (1ULL << 32), // Is "int" token implicitly added? + fIsInline = (1ULL << 33), // Is this a inline type + fIsTemplate = (1ULL << 34), + fIsSimplifedScope = (1ULL << 35), // scope added when simplifying e.g. if (int i = ...; ...) + fIsRemovedVoidParameter = (1ULL << 36), // A void function parameter has been removed + fIsIncompleteConstant = (1ULL << 37), + fIsRestrict = (1ULL << 38), // Is this a restrict pointer type + fIsSimplifiedTypedef = (1ULL << 39), + fIsFinalType = (1ULL << 40), // Is this a type with final specifier }; Token::Type mTokType; diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 7c7a9cb66..7ed1d9461 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -5802,6 +5802,8 @@ void Tokenizer::dump(std::ostream &out) const out << " isComplex=\"true\""; if (tok->isRestrict()) out << " isRestrict=\"true\""; + if (tok->isAttributeExport()) + out << " isAttributeExport=\"true\""; if (tok->link()) out << " link=\"" << tok->link() << '\"'; if (tok->varId() > 0) @@ -8688,20 +8690,65 @@ void Tokenizer::simplifyCallingConvention() } } +static bool isAttribute(const Token* tok, bool gcc) { + return gcc ? Token::Match(tok, "__attribute__|__attribute (") : Token::Match(tok, "__declspec|_declspec ("); +} + +static Token* getTokenAfterAttributes(Token* tok, bool gccattr) { + Token* after = tok; + while (isAttribute(after, gccattr)) + after = after->linkAt(1)->next(); + return after; +} + +Token* Tokenizer::getAttributeFuncTok(Token* tok, bool gccattr) const { + if (!Token::Match(tok, "%name% (")) + return nullptr; + Token* const after = getTokenAfterAttributes(tok, gccattr); + if (!after) + syntaxError(tok); + + if (Token::Match(after, "%name%|*|&|(")) { + Token *ftok = after; + while (Token::Match(ftok, "%name%|::|<|*|& !!(")) { + if (ftok->str() == "<") { + ftok = ftok->findClosingBracket(); + if (!ftok) + break; + } + ftok = ftok->next(); + } + if (Token::simpleMatch(ftok, "( *")) + ftok = ftok->tokAt(2); + if (Token::Match(ftok, "%name% (|)")) + return ftok; + } else if (Token::Match(after, "[;{=:]")) { + Token *prev = tok->previous(); + while (Token::Match(prev, "%name%")) + prev = prev->previous(); + if (Token::simpleMatch(prev, ")") && Token::Match(prev->link()->previous(), "%name% (")) + return prev->link()->previous(); + else if (Token::simpleMatch(prev, ")") && Token::Match(prev->link()->tokAt(-2), "operator %op% (") && isCPP()) + return prev->link()->tokAt(-2); + else if ((!prev || Token::Match(prev, "[;{}*]")) && Token::Match(tok->previous(), "%name%")) + return tok->previous(); + } + return nullptr; +} + void Tokenizer::simplifyDeclspec() { for (Token *tok = list.front(); tok; tok = tok->next()) { - while (Token::Match(tok, "__declspec|_declspec (") && tok->next()->link() && tok->next()->link()->next()) { - if (Token::Match(tok->tokAt(2), "noreturn|nothrow")) { - Token *tok1 = tok->next()->link()->next(); - while (tok1 && !Token::Match(tok1, "%name%")) { - tok1 = tok1->next(); - } - if (tok1) { + while (isAttribute(tok, false)) { + Token *functok = getAttributeFuncTok(tok, false); + if (Token::Match(tok->tokAt(2), "noreturn|nothrow|dllexport")) { + if (functok) { if (tok->strAt(2) == "noreturn") - tok1->isAttributeNoreturn(true); + functok->isAttributeNoreturn(true); + else if (tok->strAt(2) == "nothrow") + functok->isAttributeNothrow(true); else - tok1->isAttributeNothrow(true); + functok->isAttributeExport(true); } } else if (tok->strAt(2) == "property") tok->next()->link()->insertToken("__property"); @@ -8721,39 +8768,8 @@ void Tokenizer::simplifyAttribute() if (mSettings->library.isFunctionConst(tok->str(), false)) tok->isAttributeConst(true); } - while (Token::Match(tok, "__attribute__|__attribute (")) { - Token *after = tok; - while (Token::Match(after, "__attribute__|__attribute (")) - after = after->linkAt(1)->next(); - if (!after) - syntaxError(tok); - - Token *functok = nullptr; - if (Token::Match(after, "%name%|*|&|(")) { - Token *ftok = after; - while (Token::Match(ftok, "%name%|::|<|*|& !!(")) { - if (ftok->str() == "<") { - ftok = ftok->findClosingBracket(); - if (!ftok) - break; - } - ftok = ftok->next(); - } - if (Token::simpleMatch(ftok, "( *")) - ftok = ftok->tokAt(2); - if (Token::Match(ftok, "%name% (|)")) - functok = ftok; - } else if (Token::Match(after, "[;{=:]")) { - Token *prev = tok->previous(); - while (Token::Match(prev, "%name%")) - prev = prev->previous(); - if (Token::simpleMatch(prev, ")") && Token::Match(prev->link()->previous(), "%name% (")) - functok = prev->link()->previous(); - else if (Token::simpleMatch(prev, ")") && Token::Match(prev->link()->tokAt(-2), "operator %op% (") && isCPP()) - functok = prev->link()->tokAt(-2); - else if ((!prev || Token::Match(prev, "[;{}*]")) && Token::Match(tok->previous(), "%name%")) - functok = tok->previous(); - } + while (isAttribute(tok, true)) { + Token *functok = getAttributeFuncTok(tok, true); for (Token *attr = tok->tokAt(2); attr->str() != ")"; attr = attr->next()) { if (Token::Match(attr, "%name% (")) @@ -8773,6 +8789,7 @@ void Tokenizer::simplifyAttribute() else if (Token::Match(attr, "[(,] unused|__unused__|used|__used__ [,)]")) { Token *vartok = nullptr; + Token *after = getTokenAfterAttributes(tok, true); // check if after variable name if (Token::Match(after, ";|=")) { @@ -8812,6 +8829,9 @@ void Tokenizer::simplifyAttribute() else if (Token::Match(attr, "[(,] packed [,)]") && Token::simpleMatch(tok->previous(), "}")) tok->previous()->isAttributePacked(true); + + else if (functok && Token::simpleMatch(attr, "( __visibility__ ( \"default\" ) )")) + functok->isAttributeExport(true); } Token::eraseTokens(tok, tok->linkAt(1)->next()); diff --git a/lib/tokenize.h b/lib/tokenize.h index e4d50709e..2ef2ac510 100644 --- a/lib/tokenize.h +++ b/lib/tokenize.h @@ -467,6 +467,9 @@ private: */ void simplifyAttribute(); + /** Get function token for a attribute */ + Token* getAttributeFuncTok(Token* tok, bool gccattr) const; + /** * Remove \__cppcheck\__ ((?)) */ diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index a301a4baa..db96a2303 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -260,6 +260,7 @@ private: TEST_CASE(functionAttributeBefore2); TEST_CASE(functionAttributeBefore3); TEST_CASE(functionAttributeBefore4); + TEST_CASE(functionAttributeBefore5); // __declspec(dllexport) TEST_CASE(functionAttributeAfter1); TEST_CASE(functionAttributeAfter2); TEST_CASE(functionAttributeListBefore); @@ -3691,8 +3692,9 @@ private: "void __attribute__((__pure__)) __attribute__((__nothrow__)) __attribute__((__const__)) func2();\n" "void __attribute__((nothrow)) __attribute__((pure)) __attribute__((const)) func3();\n" "void __attribute__((__nothrow__)) __attribute__((__pure__)) __attribute__((__const__)) func4();\n" - "void __attribute__((noreturn)) func5();"; - const char expected[] = "void func1 ( ) ; void func2 ( ) ; void func3 ( ) ; void func4 ( ) ; void func5 ( ) ;"; + "void __attribute__((noreturn)) func5();\n" + "void __attribute__((__visibility__(\"default\"))) func6();"; + const char expected[] = "void func1 ( ) ; void func2 ( ) ; void func3 ( ) ; void func4 ( ) ; void func5 ( ) ; void func6 ( ) ;"; errout.str(""); @@ -3709,12 +3711,14 @@ private: const Token * func3 = Token::findsimplematch(tokenizer.tokens(), "func3"); const Token * func4 = Token::findsimplematch(tokenizer.tokens(), "func4"); const Token * func5 = Token::findsimplematch(tokenizer.tokens(), "func5"); + const Token * func6 = Token::findsimplematch(tokenizer.tokens(), "func6"); - ASSERT(func1 && func1->isAttributePure() && func1->isAttributeNothrow() && func1->isAttributeConst()); - ASSERT(func2 && func2->isAttributePure() && func2->isAttributeNothrow() && func2->isAttributeConst()); - ASSERT(func3 && func3->isAttributePure() && func3->isAttributeNothrow() && func3->isAttributeConst()); - ASSERT(func4 && func4->isAttributePure() && func4->isAttributeNothrow() && func4->isAttributeConst()); + ASSERT(func1 && func1->isAttributePure() && func1->isAttributeNothrow() && func1->isAttributeConst() && !func1->isAttributeExport()); + ASSERT(func2 && func2->isAttributePure() && func2->isAttributeNothrow() && func2->isAttributeConst() && !func2->isAttributeExport()); + ASSERT(func3 && func3->isAttributePure() && func3->isAttributeNothrow() && func3->isAttributeConst() && !func3->isAttributeExport()); + ASSERT(func4 && func4->isAttributePure() && func4->isAttributeNothrow() && func4->isAttributeConst() && !func4->isAttributeExport()); ASSERT(func5 && func5->isAttributeNoreturn()); + ASSERT(func6 && func6->isAttributeExport()); } void functionAttributeBefore2() { @@ -3765,6 +3769,25 @@ private: ASSERT(foo && foo->isAttributeConst()); } + void functionAttributeBefore5() { // __declspec(dllexport) + const char code[] = "void __declspec(dllexport) func1();\n"; + const char expected[] = "void func1 ( ) ;"; + + errout.str(""); + + // tokenize.. + Tokenizer tokenizer(&settings0, this); + std::istringstream istr(code); + ASSERT(tokenizer.tokenize(istr, "test.cpp")); + + // Expected result.. + ASSERT_EQUALS(expected, tokenizer.tokens()->stringifyList(nullptr, false)); + + const Token * func1 = Token::findsimplematch(tokenizer.tokens(), "func1"); + + ASSERT(func1 && func1->isAttributeExport()); + } + void functionAttributeAfter1() { const char code[] = "void func1() __attribute__((pure)) __attribute__((nothrow)) __attribute__((const));\n" "void func2() __attribute__((__pure__)) __attribute__((__nothrow__)) __attribute__((__const__));\n"