From 358f0c473d5e5958e571e57cc0ac1d0d7068f5fb Mon Sep 17 00:00:00 2001 From: IOBYTE Date: Fri, 23 Nov 2018 05:36:09 -0500 Subject: [PATCH] =?UTF-8?q?Modify=20template=20simplifier=20to=20add=20for?= =?UTF-8?q?ward=20declarations=20of=20some=20templa=E2=80=A6=20(#1489)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Modify template simplifier to add forward declarations of some template functions so symbol database can make sense of the expanded templates. * Fix travis. --- lib/templatesimplifier.cpp | 114 ++++++++++++++++++---- lib/templatesimplifier.h | 7 +- test/testsimplifytemplate.cpp | 175 +++++++++++++++++++++++++++++++--- test/testtokenize.cpp | 9 +- 4 files changed, 272 insertions(+), 33 deletions(-) diff --git a/lib/templatesimplifier.cpp b/lib/templatesimplifier.cpp index 410435664..1ec20b7b7 100644 --- a/lib/templatesimplifier.cpp +++ b/lib/templatesimplifier.cpp @@ -53,8 +53,8 @@ namespace { }; } -TemplateSimplifier::TokenAndName::TokenAndName(Token *tok, const std::string &s, const std::string &n) : - token(tok), scope(s), name(n) +TemplateSimplifier::TokenAndName::TokenAndName(Token *tok, const std::string &s, const std::string &n, const Token *nt) : + token(tok), scope(s), name(n), nameToken(nt) { token->hasTemplateSimplifierPointer(true); } @@ -513,14 +513,14 @@ bool TemplateSimplifier::getTemplateDeclarations() else if (tok2->str() == ";") { const int namepos = getTemplateNamePosition(parmEnd, true); if (namepos > 0) - mTemplateForwardDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos)); + mTemplateForwardDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos), parmEnd->tokAt(namepos)); break; } // Implementation => add to mTemplateDeclarations else if (tok2->str() == "{") { const int namepos = getTemplateNamePosition(parmEnd, false); if (namepos > 0) - mTemplateDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos)); + mTemplateDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos), parmEnd->tokAt(namepos)); break; } } @@ -549,7 +549,7 @@ void TemplateSimplifier::getTemplateInstantiations() const Token *tok2 = Token::findmatch(tok, "{|;"); if (tok2 && tok2->str() == "{") tok = tok2->link(); - } else if (Token::Match(tok->previous(), "[({};=] %name% ::|<") || + } else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|. %name% ::|<") || Token::Match(tok->previous(), "%type% %name% ::|<") || Token::Match(tok->tokAt(-2), "[,:] private|protected|public %name% ::|<")) { @@ -570,7 +570,7 @@ void TemplateSimplifier::getTemplateInstantiations() for (; tok2 && tok2 != tok; tok2 = tok2->previous()) { if (Token::Match(tok2, ", %name% <") && templateParameters(tok2->tokAt(2))) { - mTemplateInstantiations.emplace_back(tok2->next(), getScopeName(scopeList), tok2->strAt(1)); + mTemplateInstantiations.emplace_back(tok2->next(), getScopeName(scopeList), tok2->strAt(1), tok2->tokAt(1)); } else if (Token::Match(tok2->next(), "class|struct")) const_cast(tok2)->deleteNext(); @@ -583,11 +583,11 @@ void TemplateSimplifier::getTemplateInstantiations() const std::string fullName = scopeName + (scopeName.empty()?"":" :: ") + tok->str(); const std::list::const_iterator it = std::find_if(mTemplateDeclarations.begin(), mTemplateDeclarations.end(), FindName(fullName)); if (it != mTemplateDeclarations.end()) { - mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), fullName); + mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), fullName, tok); break; } else { if (scopeName.empty()) { - mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), scopeName1 + (scopeName1.empty()?"":" :: ") + tok->str()); + mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), scopeName1 + (scopeName1.empty()?"":" :: ") + tok->str(), tok); break; } const std::string::size_type pos = scopeName.rfind(" :: "); @@ -831,7 +831,7 @@ void TemplateSimplifier::simplifyTemplateAliases() mTemplateInstantiations.end(), FindToken(tok1)); if (it != mTemplateInstantiations.end()) - mTemplateInstantiations.emplace_back(tok2, it->scope, it->name); + mTemplateInstantiations.emplace_back(tok2, it->scope, it->name, it->nameToken); } continue; } @@ -1016,12 +1016,66 @@ void TemplateSimplifier::expandTemplate( const Token *endOfTemplateDefinition = nullptr; const Token * const templateDeclarationNameToken = templateDeclarationToken->tokAt(getTemplateNamePosition(templateDeclarationToken)); const bool isClass = Token::Match(templateDeclarationToken->next(), "class|struct|union %name% <|{|:"); + const bool isFunction = templateDeclarationNameToken->strAt(1) == "("; - // add forward declaration for classes + // add forward declarations if (copy && isClass) { templateDeclaration.token->insertToken(templateDeclarationToken->strAt(1), "", true); templateDeclaration.token->insertToken(newName, "", true); templateDeclaration.token->insertToken(";", "", true); + } else if (copy && isFunction) { + // check if this is an explicit instantiation + Token * temp = templateInstantiation.token; + while (temp && !Token::Match(temp->previous(), "}|;")) + temp = temp->previous(); + if (Token::Match(temp, "template !!<")) { + // just delete "template" + deleteToken(temp); + } else { + // add forward declaration + Token * dst = templateDeclaration.token; + Token * start; + Token * end; + auto it = mTemplateForwardDeclarationsMap.find(dst); + if (it != mTemplateForwardDeclarationsMap.end()) { + dst = it->second; + const Token * temp1 = dst->tokAt(1)->findClosingBracket(); + const Token * temp2 = temp1->tokAt(getTemplateNamePosition(temp1)); + start = temp1->next(); + end = temp2->linkAt(1)->next(); + } else { + start = templateDeclarationToken->next(); + end = templateDeclarationNameToken->linkAt(1)->next(); + } + + std::map links; + while (start && start != end) { + unsigned int itype = 0; + while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != start->str()) + ++itype; + + if (itype < typeParametersInDeclaration.size()) { + dst->insertToken(mTypesUsedInTemplateInstantiation[itype]->str(), "", true); + dst->previous()->isTemplateArg(true); + } else { + if (start->str() == templateDeclarationNameToken->str()) + dst->insertToken(newName, "", true); + else + dst->insertToken(start->str(), "", true); + if (start->link()) { + if (Token::Match(start, "[|{|(")) { + links[start->link()] = dst->previous(); + } else if (Token::Match(start, "]|}|)")) { + Token::createMutualLinks(links[start], dst->previous()); + links.erase(start); + } + } + } + + start = start->next(); + } + dst->insertToken(";", "", true); + } } for (Token *tok3 = mTokenList.front(); tok3; tok3 = tok3 ? tok3->next() : nullptr) { @@ -1220,7 +1274,7 @@ void TemplateSimplifier::expandTemplate( std::string name = tok3->str(); for (const Token *prev = tok3->tokAt(-2); Token::Match(prev, "%name% ::"); prev = prev->tokAt(-2)) name = prev->str() + " :: " + name; - mTemplateInstantiations.emplace_back(mTokenList.back(), getScopeName(scopeInfo), name); + mTemplateInstantiations.emplace_back(mTokenList.back(), getScopeName(scopeInfo), name, tok3); } // link() newly tokens manually @@ -1757,8 +1811,8 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( startToken = startToken->tokAt(-2); } - if (Token::Match(startToken->previous(), "[;{}=]") && - (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*| %name%"))) + if (Token::Match(startToken->previous(), ";|{|}|=|const") && + (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*|&| %name%"))) continue; // New type.. @@ -1823,8 +1877,8 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( startToken = startToken->tokAt(-2); } - if (Token::Match(startToken->previous(), "[;{}=]") && - (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*| %name%"))) + if (Token::Match(startToken->previous(), ";|{|}|=|const") && + (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*|&| %name%"))) return false; // already simplified @@ -1988,8 +2042,35 @@ void TemplateSimplifier::fixForwardDeclaredDefaultArgumentValues() // make sure the number of arguments match if (params1.size() == params2.size()) { + std::string declName = decl.scope; + if (!declName.empty()) + declName += " :: "; + if (decl.nameToken->strAt(-1) == "::") { + const Token * start = decl.nameToken; + + while (Token::Match(start->tokAt(-2), "%name% ::")) + start = start->tokAt(-2); + + while (start != decl.nameToken) { + declName += (start->str() + " "); + start = start->next(); + } + } + declName += decl.name; + + std::string forwardDeclName = forwardDecl.scope; + if (!forwardDeclName.empty()) + forwardDeclName += " :: "; + forwardDeclName += forwardDecl.name; + // make sure the scopes and names match - if (forwardDecl.scope == decl.scope && forwardDecl.name == decl.name) { + if (forwardDeclName == declName) { + // save forward declaration for lookup later + if ((decl.nameToken->strAt(1) == "(" && forwardDecl.nameToken->strAt(1) == "(") || + (decl.nameToken->strAt(1) == "{" && forwardDecl.nameToken->strAt(1) == ";")) { + mTemplateForwardDeclarationsMap[decl.token] = forwardDecl.token; + } + for (size_t k = 0; k < params1.size(); k++) { // copy default value to declaration if not present if (params1[k]->strAt(1) == "=" && params2[k]->strAt(1) != "=") { @@ -2018,6 +2099,7 @@ void TemplateSimplifier::simplifyTemplates( if (i) { mTemplateDeclarations.clear(); mTemplateForwardDeclarations.clear(); + mTemplateForwardDeclarationsMap.clear(); mTemplateInstantiations.clear(); mInstantiatedTemplates.clear(); } diff --git a/lib/templatesimplifier.h b/lib/templatesimplifier.h index 4a04d4e27..85a2c006d 100644 --- a/lib/templatesimplifier.h +++ b/lib/templatesimplifier.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -68,13 +69,14 @@ public: * Token and its full scopename */ struct TokenAndName { - TokenAndName(Token *tok, const std::string &s, const std::string &n); + TokenAndName(Token *tok, const std::string &s, const std::string &n, const Token *nt); bool operator == (const TokenAndName & rhs) const { - return token == rhs.token && scope == rhs.scope && name == rhs.name; + return token == rhs.token && scope == rhs.scope && name == rhs.name && nameToken == rhs.nameToken; } Token *token; std::string scope; std::string name; + const Token *nameToken; }; /** @@ -263,6 +265,7 @@ private: std::list mTemplateDeclarations; std::list mTemplateForwardDeclarations; + std::map mTemplateForwardDeclarationsMap; std::list mTemplateInstantiations; std::list mInstantiatedTemplates; std::list mMemberFunctionsToDelete; diff --git a/test/testsimplifytemplate.cpp b/test/testsimplifytemplate.cpp index cb9084ed9..bbe2e3211 100644 --- a/test/testsimplifytemplate.cpp +++ b/test/testsimplifytemplate.cpp @@ -109,6 +109,14 @@ private: TEST_CASE(template69); // #8791 TEST_CASE(template70); // #5289 TEST_CASE(template71); // #8821 + TEST_CASE(template72); + TEST_CASE(template73); + TEST_CASE(template74); + TEST_CASE(template75); + TEST_CASE(template76); + TEST_CASE(template77); + TEST_CASE(template78); + TEST_CASE(template79); // #5133 TEST_CASE(template_specialization_1); // #7868 - template specialization template struct S> {..}; TEST_CASE(template_specialization_2); // #7868 - template specialization template struct S> {..}; TEST_CASE(template_enum); // #6299 Syntax error in complex enum declaration (including template) @@ -184,7 +192,8 @@ private: const char code[] = "template void f(T val) { T a; }\n" "f(10);"; - const char expected[] = "f ( 10 ) ; " + const char expected[] = "void f ( int val ) ; " + "f ( 10 ) ; " "void f ( int val ) { }"; ASSERT_EQUALS(expected, tok(code)); @@ -374,7 +383,8 @@ private: "}\n"; // The expected result.. - const char expected[] = "void f ( ) " + const char expected[] = "int * foo<3,int> ( ) ; " + "void f ( ) " "{" " foo<3,int> ( ) ; " "} " @@ -392,7 +402,8 @@ private: "}\n"; // The expected result.. - const char expected[] = "void f ( ) " + const char expected[] = "char * foo<3,char> ( ) ; " + "void f ( ) " "{" " char * p ; p = foo<3,char> ( ) ; " "} " @@ -484,7 +495,9 @@ private: "}\n"; // The expected result.. - const char expected[] = "void a<0> ( ) { } " + const char expected[] = "void a<2> ( ) ; " + "void a<1> ( ) ; " + "void a<0> ( ) { } " "int main ( ) " "{ a<2> ( ) ; return 0 ; } " "void a<2> ( ) { a<1> ( ) ; } " @@ -522,7 +535,9 @@ private: " return 0;\n" "}\n"; - const char expected[] = "int main ( ) { b<2> ( ) ; return 0 ; } " + const char expected[] = "void a<2> ( ) ; " + "void b<2> ( ) ; " + "int main ( ) { b<2> ( ) ; return 0 ; } " "void b<2> ( ) { a<2> ( ) ; } " "void a<2> ( ) { }"; @@ -566,7 +581,8 @@ private: "}\n"; // The expected result.. - const char expected[] = "void f ( ) " + const char expected[] = "char & foo ( ) ; " + "void f ( ) " "{" " char p ; p = foo ( ) ; " "} " @@ -653,7 +669,8 @@ private: " std::cout << (foo());\n" "}"; - const char expected[] = "void bar ( ) {" + const char expected[] = "void foo ( ) ; " + "void bar ( ) {" " std :: cout << ( foo ( ) ) ; " "} " "void foo ( ) { }"; @@ -906,7 +923,9 @@ private: const char code2[] = "template T f(T t) { return t; }\n" "int x() { return f(123); }"; - ASSERT_EQUALS("int x ( ) { return f ( 123 ) ; } int f ( int t ) { return t ; }", tok(code2)); + ASSERT_EQUALS("int f ( int t ) ; " + "int x ( ) { return f ( 123 ) ; } " + "int f ( int t ) { return t ; }", tok(code2)); } void template42() { // #4878 cpcheck aborts in ext-blocks.cpp (clang testcode) @@ -1123,7 +1142,8 @@ private: "void foo() {\n" " TestArithmetic();\n" "}"; - const char exp[] = "void foo ( ) {" + const char exp[] = "void TestArithmetic ( ) ; " + "void foo ( ) {" " TestArithmetic ( ) ; " "} " "void TestArithmetic ( ) {" @@ -1153,6 +1173,7 @@ private: "struct Factorial<2> ; " "struct Factorial<1> ; " "struct Factorial<0> { enum FacHelper { value = 1 } ; } ; " + "int diagonalGroupTest<4> ( ) ; " "int main ( ) { return diagonalGroupTest<4> ( ) ; } " "int diagonalGroupTest<4> ( ) { return Factorial<4> :: value ; } " "struct Factorial<4> { enum FacHelper { value = 4 * Factorial<3> :: value } ; } ; " @@ -1170,6 +1191,7 @@ private: "void j() { h(); }"; const char exp[] = "struct S ; " "template < typename T > void f ( ) { } " // <- TODO: This template is not expanded + "void h ( ) ; " "void j ( ) { h ( ) ; } " "void h ( ) { f < S :: type ( 0 ) > ( ) ; } " "struct S { } ;"; @@ -1327,6 +1349,7 @@ private: "};"; const char exp [] = "class Test { " "int test ; " + "int lookup ( ) ; " "int Fun ( ) { return lookup ( ) ; } " "} ; " "int Test :: lookup ( ) { return test ; }"; @@ -1373,6 +1396,127 @@ private: ASSERT_EQUALS(exp, tok(code)); } + void template72() { + const char code[] = "template class Tokenizer;\n" + "const Tokenizer *tokenizer() const;\n" + "template \n" + "Tokenizer::Tokenizer() { }"; + const char exp [] = "template < typename N , typename P > class Tokenizer ; " + "const Tokenizer < Node , Path > * tokenizer ( ) const ; " + "template < typename N , typename P > " + "Tokenizer < N , P > :: Tokenizer ( ) { }"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template73() { + const char code[] = "template\n" + "void keep_range(T& value, const T mini, const T maxi){}\n" + "template void keep_range(float& v, const float l, const float u);\n" + "template void keep_range(int& v, const int l, const int u);"; + const char exp[] = "void keep_range ( float & v , const float l , const float u ) ; " + "void keep_range ( int & v , const int l , const int u ) ; " + "void keep_range ( float & value , const float mini , const float maxi ) { } " + "void keep_range ( int & value , const int mini , const int maxi ) { }"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template74() { + const char code[] = "template class BTlist { };\n" + "class PushBackStreamBuf {\n" + "public:\n" + " void pushBack(const BTlist &vec);\n" + "};"; + const char exp[] = "class BTlist ; " + "class PushBackStreamBuf { " + "public: " + "void pushBack ( const BTlist & vec ) ; " + "} ; " + "class BTlist { } ;"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template75() { + const char code[] = "template\n" + "T foo(T& value){ return value; }\n" + "template std::vector> foo>>(std::vector>& v);"; + const char exp[] = "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & v ) ; " + "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & value ) { return value ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template76() { + const char code[] = "namespace NS {\n" + " template T foo(T& value) { return value; }\n" + " template std::vector> foo>>(std::vector>& v);\n" + "}\n" + "std::vector> v;\n" + "v = foo>>(v);\n"; + const char exp[] = "namespace NS { " + "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & v ) ; " + "} " + "std :: vector < std :: vector < int > > v ; " + "v = foo>> ( v ) ; " + "std :: vector < std :: vector < int > > NS :: foo>> ( std :: vector < std :: vector < int > > & value ) { return value ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template77() { + const char code[] = "template\n" + "struct is_void : std::false_type { };\n" + "template<>\n" + "struct is_void : std::true_type { };\n" + "int main() {\n" + " std::cout << is_void::value << std::endl;\n" + " std::cout << is_void::value << std::endl;\n" + "}"; + const char exp[] = "struct is_void ; " + "struct is_void : std :: true_type { } ; " + "int main ( ) { " + "std :: cout << is_void :: value << std :: endl ; " + "std :: cout << is_void :: value << std :: endl ; " + "} " + "struct is_void : std :: false_type { } ;"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template78() { + const char code[] = "template \n" + "struct Base { };\n" + "struct S : Base ::Type { };"; + const char exp[] = "struct Base ; " + "struct S : Base :: Type { } ; " + "struct Base { } ;"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template79() { // #5133 + const char code[] = "class Foo {\n" + "public:\n" + " template void foo() { bar(); }\n" + "private:\n" + " template void bar() { bazz(); }\n" + " void bazz() { }\n" + "};\n" + "void some_func() {\n" + " Foo x;\n" + " x.foo();\n" + "}"; + const char exp[] = "class Foo { " + "public: " + "void foo ( ) ; " + "private: " + "void bar ( ) ; " + "void bazz ( ) { } " + "} ; " + "void some_func ( ) { " + "Foo x ; " + "x . foo ( ) ; " + "} " + "void Foo :: foo ( ) { bar ( ) ; } " + "void Foo :: bar ( ) { bazz ( ) ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + void template_specialization_1() { // #7868 - template specialization template struct S> {..}; const char code[] = "template struct C {};\n" "template struct S {a};\n" @@ -1761,7 +1905,9 @@ private: " template void Fred(T value) { }\n" "}\n" "Fred(123);"; - ASSERT_EQUALS("namespace { } " + ASSERT_EQUALS("namespace { " + "void Fred ( int value ) ; " + "} " "Fred ( 123 ) ; " "void Fred ( int value ) { }", tok(code)); } @@ -1985,9 +2131,9 @@ private: " }\n" " void SomeFunction() { }\n" "private:\n" - " template< typename T > void TemplatedMethod();\n" + " template< typename T > T TemplatedMethod(T);\n" "};\n" - "template< typename T > void TestClass::TemplatedMethod() { }\n" + "template< typename T > T TestClass::TemplatedMethod(T t) { return t; }\n" "}"; ASSERT_EQUALS("namespace MyNamespace { " "class TestClass { " @@ -1998,9 +2144,10 @@ private: "} " "void SomeFunction ( ) { } " "private: " - "template < typename T > void TemplatedMethod ( ) ; " + "int TemplatedMethod ( int ) ; " + "template < typename T > T TemplatedMethod ( T ) ; " // this should be removed "} ; " - "} void MyNamespace :: TestClass :: TemplatedMethod ( ) { }", tok(code)); + "} int MyNamespace :: TestClass :: TemplatedMethod ( int t ) { return t ; }", tok(code)); } unsigned int templateParameters(const char code[]) { diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index a5c822b4e..199f026d0 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -5041,7 +5041,14 @@ private: "{\n" " fn2();\n" "}\n"; - ASSERT_EQUALS("int main ( )\n{\nfn2 ( ) ;\n} void fn2 ( int t = [ ] { return 1 ; } ( ) )\n{ }", tokenizeAndStringify(code)); + ASSERT_EQUALS("void fn2 ( int t = [ ] { return 1 ; } ( ) ) ;\n" + "\n" + "\n" + "int main ( )\n" + "{\n" + "fn2 ( ) ;\n" + "} void fn2 ( int t = [ ] { return 1 ; } ( ) )\n" + "{ }", tokenizeAndStringify(code)); } void cpp0xtemplate2() {