From 2090866cd09227699e2cd75cba9a2d02b0c05a94 Mon Sep 17 00:00:00 2001 From: IOBYTE Date: Sun, 16 Dec 2018 23:58:48 -0500 Subject: [PATCH] template simplifier: remove explicit instantiations after instantiation (#1523) * template simplifier: remove explicit instantiations after instantiation * Fix use after free crash in clang test suite. --- lib/templatesimplifier.cpp | 224 +++++++++++++++++++--------------- lib/templatesimplifier.h | 1 + test/testsimplifytemplate.cpp | 27 ++-- test/testsymboldatabase.cpp | 2 +- 4 files changed, 147 insertions(+), 107 deletions(-) diff --git a/lib/templatesimplifier.cpp b/lib/templatesimplifier.cpp index b1c7c9326..a20b1330c 100644 --- a/lib/templatesimplifier.cpp +++ b/lib/templatesimplifier.cpp @@ -368,6 +368,12 @@ void TemplateSimplifier::eraseTokens(Token *begin, const Token *end) mTypesUsedInTemplateInstantiation[i] = nullptr; } } + for (size_t i = 0; i < mExplicitInstantiationsToDelete.size(); ++i) { + if (mExplicitInstantiationsToDelete[i] == begin->next()) { + mExplicitInstantiationsToDelete[i]->hasTemplateSimplifierPointer(false); + mExplicitInstantiationsToDelete[i] = nullptr; + } + } } begin->deleteNext(); } @@ -1023,109 +1029,112 @@ void TemplateSimplifier::expandTemplate( templateDeclaration.token->insertToken(newName, "", true); templateDeclaration.token->insertToken(";", "", true); } else if (copy && isFunction) { - // check if this is an explicit instantiation - Token * temp = templateInstantiation.token; - while (temp && !Token::Match(temp->previous(), "}|;|extern")) - temp = temp->previous(); - if (Token::Match(temp, "template !!<")) { - // just delete "template" - deleteToken(temp); + Token * dst = templateDeclaration.token; + Token * start; + Token * end; + auto it = mTemplateForwardDeclarationsMap.find(dst); + if (it != mTemplateForwardDeclarationsMap.end()) { + dst = it->second; + const Token * temp1 = dst->tokAt(1)->findClosingBracket(); + const Token * temp2 = temp1->tokAt(getTemplateNamePosition(temp1)); + start = temp1->next(); + end = temp2->linkAt(1)->next(); } else { - // add forward declaration - Token * dst = templateDeclaration.token; - Token * start; - Token * end; - auto it = mTemplateForwardDeclarationsMap.find(dst); - if (it != mTemplateForwardDeclarationsMap.end()) { - dst = it->second; - const Token * temp1 = dst->tokAt(1)->findClosingBracket(); - const Token * temp2 = temp1->tokAt(getTemplateNamePosition(temp1)); - start = temp1->next(); - end = temp2->linkAt(1)->next(); + start = templateDeclarationToken->next(); + end = templateDeclarationNameToken->linkAt(1)->next(); + } + unsigned int typeindentlevel = 0; + while (!(typeindentlevel == 0 && Token::Match(end, ";|{|:"))) { + if (Token::Match(end, "<|(|{")) + ++typeindentlevel; + else if (Token::Match(end, ">|)|}")) + --typeindentlevel; + end = end->next(); + } + + std::map links; + while (start && start != end) { + unsigned int itype = 0; + while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != start->str()) + ++itype; + + if (itype < typeParametersInDeclaration.size()) { + typeindentlevel = 0; + for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype]; + typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>")); + typetok = typetok->next()) { + if (Token::simpleMatch(typetok, ". . .")) { + typetok = typetok->tokAt(2); + continue; + } + if (Token::Match(typetok, "%name% <") && templateParameters(typetok->next()) > 0) + ++typeindentlevel; + else if (typeindentlevel > 0 && typetok->str() == ">") + --typeindentlevel; + dst->insertToken(typetok->str(), typetok->originalName(), true); + dst->previous()->isTemplateArg(true); + dst->previous()->isSigned(typetok->isSigned()); + dst->previous()->isUnsigned(typetok->isUnsigned()); + dst->previous()->isLong(typetok->isLong()); + } } else { - start = templateDeclarationToken->next(); - end = templateDeclarationNameToken->linkAt(1)->next(); - } - unsigned int typeindentlevel = 0; - while (!(typeindentlevel == 0 && Token::Match(end, ";|{|:"))) { - if (Token::Match(end, "<|(|{")) - ++typeindentlevel; - else if (Token::Match(end, ">|)|}")) - --typeindentlevel; - end = end->next(); - } - - std::map links; - while (start && start != end) { - unsigned int itype = 0; - while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != start->str()) - ++itype; - - if (itype < typeParametersInDeclaration.size()) { - typeindentlevel = 0; - for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype]; - typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>")); - typetok = typetok->next()) { - if (Token::simpleMatch(typetok, ". . .")) { - typetok = typetok->tokAt(2); - continue; - } - if (Token::Match(typetok, "%name% <") && templateParameters(typetok->next()) > 0) - ++typeindentlevel; - else if (typeindentlevel > 0 && typetok->str() == ">") - --typeindentlevel; - dst->insertToken(typetok->str(), typetok->originalName(), true); - dst->previous()->isTemplateArg(true); - dst->previous()->isSigned(typetok->isSigned()); - dst->previous()->isUnsigned(typetok->isUnsigned()); - dst->previous()->isLong(typetok->isLong()); - } + if (start->str() == templateDeclarationNameToken->str()) { + dst->insertToken(newName, "", true); + if (start->strAt(1) == "<") + start = start->next()->findClosingBracket(); } else { - if (start->str() == templateDeclarationNameToken->str()) { - dst->insertToken(newName, "", true); - if (start->strAt(1) == "<") - start = start->next()->findClosingBracket(); - } else { - // check if type is a template - if (start->strAt(1) == "<") { - // get the instantiated name - Token * closing = start->next()->findClosingBracket(); - std::string name; - const Token * type = start; - while (type && type != closing->next()) { - if (!name.empty()) - name += " "; - name += type->str(); - type = type->next(); - } - // check if type is instantiated - for (const auto & inst : mTemplateInstantiations) { - if (Token::simpleMatch(inst.nameToken, name.c_str())) { - // use the instantiated name - dst->insertToken(name, "", true); - start = closing; - break; - } - } - // just copy the token if it wasn't instantiated - if (start != closing) - dst->insertToken(start->str(), "", true); - } else - dst->insertToken(start->str(), "", true); - } - if (start->link()) { - if (Token::Match(start, "[|{|(")) { - links[start->link()] = dst->previous(); - } else if (Token::Match(start, "]|}|)")) { - Token::createMutualLinks(links[start], dst->previous()); - links.erase(start); + // check if type is a template + if (start->strAt(1) == "<") { + // get the instantiated name + Token * closing = start->next()->findClosingBracket(); + std::string name; + const Token * type = start; + while (type && type != closing->next()) { + if (!name.empty()) + name += " "; + name += type->str(); + type = type->next(); } + // check if type is instantiated + for (const auto & inst : mTemplateInstantiations) { + if (Token::simpleMatch(inst.nameToken, name.c_str())) { + // use the instantiated name + dst->insertToken(name, "", true); + start = closing; + break; + } + } + // just copy the token if it wasn't instantiated + if (start != closing) + dst->insertToken(start->str(), "", true); + } else + dst->insertToken(start->str(), "", true); + } + if (start->link()) { + if (Token::Match(start, "[|{|(")) { + links[start->link()] = dst->previous(); + } else if (Token::Match(start, "]|}|)")) { + Token::createMutualLinks(links[start], dst->previous()); + links.erase(start); } } - - start = start->next(); } - dst->insertToken(";", "", true); + + start = start->next(); + } + dst->insertToken(";", "", true); + } + + if (copy && (isClass || isFunction)) { + // check if this is an explicit instantiation + Token * start = templateInstantiation.token; + while (start && !Token::Match(start->previous(), "}|;|extern")) + start = start->previous(); + if (Token::Match(start, "template !!<")) { + if (start->strAt(-1) == "extern") + start = start->previous(); + start->hasTemplateSimplifierPointer(true); + mExplicitInstantiationsToDelete.push_back(start); } } @@ -1781,7 +1790,8 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( // Contains tokens such as "T" std::vector typeParametersInDeclaration; - const Token * const tok = getTemplateParametersInDeclaration(templateDeclaration.token->tokAt(2), typeParametersInDeclaration); + getTemplateParametersInDeclaration(templateDeclaration.token->tokAt(2), typeParametersInDeclaration); + const Token * const tok = templateDeclaration.token->next()->findClosingBracket(); // bail out if the end of the file was reached if (!tok) @@ -2084,12 +2094,12 @@ void TemplateSimplifier::fixForwardDeclaredDefaultArgumentValues() for (const auto & forwardDecl : mTemplateForwardDeclarations) { std::vector params1; - getTemplateParametersInDeclaration(forwardDecl.token, params1); + getTemplateParametersInDeclaration(forwardDecl.token->tokAt(2), params1); for (auto & decl : mTemplateDeclarations) { std::vector params2; - getTemplateParametersInDeclaration(decl.token, params2); + getTemplateParametersInDeclaration(decl.token->tokAt(2), params2); // make sure the number of arguments match if (params1.size() == params2.size()) { @@ -2153,6 +2163,7 @@ void TemplateSimplifier::simplifyTemplates( mTemplateForwardDeclarationsMap.clear(); mTemplateInstantiations.clear(); mInstantiatedTemplates.clear(); + mExplicitInstantiationsToDelete.clear(); } bool hasTemplates = getTemplateDeclarations(); @@ -2244,6 +2255,21 @@ void TemplateSimplifier::simplifyTemplates( } mMemberFunctionsToDelete.erase(mMemberFunctionsToDelete.begin()); } + + // remove explicit instantiations + for (size_t j = 0; j < mExplicitInstantiationsToDelete.size(); ++j) { + Token * start = mExplicitInstantiationsToDelete[j]; + if (start) { + Token * end = start->next(); + while (end && end->str() != ";") + end = end->next(); + if (start->previous()) + start = start->previous(); + if (end->next()) + end = end->next(); + eraseTokens(start, end); + } + } } } diff --git a/lib/templatesimplifier.h b/lib/templatesimplifier.h index 85a2c006d..64be16569 100644 --- a/lib/templatesimplifier.h +++ b/lib/templatesimplifier.h @@ -269,6 +269,7 @@ private: std::list mTemplateInstantiations; std::list mInstantiatedTemplates; std::list mMemberFunctionsToDelete; + std::vector mExplicitInstantiationsToDelete; std::vector mTypesUsedInTemplateInstantiation; }; diff --git a/test/testsimplifytemplate.cpp b/test/testsimplifytemplate.cpp index 8a686cad3..c3bd02c76 100644 --- a/test/testsimplifytemplate.cpp +++ b/test/testsimplifytemplate.cpp @@ -123,6 +123,7 @@ private: TEST_CASE(template83); TEST_CASE(template84); // #8880 TEST_CASE(template85); // #8902 crash + TEST_CASE(template86); // crash TEST_CASE(template_specialization_1); // #7868 - template specialization template struct S> {..}; TEST_CASE(template_specialization_2); // #7868 - template specialization template struct S> {..}; TEST_CASE(template_enum); // #6299 Syntax error in complex enum declaration (including template) @@ -1020,8 +1021,6 @@ private: const char expected[] = "class Fred ; " "class Fred ; " - "template void Fred :: f ( ) ; " - "template void Fred :: g ( ) ; " "class Fred { void f ( ) ; void g ( ) ; } ; " "void Fred :: f ( ) { } " "void Fred :: g ( ) { } " @@ -1419,8 +1418,8 @@ private: "void keep_range(T& value, const T mini, const T maxi){}\n" "template void keep_range(float& v, const float l, const float u);\n" "template void keep_range(int& v, const int l, const int u);"; - const char exp[] = "void keep_range ( float & v , const float l , const float u ) ; " - "void keep_range ( int & v , const int l , const int u ) ; " + const char exp[] = "void keep_range ( float & value , const float mini , const float maxi ) ; " + "void keep_range ( int & value , const int mini , const int maxi ) ; " "void keep_range ( float & value , const float mini , const float maxi ) { } " "void keep_range ( int & value , const int mini , const int maxi ) { }"; ASSERT_EQUALS(exp, tok(code)); @@ -1445,7 +1444,7 @@ private: const char code[] = "template\n" "T foo(T& value){ return value; }\n" "template std::vector> foo>>(std::vector>& v);"; - const char exp[] = "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & v ) ; " + const char exp[] = "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & value ) ; " "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & value ) { return value ; }"; ASSERT_EQUALS(exp, tok(code)); } @@ -1458,7 +1457,7 @@ private: "std::vector> v;\n" "v = foo>>(v);\n"; const char exp[] = "namespace NS { " - "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & v ) ; " + "std :: vector < std :: vector < int > > foo>> ( std :: vector < std :: vector < int > > & value ) ; " "} " "std :: vector < std :: vector < int > > v ; " "v = foo>> ( v ) ; " @@ -1637,6 +1636,21 @@ private: tok(code); } + void template86() { // crash + const char code[] = "struct S {\n" + " S();\n" + "};\n" + "template \n" + "struct U {\n" + " static S u;\n" + "};\n" + "template \n" + "S U::u;\n" + "template S U::u;\n" + "S &i = U::u;"; + tok(code); + } + void template_specialization_1() { // #7868 - template specialization template struct S> {..}; const char code[] = "template struct C {};\n" "template struct S {a};\n" @@ -2206,7 +2220,6 @@ private: "template < int type > struct Barney ; " "struct Barney<1> { } ; " "class Fred<1> ; " - "template class Fred<1> ; " "} " "class NS :: Fred<1> { " "public: " diff --git a/test/testsymboldatabase.cpp b/test/testsymboldatabase.cpp index e3fb1f421..1c2a37677 100644 --- a/test/testsymboldatabase.cpp +++ b/test/testsymboldatabase.cpp @@ -2391,7 +2391,7 @@ private: "public:\n" " int f() { return C< ::D,int>::f(); }\n" "};"); - ASSERT_EQUALS("[test.cpp:1]: (debug) simplifyTemplates: bailing out\n", errout.str()); + ASSERT_EQUALS("[test.cpp:6]: (debug) Failed to instantiate template \"C\". The checking continues anyway.\n", errout.str()); } void symboldatabase8() {