diff --git a/lib/templatesimplifier.cpp b/lib/templatesimplifier.cpp index 7e223feec..4a121e642 100644 --- a/lib/templatesimplifier.cpp +++ b/lib/templatesimplifier.cpp @@ -206,6 +206,26 @@ TemplateSimplifier::~TemplateSimplifier() { } +void TemplateSimplifier::fixAngleBrackets() +{ + for (Token *tok = mTokenList.front(); tok; tok = tok->next()) { + // Ticket #6181: normalize C++11 template parameter list closing syntax + if (tok->str() == "<" && templateParameters(tok)) { + Token *endTok = tok->findClosingBracket(); + if (endTok && endTok->str() == ">>") { + endTok->str(">"); + endTok->insertToken(">"); + } + } else if (Token::Match(tok, "class|struct|union|=|:|public|protected|private %name% <")) { + Token *endTok = tok->tokAt(2)->findClosingBracket(); + if (Token::Match(endTok, ">> ;|{")) { + endTok->str(">"); + endTok->insertToken(">"); + } + } + } +} + void TemplateSimplifier::cleanupAfterSimplify() { bool goback = false; @@ -368,6 +388,8 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok) if (Token::Match(tok->previous(), "%var% <")) return 0; tok = tok->next(); + if (tok->str() == ">") + return 0; unsigned int level = 0; @@ -375,9 +397,18 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok) // skip template template if (level == 0 && Token::simpleMatch(tok, "template <")) { const Token *closing = tok->next()->findClosingBracket(); - if (closing) + if (closing) { + if (closing->str() == ">>") + return numberOfParameters; tok = closing->next(); - else + if (tok->str() == ">" || tok->str() == ">>") + return numberOfParameters; + else if (tok->str() == ",") { + ++numberOfParameters; + tok = tok->next(); + continue; + } + } else return 0; } @@ -394,9 +425,25 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok) tok = tok->next(); // Skip variadic types (Ticket #5774, #6059, #6172) - if (Token::Match(tok, "%type% . . .")) { - tok = tok->tokAt(4); - continue; + if (Token::simpleMatch(tok, ". . .")) { + if ((tok->previous()->isName() && !Token::Match(tok->tokAt(-2), "<|,")) || + (!tok->previous()->isName() && tok->strAt(-1) != ">")) + return 0; // syntax error + tok = tok->tokAt(3); + if (!tok) + return 0; + if (tok->str() == ">") { + if (level == 0) + return numberOfParameters; + --level; + } else if (tok->str() == ">>") { + if (level == 1) + return numberOfParameters; + } else if (tok->str() == "," && level == 0) { + ++numberOfParameters; + tok = tok->next(); + continue; + } } // Skip '=', '?', ':' @@ -414,6 +461,8 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok) return 0; if (tok->str() == ">" && level == 0) return numberOfParameters; + else if (tok->str() == ">>" && level == 1) + return numberOfParameters; else if (tok->str() == "," && level == 0) { ++numberOfParameters; tok = tok->next(); @@ -900,8 +949,8 @@ void TemplateSimplifier::getTemplateInstantiations() // parse backwards and add template instantiations // TODO for (; tok2 && tok2 != tok; tok2 = tok2->previous()) { - if (Token::Match(tok2, ", %name% <") && - templateParameters(tok2->tokAt(2))) { + if (Token::Match(tok2, ",|< %name% <") && + (tok2->strAt(3) == ">" || templateParameters(tok2->tokAt(2)))) { addInstantiation(tok2->next(), getScopeName(scopeList)); } else if (Token::Match(tok2->next(), "class|struct")) tok2->deleteNext(); @@ -1351,7 +1400,14 @@ bool TemplateSimplifier::getTemplateNamePositionTemplateVariable(const Token *to while (tok && tok->next()) { if (Token::Match(tok->next(), ";|{|(|using")) return false; - else if (Token::Match(tok->next(), "%type% <")) { + // skip decltype(...) + else if (Token::simpleMatch(tok->next(), "decltype (")) { + const Token * end = tok->linkAt(2); + while (tok && tok->next() && tok != end) { + tok = tok->next(); + namepos++; + } + } else if (Token::Match(tok->next(), "%type% <")) { const Token *closing = tok->tokAt(2)->findClosingBracket(); if (closing) { if (Token::Match(closing->next(), "=|;")) @@ -1401,8 +1457,7 @@ bool TemplateSimplifier::getTemplateNamePositionTemplateClass(const Token *tok, int TemplateSimplifier::getTemplateNamePosition(const Token *tok) { - // FIXME: tok == ">>" is a tokenizer bug that needs to be fixed - assert(tok && (tok->str() == ">" || tok->str() == ">>")); + assert(tok && tok->str() == ">"); auto it = mTemplateNamePos.find(tok); if (!mSettings->debugtemplate && it != mTemplateNamePos.end()) { @@ -1540,8 +1595,15 @@ void TemplateSimplifier::expandTemplate( end = end->findClosingBracket()->next(); if (end->str() == "(") end = end->link()->next(); - else if (isVariable && end->str() == "=") - end = const_cast(Token::findsimplematch(templateDeclarationNameToken, ";")); + else if (isVariable && end->str() == "=") { + Token *temp = end->next(); + while (temp && temp->str() != ";") { + if (temp->link() && Token::Match(temp, "{|[|(")) + temp = temp->link(); + temp = temp->next(); + } + end = temp; + } } unsigned int typeindentlevel = 0; while (!(typeindentlevel == 0 && Token::Match(end, ";|{|:"))) { @@ -1697,9 +1759,17 @@ void TemplateSimplifier::expandTemplate( } if (inTemplateDefinition) { if (!endOfTemplateDefinition) { - if (isVariable) - endOfTemplateDefinition = Token::findsimplematch(tok3, ";"); - else if (tok3->str() == "{") + if (isVariable) { + Token *temp = tok3->findClosingBracket(); + if (temp) { + while (temp && temp->str() != ";") { + if (temp->link() && Token::Match(temp, "{|[|(")) + temp = temp->link(); + temp = temp->next(); + } + endOfTemplateDefinition = temp; + } + } else if (tok3->str() == "{") endOfTemplateDefinition = tok3->link(); } if (tok3 == endOfTemplateDefinition) { @@ -1773,7 +1843,7 @@ void TemplateSimplifier::expandTemplate( } } else if (copy) { bool added = false; - if (tok5->isName() && !Token::Match(tok5, "class|typename|struct")) { + if (tok5->isName() && !Token::Match(tok5, "class|typename|struct") && !tok5->isStandardType()) { // search for this token in the type vector unsigned int itype = 0; while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != tok5->str()) @@ -1852,7 +1922,7 @@ void TemplateSimplifier::expandTemplate( const std::string lastName = (templateInstantiation.name.find(' ') != std::string::npos) ? templateInstantiation.name.substr(templateInstantiation.name.rfind(' ')+1) : templateInstantiation.name; for (; tok3; tok3 = tok3->next()) { - if (tok3->isName() && !Token::Match(tok3, "class|typename|struct")) { + if (tok3->isName() && !Token::Match(tok3, "class|typename|struct") && !tok3->isStandardType()) { // search for this token in the type vector unsigned int itype = 0; while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != tok3->str()) @@ -2746,7 +2816,7 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( } if (Token::Match(startToken->previous(), ";|{|}|=|const") && - (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : isVar ? ";|%op%" : "*|&|::| %name%"))) + (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : isVar ? ";|%op%|(" : "*|&|::| %name%"))) continue; // New type.. @@ -2808,7 +2878,7 @@ bool TemplateSimplifier::simplifyTemplateInstantiations( } if (Token::Match(startToken->previous(), ";|{|}|=|const") && - (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : isVar ? ";|%op%" : "*|&|::| %name%"))) + (!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : isVar ? ";|%op%|(" : "*|&|::| %name%"))) return false; // already simplified @@ -3282,6 +3352,9 @@ void TemplateSimplifier::simplifyTemplates( const std::time_t maxtime, bool &codeWithTemplates) { + // split ">>" into "> >" + fixAngleBrackets(); + // Remove "typename" unless used in template arguments or using type alias.. for (Token *tok = mTokenList.front(); tok; tok = tok->next()) { if (Token::Match(tok, "typename %name%") && !Token::Match(tok->tokAt(-3), "using %name% =")) diff --git a/lib/templatesimplifier.h b/lib/templatesimplifier.h index a90ba8ac8..1d1ad4112 100644 --- a/lib/templatesimplifier.h +++ b/lib/templatesimplifier.h @@ -294,6 +294,11 @@ public: */ void simplifyTemplateArgs(Token *start, Token *end); + /** Fix angle brackets. + * foo < bar < >> => foo < bar < > > + */ + void fixAngleBrackets(); + private: /** * Get template declarations diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index cf199aa89..55ce26499 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -2817,23 +2817,6 @@ void Tokenizer::simplifyTemplates() if (isC()) return; - for (Token *tok = list.front(); tok; tok = tok->next()) { - // Ticket #6181: normalize C++11 template parameter list closing syntax - if (tok->str() == "<" && mTemplateSimplifier->templateParameters(tok)) { - Token *endTok = tok->findClosingBracket(); - if (endTok && endTok->str() == ">>") { - endTok->str(">"); - endTok->insertToken(">"); - } - } else if (Token::Match(tok, "class|struct|union|=|:|public|protected|private %name% <")) { - Token *endTok = tok->tokAt(2)->findClosingBracket(); - if (Token::Match(endTok, ">> ;|{")) { - endTok->str(">"); - endTok->insertToken(">"); - } - } - } - mTemplateSimplifier->simplifyTemplates( #ifdef MAXTIME mMaxTime, diff --git a/test/testsimplifytemplate.cpp b/test/testsimplifytemplate.cpp index e7dce5942..2a6968827 100644 --- a/test/testsimplifytemplate.cpp +++ b/test/testsimplifytemplate.cpp @@ -154,6 +154,9 @@ private: TEST_CASE(template114); // #9155 TEST_CASE(template115); // #9153 TEST_CASE(template116); // #9178 + TEST_CASE(template117); + TEST_CASE(template118); + TEST_CASE(template119); // #9186 TEST_CASE(template_specialization_1); // #7868 - template specialization template struct S> {..}; TEST_CASE(template_specialization_2); // #7868 - template specialization template struct S> {..}; TEST_CASE(template_enum); // #6299 Syntax error in complex enum declaration (including template) @@ -207,6 +210,7 @@ private: TEST_CASE(template_variable_1); TEST_CASE(template_variable_2); TEST_CASE(template_variable_3); + TEST_CASE(template_variable_4); } std::string tok(const char code[], bool debugwarnings = false, Settings::PlatformType type = Settings::Native) { @@ -945,19 +949,19 @@ private: " A> gna1;\n" " A gna2;\n" "}\n"; - const char expected[] = "class A> ; " - "class A ; " + const char expected[] = "class A ; " + "class A> ; " "int main ( ) { " "A> gna1 ; " "A gna2 ; " "} " - "class A> { " - "A mT ; " + "class A { " + "BLA mT ; " "public: " "void foo ( ) { } " "} ; " - "class A { " - "BLA mT ; " + "class A> { " + "A mT ; " "public: " "void foo ( ) { } " "} ;"; @@ -2763,6 +2767,58 @@ private: ASSERT_EQUALS(exp, tok(code)); } + void template117() { + const char code[] = "template struct X {};\n" + "X> x;"; + const char exp[] = "struct X ; " + "struct X> ; " + "X> x ; " + "struct X { } ; " + "struct X> { } ;"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template118() { + const char code[] = "template struct S { void f(int i); };\n" + "S<1> s;"; + const char exp[] = "struct S<1> ; " + "S<1> s ; struct S<1> { " + "void f ( int i ) ; " + "} ;"; + ASSERT_EQUALS(exp, tok(code)); + } + + void template119() { // #9186 + { + const char code[] = "template \n" + "constexpr auto func = [](auto x){ return T(x);};\n" + "template \n" + "constexpr auto funcBraced = [](auto x){ return T{x};};\n" + "double f(int x) { return func(x); }\n" + "double fBraced(int x) { return funcBraced(x); }"; + const char exp[] = "const auto func = [ ] ( auto x ) { return double ( x ) ; } ; " + "const auto funcBraced = [ ] ( auto x ) { return int { x } ; } ; " + "double f ( int x ) { return func ( x ) ; } " + "double fBraced ( int x ) { return funcBraced ( x ) ; }"; + ASSERT_EQUALS(exp, tok(code)); + } + { + const char code[] = "template \n" + "constexpr auto func = [](auto x){ return T(x);};\n" + "void foo() {\n" + " func(x);\n" + " func(x);\n" + "}"; + const char exp[] = "const auto func = [ ] ( auto x ) { return int ( x ) ; } ; " + "const auto func = [ ] ( auto x ) { return double ( x ) ; } ; " + "void foo ( ) { " + "func ( x ) ; " + "func ( x ) ; " + "}"; + ASSERT_EQUALS(exp, tok(code)); + } + } + void template_specialization_1() { // #7868 - template specialization template struct S> {..}; const char code[] = "template struct C {};\n" "template struct S {a};\n" @@ -3433,11 +3489,13 @@ private: Tokenizer tokenizer(&settings, this); std::istringstream istr(code); - tokenizer.tokenize(istr, "test.cpp", ""); + tokenizer.createTokens(istr, "test.cpp"); + tokenizer.createLinks(); + tokenizer.mTemplateSimplifier->fixAngleBrackets(); - for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) { - if (tok->str() == "var1") - (const_cast(tok))->varId(1); + for (const Token *tok1 = tokenizer.tokens(); tok1; tok1 = tok1->next()) { + if (tok1->str() == "var1") + (const_cast(tok1))->varId(1); } return TemplateSimplifier::templateParameters(tokenizer.tokens()->next()); @@ -3459,14 +3517,28 @@ private: ASSERT_EQUALS(2U, templateParameters("X x;")); ASSERT_EQUALS(2U, templateParameters("X x;")); ASSERT_EQUALS(3U, templateParameters("X x;")); - TODO_ASSERT_EQUALS(1U, 0U, templateParameters("X x;")); // Mishandled valid syntax - TODO_ASSERT_EQUALS(2U, 0U, templateParameters("X x;")); // Mishandled valid syntax + ASSERT_EQUALS(1U, templateParameters("X x;")); + ASSERT_EQUALS(2U, templateParameters("X x;")); ASSERT_EQUALS(2U, templateParameters("X<1, T> x;")); ASSERT_EQUALS(1U, templateParameters("X x;")); ASSERT_EQUALS(2U, templateParameters("X=0> x;")); ASSERT_EQUALS(3U, templateParameters("X=0, i - 2> x;")); ASSERT_EQUALS(0U, templateParameters("var1<1> x;")); ASSERT_EQUALS(0U, templateParameters("X<1>2;")); + ASSERT_EQUALS(2U, templateParameters("template> x;")); + ASSERT_EQUALS(2U, templateParameters("template > x;")); + ASSERT_EQUALS(1U, templateParameters("template...Foo> x;")); + ASSERT_EQUALS(1U, templateParameters("template> x;")); + ASSERT_EQUALS(1U, templateParameters("template>> x;")); + ASSERT_EQUALS(1U, templateParameters("template>>> x;")); + ASSERT_EQUALS(1U, templateParameters("template>>>> x;")); + ASSERT_EQUALS(2U, templateParameters("template,int> x;")); + ASSERT_EQUALS(2U, templateParameters("template>,int> x;")); + ASSERT_EQUALS(2U, templateParameters("template>>,int> x;")); + ASSERT_EQUALS(2U, templateParameters("template>>>,int> x;")); + ASSERT_EQUALS(2U, templateParameters("template...Foo,template>>> x;")); + ASSERT_EQUALS(3U, templateParameters("template...Foo,int,template>>> x;")); + ASSERT_EQUALS(4U, templateParameters("template...Foo,int,template>>,int> x;")); } // Helper function to unit test TemplateSimplifier::getTemplateNamePosition @@ -3474,7 +3546,9 @@ private: Tokenizer tokenizer(&settings, this); std::istringstream istr(code); - tokenizer.tokenize(istr, "test.cpp", emptyString); + tokenizer.createTokens(istr, "test.cpp"); + tokenizer.createLinks(); + tokenizer.mTemplateSimplifier->fixAngleBrackets(); const Token *_tok = tokenizer.tokens(); for (unsigned i = 0 ; i < offset ; ++i) @@ -4040,6 +4114,16 @@ private: } } + void template_variable_4() { + const char code[] = "template void test() { }\n" + "template decltype(test)* foo = &(test);\n" + "void bar() { foo(); }"; + const char expected[] = "void test ( ) ; " + "decltype ( test ) * foo = & ( test ) ; " + "void bar ( ) { foo ( ) ; } " + "void test ( ) { }"; + ASSERT_EQUALS(expected, tok(code)); + } }; REGISTER_TEST(TestSimplifyTemplate)