template simplifier: fix new daca crashes (#2093)

lib/templatesimplifier.cpp

@@ -1237,7 +1237,9 @@ void TemplateSimplifier::simplifyTemplateAliases()
                     break;
                 }
             }
-            if (!tok2 || tok2->str() != ">" || (!aliasDeclaration.isVariadic() && (args.size() != aliasParameters.size()))) {
+            if (!tok2 || tok2->str() != ">" ||
+                (!aliasDeclaration.isVariadic() && (args.size() != aliasParameters.size())) ||
+                (aliasDeclaration.isVariadic() && (args.size() < aliasParameters.size()))) {
                 ++it2;
                 continue;
             }
@@ -1309,23 +1311,13 @@ void TemplateSimplifier::simplifyTemplateAliases()
 
 bool TemplateSimplifier::instantiateMatch(const Token *instance, const std::size_t numberOfArguments, const char patternAfter[])
 {
-//    if (!Token::simpleMatch(instance, (name + " <").c_str()))
+    assert(instance->strAt(1) == "<");
-//        return false;
 
     if (numberOfArguments != templateParameters(instance->next()))
         return false;
 
     if (patternAfter) {
-        const Token *tok = instance;
+        const Token *tok = instance->next()->findClosingBracket();
-        unsigned int indentlevel = 0;
-        for (tok = instance; tok && (tok->str() != ">" || indentlevel > 0); tok = tok->next()) {
-            if (Token::Match(tok, "<|,|(|:: %name% <") && (tok->strAt(3) == ">" || templateParameters(tok->tokAt(2))))
-                ++indentlevel;
-            if (indentlevel > 0 && tok->str() == ">")
-                --indentlevel;
-            if (indentlevel > 0 && Token::Match(tok, "{|(|["))
-                tok = tok->link();
-        }
         if (!tok || !Token::Match(tok->next(), patternAfter))
             return false;
     }
@@ -1464,25 +1456,49 @@ void TemplateSimplifier::addNamespace(const TokenAndName &templateDeclaration, c
 
     std::string::size_type start = 0;
     std::string::size_type end = 0;
+    bool inTemplate = false;
+    int level = 0;
     while ((end = templateDeclaration.scope().find(" ", start)) != std::string::npos) {
         std::string token = templateDeclaration.scope().substr(start, end - start);
         // done if scopes overlap
         if (token == tokStart->str() && tok->strAt(-1) != "::")
             break;
-        if (insert)
+        if (token == "<") {
-            mTokenList.back()->tokAt(offset)->insertToken(token, "");
+            inTemplate = true;
-        else
+            ++level;
-            mTokenList.addtoken(token, tok->linenr(), tok->fileIndex());
+        }
+        if (inTemplate) {
+            if (insert)
+                mTokenList.back()->tokAt(offset)->str(mTokenList.back()->tokAt(offset)->str() + token);
+            else
+                mTokenList.back()->str(mTokenList.back()->str() + token);
+            if (token == ">") {
+                --level;
+                if (level == 0)
+                    inTemplate = false;
+            }
+        } else {
+            if (insert)
+                mTokenList.back()->tokAt(offset)->insertToken(token, "");
+            else
+                mTokenList.addtoken(token, tok->linenr(), tok->fileIndex());
+        }
         start = end + 1;
     }
     // don't add if it already exists
     std::string token = templateDeclaration.scope().substr(start, end - start);
     if (token != tokStart->str() || tok->strAt(-1) != "::") {
         if (insert) {
-            mTokenList.back()->tokAt(offset)->insertToken(templateDeclaration.scope().substr(start), "");
+            if (!inTemplate)
+                mTokenList.back()->tokAt(offset)->insertToken(templateDeclaration.scope().substr(start), "");
+            else
+                mTokenList.back()->tokAt(offset)->str(mTokenList.back()->tokAt(offset)->str() + templateDeclaration.scope().substr(start));
             mTokenList.back()->tokAt(offset)->insertToken("::", "");
         } else {
-            mTokenList.addtoken(templateDeclaration.scope().substr(start), tok->linenr(), tok->fileIndex());
+            if (!inTemplate)
+                mTokenList.addtoken(templateDeclaration.scope().substr(start), tok->linenr(), tok->fileIndex());
+            else
+                mTokenList.back()->str(mTokenList.back()->str() + templateDeclaration.scope().substr(start));
             mTokenList.addtoken("::", tok->linenr(), tok->fileIndex());
         }
     }
@@ -1842,7 +1858,7 @@ void TemplateSimplifier::expandTemplate(
                                         ++typeindentlevel;
                                     else if (typetok->str() == ")")
                                         --typeindentlevel;
-                                    mTokenList.addtoken(typetok, tok5->linenr(), tok5->fileIndex());
+                                    mTokenList.addtoken(typetok, tok5);
                                     Token *back = mTokenList.back();
                                     if (Token::Match(back, "{|(|[")) {
                                         brackets1.push(back);
@@ -1863,7 +1879,6 @@ void TemplateSimplifier::expandTemplate(
                                         brackets1.pop();
                                     }
                                     back->isTemplateArg(true);
-                                    back->col(tok5->col());
                                     added = true;
                                     break;
                                 }
@@ -1871,9 +1886,8 @@ void TemplateSimplifier::expandTemplate(
                         }
                     }
                     if (!added) {
-                        mTokenList.addtoken(tok5, tok5->linenr(), tok5->fileIndex());
+                        mTokenList.addtoken(tok5);
                         Token *back = mTokenList.back();
-                        back->col(tok5->col());
                         if (Token::Match(back, "{|(|[")) {
                             brackets2.push(back);
                         } else if (back->str() == "}") {
@@ -1922,6 +1936,7 @@ void TemplateSimplifier::expandTemplate(
         // FIXME use full name matching somehow
         const std::string lastName = (templateInstantiation.name().find(' ') != std::string::npos) ? templateInstantiation.name().substr(templateInstantiation.name().rfind(' ')+1) : templateInstantiation.name();
 
+        std::stack<const Token *> templates;
         for (; tok3; tok3 = tok3->next()) {
             if (tok3->isName() && !Token::Match(tok3, "class|typename|struct") && !tok3->isStandardType()) {
                 // search for this token in the type vector
@@ -1934,46 +1949,52 @@ void TemplateSimplifier::expandTemplate(
                     unsigned int typeindentlevel = 0;
                     std::stack<Token *> brackets1; // holds "(" and "{" tokens
                     for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
-                         typetok && (typeindentlevel>0 || !Token::Match(typetok, ",|>"));
+                         typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>"));
                          typetok = typetok->next()) {
                         if (Token::simpleMatch(typetok, ". . .")) {
                             typetok = typetok->tokAt(2);
                             continue;
                         }
-                        if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next())))
+                        if (Token::Match(typetok, "%name% <") &&
+                            (typetok->strAt(2) == ">" || templateParameters(typetok->next()))) {
+                            brackets1.push(typetok->next());
                             ++typeindentlevel;
-                        else if (typeindentlevel > 0 && typetok->str() == ">")
+                        } else if (typeindentlevel > 0 && typetok->str() == ">" && brackets1.top()->str() == "<") {
                             --typeindentlevel;
-                        else if (typetok->str() == "(")
+                            brackets1.pop();
+                        } else if (typetok->str() == "(")
                             ++typeindentlevel;
                         else if (typetok->str() == ")")
                             --typeindentlevel;
+                        Token *back;
                         if (copy) {
-                            mTokenList.addtoken(typetok, tok3->linenr(), tok3->fileIndex());
+                            mTokenList.addtoken(typetok, tok3);
-                            Token *back = mTokenList.back();
+                            back = mTokenList.back();
-                            back->col(tok3->col());
+                        } else
-                            if (Token::Match(back, "{|(|[")) {
+                            back = const_cast<Token *>(typetok);
-                                brackets1.push(back);
+                        if (Token::Match(back, "{|(|["))
-                            } else if (back->str() == "(") {
+                            brackets1.push(back);
-                                brackets1.push(back);
+                        else if (back->str() == "}") {
-                            } else if (back->str() == "}") {
+                            assert(brackets1.empty() == false);
-                                assert(brackets1.empty() == false);
+                            assert(brackets1.top()->str() == "{");
-                                assert(brackets1.top()->str() == "{");
+                            if (copy)
                                 Token::createMutualLinks(brackets1.top(), back);
-                                brackets1.pop();
+                            brackets1.pop();
-                            } else if (back->str() == ")") {
+                        } else if (back->str() == ")") {
-                                assert(brackets1.empty() == false);
+                            assert(brackets1.empty() == false);
-                                assert(brackets1.top()->str() == "(");
+                            assert(brackets1.top()->str() == "(");
+                            if (copy)
                                 Token::createMutualLinks(brackets1.top(), back);
-                                brackets1.pop();
+                            brackets1.pop();
-                            } else if (back->str() == "]") {
+                        } else if (back->str() == "]") {
-                                assert(brackets1.empty() == false);
+                            assert(brackets1.empty() == false);
-                                assert(brackets1.top()->str() == "[");
+                            assert(brackets1.top()->str() == "[");
+                            if (copy)
                                 Token::createMutualLinks(brackets1.top(), back);
-                                brackets1.pop();
+                            brackets1.pop();
-                            }
-                            back->isTemplateArg(true);
                         }
+                        if (copy)
+                            back->isTemplateArg(true);
                     }
                     continue;
                 }
@@ -1988,9 +2009,7 @@ void TemplateSimplifier::expandTemplate(
                         if (tok3 == templateDeclarationNameToken ||
                             Token::Match(tok3, newName.c_str())) {
                             if (copy) {
-                                mTokenList.addtoken(newName, tok3->linenr(), tok3->fileIndex());
+                                mTokenList.addtoken(newName, tok3);
-                                Token *back = mTokenList.back();
-                                back->col(tok3->col());
                                 tok3 = closingBracket;
                             } else {
                                 tok3->str(newName);
@@ -2011,9 +2030,7 @@ void TemplateSimplifier::expandTemplate(
                             (isClass ? tok3->strAt(1) != "(" : true)) {
                             addNamespace(templateDeclaration, tok3);
                         }
-                        mTokenList.addtoken(newName, tok3->linenr(), tok3->fileIndex());
+                        mTokenList.addtoken(newName, tok3);
-                        Token *back = mTokenList.back();
-                        back->col(tok3->col());
                     } else if (!Token::Match(tok3->next(), ":|{|="))
                         tok3->str(newName);
                     continue;
@@ -2021,11 +2038,16 @@ void TemplateSimplifier::expandTemplate(
             }
 
             // copy
-            if (copy) {
+            if (copy)
-                mTokenList.addtoken(tok3, tok3->linenr(), tok3->fileIndex());
+                mTokenList.addtoken(tok3);
-                Token *back = mTokenList.back();
+
-                back->col(tok3->col());
+            // look for template definitions
-            }
+            if (Token::Match(tok3, "template <")) {
+                Token * tok2 = findTemplateDeclarationEnd(tok3);
+                if (tok2)
+                    templates.push(tok2);
+            } else if (!templates.empty() && templates.top() == tok3)
+                templates.pop();
 
             if (Token::Match(tok3, "%type% <") &&
                 !Token::Match(tok3, "template|static_cast|const_cast|reinterpret_cast|dynamic_cast") &&
@@ -2052,10 +2074,14 @@ void TemplateSimplifier::expandTemplate(
                     else
                         scope = prev->str() + " :: " + scope;
                 }
-                if (copy)
+
-                    newInstantiations.emplace_back(mTokenList.back(), scope);
+                // don't add instantiations in template definitions
-                else if (!inTemplateDefinition)
+                if (templates.empty()) {
-                    newInstantiations.emplace_back(tok3, scope);
+                    if (copy)
+                        newInstantiations.emplace_back(mTokenList.back(), scope);
+                    else if (!inTemplateDefinition)
+                        newInstantiations.emplace_back(tok3, scope);
+                }
             }
 
             // link() newly tokens manually
@@ -3488,6 +3514,11 @@ void TemplateSimplifier::simplifyTemplates(
         if (mTemplateDeclarations.empty() && mTemplateForwardDeclarations.empty())
             return;
 
+        if (passCount != 0 && mSettings->debugtemplate && mSettings->debugnormal) {
+            std::string title("Template Simplifier pass " + std::to_string(passCount + 1));
+            mTokenList.front()->printOut(title.c_str(), mTokenList.getFiles());
+        }
+
         // Copy default argument values from forward declaration to declaration
         fixForwardDeclaredDefaultArgumentValues();
 

lib/tokenlist.cpp

@@ -155,6 +155,40 @@ void TokenList::addtoken(std::string str, const nonneg int lineno, const nonneg
     mTokensFrontBack.back->fileIndex(fileno);
 }
 
+void TokenList::addtoken(std::string str, const Token *locationTok)
+{
+    if (str.empty())
+        return;
+
+    // Replace hexadecimal value with decimal
+    const bool isHex = MathLib::isIntHex(str) ;
+    if (isHex || MathLib::isOct(str) || MathLib::isBin(str)) {
+        // TODO: It would be better if TokenList didn't simplify hexadecimal numbers
+        std::string suffix;
+        if (isHex &&
+            str.size() == (2 + mSettings->int_bit / 4) &&
+            (str[2] >= '8') &&  // includes A-F and a-f
+            MathLib::getSuffix(str).empty()
+           )
+            suffix = "U";
+        str = MathLib::value(str).str() + suffix;
+    }
+
+    if (mTokensFrontBack.back) {
+        mTokensFrontBack.back->insertToken(str);
+    } else {
+        mTokensFrontBack.front = new Token(&mTokensFrontBack);
+        mTokensFrontBack.back = mTokensFrontBack.front;
+        mTokensFrontBack.back->str(str);
+    }
+
+    if (isCPP() && str == "delete")
+        mTokensFrontBack.back->isKeyword(true);
+    mTokensFrontBack.back->linenr(locationTok->linenr());
+    mTokensFrontBack.back->col(locationTok->col());
+    mTokensFrontBack.back->fileIndex(locationTok->fileIndex());
+}
+
 void TokenList::addtoken(const Token * tok, const nonneg int lineno, const nonneg int fileno)
 {
     if (tok == nullptr)
@@ -175,6 +209,48 @@ void TokenList::addtoken(const Token * tok, const nonneg int lineno, const nonne
     mTokensFrontBack.back->flags(tok->flags());
 }
 
+void TokenList::addtoken(const Token *tok, const Token *locationTok)
+{
+    if (tok == nullptr || locationTok == nullptr)
+        return;
+
+    if (mTokensFrontBack.back) {
+        mTokensFrontBack.back->insertToken(tok->str(), tok->originalName());
+    } else {
+        mTokensFrontBack.front = new Token(&mTokensFrontBack);
+        mTokensFrontBack.back = mTokensFrontBack.front;
+        mTokensFrontBack.back->str(tok->str());
+        if (!tok->originalName().empty())
+            mTokensFrontBack.back->originalName(tok->originalName());
+    }
+
+    mTokensFrontBack.back->flags(tok->flags());
+    mTokensFrontBack.back->linenr(locationTok->linenr());
+    mTokensFrontBack.back->col(locationTok->col());
+    mTokensFrontBack.back->fileIndex(locationTok->fileIndex());
+}
+
+void TokenList::addtoken(const Token *tok)
+{
+    if (tok == nullptr)
+        return;
+
+    if (mTokensFrontBack.back) {
+        mTokensFrontBack.back->insertToken(tok->str(), tok->originalName());
+    } else {
+        mTokensFrontBack.front = new Token(&mTokensFrontBack);
+        mTokensFrontBack.back = mTokensFrontBack.front;
+        mTokensFrontBack.back->str(tok->str());
+        if (!tok->originalName().empty())
+            mTokensFrontBack.back->originalName(tok->originalName());
+    }
+
+    mTokensFrontBack.back->flags(tok->flags());
+    mTokensFrontBack.back->linenr(tok->linenr());
+    mTokensFrontBack.back->col(tok->col());
+    mTokensFrontBack.back->fileIndex(tok->fileIndex());
+}
+
 
 //---------------------------------------------------------------------------
 // copyTokens - Copy and insert tokens

lib/tokenlist.h

@@ -70,7 +70,11 @@ public:
     static void deleteTokens(Token *tok);
 
     void addtoken(std::string str, const nonneg int lineno, const nonneg int fileno, bool split = false);
+    void addtoken(std::string str, const Token *locationTok);
+
     void addtoken(const Token *tok, const nonneg int lineno, const nonneg int fileno);
+    void addtoken(const Token *tok, const Token *locationTok);
+    void addtoken(const Token *tok);
 
     static void insertTokens(Token *dest, const Token *src, nonneg int n);
 

test/testsimplifytemplate.cpp

@@ -170,6 +170,9 @@ private:
         TEST_CASE(template130); // #9246
         TEST_CASE(template131); // #9249
         TEST_CASE(template132); // #9250
+        TEST_CASE(template133);
+        TEST_CASE(template134);
+        TEST_CASE(template135);
         TEST_CASE(template_specialization_1);  // #7868 - template specialization template <typename T> struct S<C<T>> {..};
         TEST_CASE(template_specialization_2);  // #7868 - template specialization template <typename T> struct S<C<T>> {..};
         TEST_CASE(template_enum);  // #6299 Syntax error in complex enum declaration (including template)
@@ -3157,6 +3160,76 @@ private:
         ASSERT_EQUALS(exp, tok(code));
     }
 
+    void template133() {
+        const char code[] = "template <typename a> struct bar {\n"
+                            "  template <typename b> static bar foo(const bar<b> &c) {\n"
+                            "    return bar();\n"
+                            "  }\n"
+                            "};\n"
+                            "bar<short> bs;\n"
+                            "bar<std::array<int,4>> ba;\n"
+                            "bar<short> b1 = bar<short>::foo<std::array<int,4>>(ba);\n"
+                            "bar<std::array<int,4>> b2 = bar<std::array<int,4>>::foo<short>(bs);";
+        const char act[] = "struct bar<short> ; struct bar<std::array<int,4>> ; "
+                           "bar<short> bs ; "
+                           "bar<std::array<int,4>> ba ; "
+                           "bar<short> b1 ; b1 = bar<short> :: foo<std::array<int,4>> ( ba ) ; "
+                           "bar<std::array<int,4>> b2 ; b2 = bar<std::array<int,4>> :: foo<short> ( bs ) ; "
+                           "struct bar<short> { "
+                           "static bar<short> foo<std::array<int,4>> ( const bar < std :: array < int , 4 > > & c ) ; "
+                           "} ; "
+                           "struct bar<std::array<int,4>> { "
+                           "static bar<std::array<int,4>> foo<short> ( const bar < short > & c ) ; "
+                           "} ; "
+                           "bar<std::array<int,4>> bar<std::array<int,4>> :: foo<short> ( const bar < short > & c ) { "
+                           "return bar<std::array<int,4>> ( ) ; "
+                           "} "
+                           "bar<short> bar<short> :: foo<std::array<int,4>> ( const bar < std :: array < int , 4 > > & c ) { "
+                           "return bar<short> ( ) ; "
+                           "}";
+        const char exp[] = "struct bar<short> ; struct bar<std::array<int,4>> ; "
+                           "bar<short> bs ; "
+                           "bar<std::array<int,4>> ba ; "
+                           "bar<short> b1 ; b1 = bar<short> :: foo<std::array<int,4>> ( ba ) ; "
+                           "bar<std::array<int,4>> b2 ; b2 = bar<std::array<int,4>> :: foo<short> ( bs ) ; "
+                           "struct bar<short> { "
+                           "static bar<short> foo<std::array<int,4>> ( const bar<std::array<int,4>> & c ) ; "
+                           "} ; "
+                           "struct bar<std::array<int,4>> { "
+                           "static bar<std::array<int,4>> foo<short> ( const bar<short> & c ) ; "
+                           "} ; "
+                           "bar<std::array<int,4>> bar<std::array<int,4>> :: foo<short> ( const bar<short> & c ) { "
+                           "return bar<std::array<int,4>> ( ) ; "
+                           "} "
+                           "bar<short> bar<short> :: foo<std::array<int,4>> ( const bar<std::array<int,4>> & c ) { "
+                           "return bar<short> ( ) ; "
+                           "}";
+        TODO_ASSERT_EQUALS(exp, act, tok(code));
+    }
+
+    void template134() {
+        const char code[] = "template <int a> class e { };\n"
+                            "template <int a> class b { e<(c > a ? 1 : 0)> d; };\n"
+                            "b<0> b0;\n"
+                            "b<1> b1;";
+        const char exp[] = "class e<(c>0)> ; class e<(c>1)> ; "
+                           "class b<0> ; class b<1> ; "
+                           "b<0> b0 ; "
+                           "b<1> b1 ; "
+                           "class b<0> { e<(c>0)> d ; } ; class b<1> { e<(c>1)> d ; } ; "
+                           "class e<(c>0)> { } ; class e<(c>1)> { } ;";
+        ASSERT_EQUALS(exp, tok(code));
+    }
+
+    void template135() {
+        const char code[] = "template <int> struct a { template <int b> void c(a<b>); };\n"
+                            "a<2> d;";
+        const char exp[] = "struct a<2> ; "
+                           "a<2> d ; "
+                           "struct a<2> { template < int b > void c ( a < b > ) ; } ;";
+        ASSERT_EQUALS(exp, tok(code));
+    }
+
     void template_specialization_1() {  // #7868 - template specialization template <typename T> struct S<C<T>> {..};
         const char code[] = "template <typename T> struct C {};\n"
                             "template <typename T> struct S {a};\n"