token simplifier: fix namespace, token link and syntax error support for template type aliases (#1863)

lib/templatesimplifier.cpp

@@ -81,6 +81,14 @@ TemplateSimplifier::TokenAndName::TokenAndName(Token *tok, const std::string &s,
         isSpecialization(Token::simpleMatch(token, "template < >"));
         isPartialSpecialization(!isSpecialization() && nameToken->strAt(1) == "<");
         isAlias(paramEnd->strAt(1) == "using");
+
+        if (isAlias() && isPartialSpecialization()) {
+            throw InternalError(tok, "partial specialization of alias templates is not permitted", InternalError::SYNTAX);
+        }
+        if (isAlias() && isSpecialization()) {
+            throw InternalError(tok, "explicit specialization of alias templates is not permitted", InternalError::SYNTAX);
+        }
+
         isClass(Token::Match(paramEnd->next(), "class|struct|union %name% <|{|:|;"));
         if (token->strAt(1) == "<" && !isSpecialization()) {
             const Token *end = token->next()->findClosingBracket();
@@ -95,10 +103,10 @@ TemplateSimplifier::TokenAndName::TokenAndName(Token *tok, const std::string &s,
                 throw InternalError(tok, "unsupported syntax", InternalError::SYNTAX);
         }
         isFunction(tok1->str() == "(");
-        isVariable(!isClass() && Token::Match(tok1, "=|;"));
+        isVariable(!isClass() && !isAlias() && Token::Match(tok1, "=|;"));
         if (isVariable())
             isForwardDeclaration(tok1->str() == ";");
-        else {
+        else if (!isAlias()) {
             if (isFunction())
                 tok1 = tok1->link()->next();
             tok1 = Token::findmatch(tok1, "{|;");
@@ -156,6 +164,31 @@ TemplateSimplifier::TokenAndName::~TokenAndName()
         token->templateSimplifierPointers().erase(this);
 }
 
+const Token * TemplateSimplifier::TokenAndName::aliasStartToken() const
+{
+    if (paramEnd)
+        return paramEnd->tokAt(4);
+    return nullptr;
+}
+
+const Token * TemplateSimplifier::TokenAndName::aliasEndToken() const
+{
+    if (aliasStartToken())
+        return Token::findsimplematch(aliasStartToken(), ";");
+    return nullptr;
+}
+
+bool TemplateSimplifier::TokenAndName::isAliasToken(const Token *tok) const
+{
+    const Token *end = aliasEndToken();
+
+    for (const Token *tok1 = aliasStartToken(); tok1 != end; tok1 = tok1->next()) {
+        if (tok1 == tok)
+            return true;
+    }
+    return false;
+}
+
 TemplateSimplifier::TemplateSimplifier(Tokenizer *tokenizer)
     : mTokenizer(tokenizer), mTokenList(tokenizer->list), mSettings(tokenizer->mSettings), mErrorLogger(tokenizer->mErrorLogger)
 {
@@ -711,13 +744,20 @@ void TemplateSimplifier::getTemplateInstantiations()
             setScopeInfo(tok, &scopeList);
             continue;
         }
+
         // template definition.. skip it
         if (Token::simpleMatch(tok, "template <")) {
             tok = tok->next()->findClosingBracket();
             if (!tok)
                 break;
+
             const bool isUsing = tok->strAt(1) == "using";
-            if (tok->strAt(-1) == "<") {
+            if (isUsing && Token::Match(tok->tokAt(2), "%name% <")) {
+                // Cant have specialized type alias so ignore it
+                const Token *tok2 = Token::findsimplematch(tok->tokAt(3), ";");
+                if (tok2)
+                    tok = const_cast<Token *>(tok2);
+            } else if (tok->strAt(-1) == "<") {
                 // Don't ignore user specialization but don't consider it an instantiation.
                 // Instantiations in return type, function parameters, and executable code
                 // are not ignored.
@@ -734,7 +774,17 @@ void TemplateSimplifier::getTemplateInstantiations()
                 else if (!isUsing && tok2 && tok2->str() == ";")
                     tok = const_cast<Token *>(tok2);
             }
-        } else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|.|*|&|return|< %name% ::|<|(") ||
+        } else if (Token::Match(tok, "template using %name% <")) {
+            // Cant have specialized type alias so ignore it
+            const Token *tok2 = Token::findsimplematch(tok->tokAt(3), ";");
+            if (tok2)
+                tok = const_cast<Token *>(tok2);
+        } else if (Token::Match(tok, "using %name% <")) {
+            // Cant have specialized type alias so ignore it
+            const Token *tok2 = Token::findsimplematch(tok->tokAt(2), ";");
+            if (tok2)
+                tok = const_cast<Token *>(tok2);
+        } else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|.|*|&|return|<|, %name% ::|<|(") ||
                    Token::Match(tok->previous(), "%type% %name% ::|<") ||
                    Token::Match(tok->tokAt(-2), "[,:] private|protected|public %name% ::|<")) {
             std::string scopeName = getScopeName(scopeList);
@@ -904,9 +954,7 @@ void TemplateSimplifier::useDefaultArgumentValues()
 void TemplateSimplifier::useDefaultArgumentValues(TokenAndName &declaration)
 {
     // Ticket #5762: Skip specialization tokens
-    // FIXME: should be: if (declaration.isSpecialization() || declaration.isAlias())
+    if (declaration.isSpecialization() || declaration.isAlias())
-    // but there is a problem with functions
-    if (!declaration.isClass())
         return;
 
     // template parameters with default value has syntax such as:
@@ -927,7 +975,7 @@ void TemplateSimplifier::useDefaultArgumentValues(TokenAndName &declaration)
 
     // Scan template declaration..
     for (Token *tok = declaration.token; tok; tok = tok->next()) {
-        if (tok->link()) { // Ticket #6835
+        if (tok->link() && Token::Match(tok, "{|(|[")) { // Ticket #6835
             tok = tok->link();
             continue;
         }
@@ -967,20 +1015,19 @@ void TemplateSimplifier::useDefaultArgumentValues(TokenAndName &declaration)
 
     // iterate through all template instantiations
     for (const TokenAndName &instantiation : mTemplateInstantiations) {
-        Token *tok = instantiation.token;
+        if (declaration.fullName != instantiation.fullName)
-
-        if (!Token::simpleMatch(tok, (declaration.name + " <").c_str()))
             continue;
 
         // instantiation arguments..
         std::vector<std::vector<const Token *>> instantiationArgs;
         std::size_t index = 0;
-        instantiationArgs.resize(1);
         const Token *end = instantiation.token->next()->findClosingBracket();
         if (!end)
             continue;
+        if (end != instantiation.token->tokAt(2))
+            instantiationArgs.resize(1);
         for (const Token *tok1 = instantiation.token->tokAt(2); tok1 && tok1!= end; tok1 = tok1->next()) {
-            if (tok1->link()) {
+            if (tok1->link() && Token::Match(tok1, "{|(|[")) {
                 const Token *endLink = tok1->link();
                 do {
                     instantiationArgs[index].push_back(tok1);
@@ -1000,8 +1047,8 @@ void TemplateSimplifier::useDefaultArgumentValues(TokenAndName &declaration)
         }
 
         // count the parameters..
-        tok = tok->next();
+        Token *tok = instantiation.token->next();
-        const unsigned int usedpar = templateParameters(tok);
+        unsigned int usedpar = templateParameters(tok);
         tok = tok->findClosingBracket();
 
         if (tok && tok->str() == ">") {
@@ -1009,9 +1056,10 @@ void TemplateSimplifier::useDefaultArgumentValues(TokenAndName &declaration)
             std::list<Token *>::const_iterator it = eq.begin();
             for (std::size_t i = (templatepar - eq.size()); it != eq.end() && i < usedpar; ++i)
                 ++it;
+            int count = 0;
             while (it != eq.end()) {
                 int indentlevel = 0;
-                if (usedpar && usedpar <= instantiationArgs.size()) {
+                if ((usedpar + count) && usedpar <= (instantiationArgs.size() + count)) {
                     tok->insertToken(",");
                     tok = tok->next();
                 }
@@ -1027,20 +1075,30 @@ void TemplateSimplifier::useDefaultArgumentValues(TokenAndName &declaration)
                         for (const Token *tok1 : instantiationArgs[entry->second]) {
                             tok->insertToken(tok1->str(), tok1->originalName());
                             tok = tok->next();
+
+                            if (Token::Match(tok, "(|[|{"))
+                                links.push(tok);
+                            else if (!links.empty() && Token::Match(tok, ")|]|}")) {
+                                Token::createMutualLinks(links.top(), tok);
+                                links.pop();
+                            }
                         }
                     } else {
                         tok->insertToken(from->str(), from->originalName());
                         tok = tok->next();
-                    }
+
-                    if (Token::Match(tok, "(|[|{"))
+                        if (Token::Match(tok, "(|[|{"))
-                        links.push(tok);
+                            links.push(tok);
-                    else if (!links.empty() && Token::Match(tok, ")|]|}")) {
+                        else if (!links.empty() && Token::Match(tok, ")|]|}")) {
-                        Token::createMutualLinks(links.top(), tok);
+                            Token::createMutualLinks(links.top(), tok);
-                        links.pop();
+                            links.pop();
+                        }
                     }
                     from = from->next();
                 }
                 ++it;
+                count++;
+                usedpar++;
             }
         }
     }
@@ -1106,8 +1164,6 @@ void TemplateSimplifier::simplifyTemplateAliases()
               Token::Match(startToken->tokAt(-5), "> using %name% = typename %name% ::|<")))
             continue;
         const bool hasTypename(startToken->strAt(-1) == "typename");
-        const std::string aliasName(startToken->strAt(hasTypename ? -3 : -2));
-        const Token * const aliasToken1 = startToken;
 
         // Get start token for alias
         startToken = startToken->tokAt(hasTypename ? -6 : -5);
@@ -1120,9 +1176,17 @@ void TemplateSimplifier::simplifyTemplateAliases()
         } else if (!Token::Match(startToken, "[;{}] template <"))
             continue;
 
+        std::list<TokenAndName>::iterator it5 = std::find_if(mTemplateDeclarations.begin(),
+                                                mTemplateDeclarations.end(),
+                                                FindToken(!startToken ? mTokenList.front() : startToken->next()));
+        if (it5 == mTemplateDeclarations.end())
+            continue;
+
+        TokenAndName &aliasDeclaration = *it5;
+
         // alias parameters..
         std::vector<const Token *> aliasParameters;
-        getTemplateParametersInDeclaration(startToken ? startToken->tokAt(3) : mTokenList.front()->tokAt(2), aliasParameters);
+        getTemplateParametersInDeclaration(aliasDeclaration.token->tokAt(2), aliasParameters);
         std::map<std::string, unsigned int> aliasParameterNames;
         for (unsigned int argnr = 0; argnr < aliasParameters.size(); ++argnr)
             aliasParameterNames[aliasParameters[argnr]->str()] = argnr;
@@ -1131,8 +1195,13 @@ void TemplateSimplifier::simplifyTemplateAliases()
         const Token *endToken = nullptr;
         for (it2 = it1; it2 != mTemplateInstantiations.end(); ++it2) {
             TokenAndName &aliasUsage = *it2;
-            if (!aliasUsage.token || aliasUsage.name != aliasName)
+            if (!aliasUsage.token || aliasUsage.fullName != aliasDeclaration.fullName)
                 continue;
+
+            // don't recurse
+            if (aliasDeclaration.isAliasToken(aliasUsage.token))
+                continue;
+
             std::vector<std::pair<Token *, Token *>> args;
             Token *tok2 = aliasUsage.token->tokAt(2);
             while (tok2) {
@@ -1157,18 +1226,27 @@ void TemplateSimplifier::simplifyTemplateAliases()
 
             // Replace template alias code..
             aliasUsage.name = templateAlias.name;
-            if (aliasUsage.name.find(' ') == std::string::npos) {
+            aliasUsage.fullName = templateAlias.fullName;
-                const Token *temp = aliasToken1;
+            aliasUsage.scope = templateAlias.scope;
-                while (temp && temp != templateAlias.token) {
+            const Token *temp = aliasDeclaration.aliasStartToken();
-                    aliasUsage.token->insertToken(temp->str(), "", true);
+            if (temp->str() == "typename")
-                    temp = temp->next();
+                temp = temp->next();
+            std::stack<Token *> links;
+            while (temp && temp != templateAlias.token) {
+                aliasUsage.token->insertToken(temp->str(), "", true);
+
+                Token * previous = aliasUsage.token->previous();
+                if (Token::Match(previous, "(|[|{"))
+                    links.push(previous);
+                else if (!links.empty() && Token::Match(previous, ")|]|}")) {
+                    Token::createMutualLinks(links.top(), previous);
+                    links.pop();
                 }
-                aliasUsage.token->str(templateAlias.token->str());
+
-            } else {
+                temp = temp->next();
-                tok2 = TokenList::copyTokens(aliasUsage.token, aliasToken1, templateAlias.token, true);
-                deleteToken(aliasUsage.token);
-                aliasUsage.token = tok2;
             }
+            aliasUsage.token->str(templateAlias.token->str());
+
             tok2 = aliasUsage.token->next(); // the '<'
             const Token * const endToken1 = templateAlias.token->next()->findClosingBracket();
             const Token * const endToken2 = TokenList::copyTokens(tok2, templateAlias.token->tokAt(2), endToken1->previous(), false)->next();
@@ -1205,7 +1283,7 @@ void TemplateSimplifier::simplifyTemplateAliases()
         }
         if (endToken) {
             // Remove all template instantiations in template alias
-            for (const Token *tok = startToken ? startToken : mTokenList.front(); tok != endToken; tok = tok->next()) {
+            for (const Token *tok = aliasDeclaration.paramEnd->tokAt(4); tok != endToken; tok = tok->next()) {
                 if (!Token::Match(tok, "%name% <"))
                     continue;
                 std::list<TokenAndName>::iterator it = std::find_if(mTemplateInstantiations.begin(),
@@ -1220,21 +1298,15 @@ void TemplateSimplifier::simplifyTemplateAliases()
                 mTemplateInstantiations.erase(it,next);
             }
 
-            // find declaration
-            const std::list<TokenAndName>::iterator it4 = std::find_if(mTemplateDeclarations.begin(),
-                    mTemplateDeclarations.end(),
-                    FindToken(startToken ? startToken->next() : mTokenList.front()));
-
             if (startToken)
-                eraseTokens(startToken, endToken);
+                eraseTokens(startToken, endToken->next() ? endToken->next() : endToken);
             else {
-                eraseTokens(mTokenList.front(), endToken);
+                eraseTokens(mTokenList.front(), endToken->next() ? endToken->next() : endToken);
                 deleteToken(mTokenList.front());
             }
 
             // remove declaration
-            if (it4 != mTemplateDeclarations.end())
+            mTemplateDeclarations.erase(it5);
-                mTemplateDeclarations.erase(it4);
         } else
             mTemplateInstantiations.erase(it3);
     }
@@ -1256,6 +1328,8 @@ bool TemplateSimplifier::instantiateMatch(const Token *instance, const std::size
                 ++indentlevel;
             if (indentlevel > 0 && tok->str() == ">")
                 --indentlevel;
+            if (indentlevel > 0 && Token::Match(tok, "{|(|["))
+                tok = tok->link();
         }
         if (!tok || !Token::Match(tok->next(), patternAfter))
             return false;
@@ -1483,7 +1557,8 @@ void TemplateSimplifier::expandTemplate(
             while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != start->str())
                 ++itype;
 
-            if (itype < typeParametersInDeclaration.size()) {
+            if (itype < typeParametersInDeclaration.size() &&
+                (!isVariable || !Token::Match(typeParametersInDeclaration[itype]->previous(), "<|, %type% >|,"))) {
                 typeindentlevel = 0;
                 for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token;
                      typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>"));
@@ -1680,6 +1755,7 @@ void TemplateSimplifier::expandTemplate(
                         // replace type with given type..
                         if (itype < typeParametersInDeclaration.size()) {
                             unsigned int typeindentlevel = 0;
+                            std::stack<Token *> brackets1; // holds "(" and "{" tokens
                             for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token;
                                  typetok && (typeindentlevel>0 || !Token::Match(typetok, ",|>"));
                                  typetok = typetok->next()) {
@@ -1691,7 +1767,23 @@ void TemplateSimplifier::expandTemplate(
                                     else if (typeindentlevel > 0 && typetok->str() == ">")
                                         --typeindentlevel;
                                     mTokenList.addtoken(typetok, tok5->linenr(), tok5->fileIndex());
-                                    mTokenList.back()->isTemplateArg(true);
+                                    Token *back = mTokenList.back();
+                                    if (back->str() == "{") {
+                                        brackets1.push(back);
+                                    } else if (back->str() == "(") {
+                                        brackets1.push(back);
+                                    } else if (back->str() == "}") {
+                                        assert(brackets1.empty() == false);
+                                        assert(brackets1.top()->str() == "{");
+                                        Token::createMutualLinks(brackets1.top(), back);
+                                        brackets1.pop();
+                                    } else if (back->str() == ")") {
+                                        assert(brackets1.empty() == false);
+                                        assert(brackets1.top()->str() == "(");
+                                        Token::createMutualLinks(brackets1.top(), back);
+                                        brackets1.pop();
+                                    }
+                                    back->isTemplateArg(true);
                                     added = true;
                                     break;
                                 }
@@ -1740,6 +1832,7 @@ void TemplateSimplifier::expandTemplate(
                 // replace type with given type..
                 if (itype < typeParametersInDeclaration.size()) {
                     unsigned int typeindentlevel = 0;
+                    std::stack<Token *> brackets1; // holds "(" and "{" tokens
                     for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token;
                          typetok && (typeindentlevel>0 || !Token::Match(typetok, ",|>"));
                          typetok = typetok->next()) {
@@ -1753,7 +1846,23 @@ void TemplateSimplifier::expandTemplate(
                             --typeindentlevel;
                         if (copy) {
                             mTokenList.addtoken(typetok, tok3->linenr(), tok3->fileIndex());
-                            mTokenList.back()->isTemplateArg(true);
+                            Token *back = mTokenList.back();
+                            if (back->str() == "{") {
+                                brackets1.push(back);
+                            } else if (back->str() == "(") {
+                                brackets1.push(back);
+                            } else if (back->str() == "}") {
+                                assert(brackets1.empty() == false);
+                                assert(brackets1.top()->str() == "{");
+                                Token::createMutualLinks(brackets1.top(), back);
+                                brackets1.pop();
+                            } else if (back->str() == ")") {
+                                assert(brackets1.empty() == false);
+                                assert(brackets1.top()->str() == "(");
+                                Token::createMutualLinks(brackets1.top(), back);
+                                brackets1.pop();
+                            }
+                            back->isTemplateArg(true);
                         }
                     }
                     continue;
@@ -2405,7 +2514,7 @@ const Token * TemplateSimplifier::getTemplateParametersInDeclaration(
             const Token *closing = tok->next()->findClosingBracket();
             if (closing)
                 tok = closing->next();
-        } else if (tok->link())
+        } else if (tok->link() && Token::Match(tok, "{|(|["))
             tok = tok->link();
         else if (Token::Match(tok, "%name% ,|>|="))
             typeParametersInDeclaration.push_back(tok);
@@ -2467,9 +2576,8 @@ std::string TemplateSimplifier::getNewName(
     unsigned int indentlevel = 0;
     const Token * endToken = tok2->next()->findClosingBracket();
     for (Token *tok3 = tok2->tokAt(2); tok3 != endToken && (indentlevel > 0 || tok3->str() != ">"); tok3 = tok3->next()) {
-        // #2648 - unhandled parentheses => bail out
         // #2721 - unhandled [ => bail out
-        if (Token::Match(tok3, "(|[")) {
+        if (tok3->str() == "[") {
             typeForNewName.clear();
             break;
         }
@@ -2964,6 +3072,15 @@ void TemplateSimplifier::printOut(const TokenAndName &tokenAndName, const std::s
             }
             std::cout << end->str() << std::endl;
         }
+    } else if (tokenAndName.isAlias() && tokenAndName.paramEnd) {
+        if (tokenAndName.aliasStartToken()) {
+            std::cout << indent << "aliasStartToken: \"" << tokenAndName.aliasStartToken()->str() << "\" "
+                      << mTokenList.fileLine(tokenAndName.aliasStartToken()) << std::endl;
+        }
+        if (tokenAndName.aliasEndToken()) {
+            std::cout << indent << "aliasEndToken: \"" << tokenAndName.aliasEndToken()->str() << "\" "
+                      << mTokenList.fileLine(tokenAndName.aliasEndToken()) << std::endl;
+        }
     }
 }
 
@@ -3131,7 +3248,7 @@ void TemplateSimplifier::simplifyTemplates(
             codeWithTemplates = hasTemplates;
 
         // Make sure there is something to simplify.
-        if (mTemplateDeclarations.empty())
+        if (mTemplateDeclarations.empty() && mTemplateForwardDeclarations.empty())
             return;
 
         // Copy default argument values from forward declaration to declaration
@@ -3157,9 +3274,15 @@ void TemplateSimplifier::simplifyTemplates(
         std::set<std::string> expandedtemplates;
 
         for (std::list<TokenAndName>::reverse_iterator iter1 = mTemplateDeclarations.rbegin(); iter1 != mTemplateDeclarations.rend(); ++iter1) {
+            if (iter1->isAlias())
+                continue;
+
             // get specializations..
             std::list<const Token *> specializations;
             for (std::list<TokenAndName>::const_iterator iter2 = mTemplateDeclarations.begin(); iter2 != mTemplateDeclarations.end(); ++iter2) {
+                if (iter2->isAlias())
+                    continue;
+
                 if (iter1->fullName == iter2->fullName)
                     specializations.push_back(iter2->nameToken);
             }

lib/templatesimplifier.h

@@ -183,6 +183,31 @@ public:
         void setFlag(unsigned int flag, bool state) {
             flags = state ? flags | flag : flags & ~flag;
         }
+
+        /**
+         * Get alias start token.
+         * template < ... > using X = foo < ... >;
+         *                            ^
+         * @return alias start token
+         */
+        const Token * aliasStartToken() const;
+
+        /**
+         * Get alias end token.
+         * template < ... > using X = foo < ... >;
+         *                                       ^
+         * @return alias end token
+         */
+        const Token * aliasEndToken() const;
+
+        /**
+         * Is token an alias token?
+         * template < ... > using X = foo < ... >;
+         *                                   ^
+         * @param tok token to check
+         * @return true if alias token, false if not
+         */
+        bool isAliasToken(const Token *tok) const;
     };
 
     /**

lib/tokenize.cpp

@@ -3847,7 +3847,7 @@ void Tokenizer::createLinks2()
 
             while (!type.empty() && type.top()->str() == "<") {
                 const Token* end = type.top()->findClosingBracket();
-                if (Token::Match(end, "> %comp%|;|."))
+                if (Token::Match(end, "> %comp%|;|.|="))
                     break;
                 // Variable declaration
                 if (Token::Match(end, "> %var% ;") && (type.top()->tokAt(-2) == nullptr || Token::Match(type.top()->tokAt(-2), ";|}|{")))
@@ -3871,7 +3871,7 @@ void Tokenizer::createLinks2()
                 if (token->str() == ">>")
                     continue;
                 if (token->next() &&
-                    !Token::Match(token->next(), "%name%|%comp%|&|&&|*|::|,|(|)|{|}|;|[|:|.") &&
+                    !Token::Match(token->next(), "%name%|%comp%|&|&&|*|::|,|(|)|{|}|;|[|:|.|=") &&
                     !Token::simpleMatch(token->next(), ". . .") &&
                     !Token::Match(token->next(), "&& %name% ="))
                     continue;

test/testsimplifytemplate.cpp

@@ -147,6 +147,8 @@ private:
         TEST_CASE(template107); // #8663
         TEST_CASE(template108); // #9109
         TEST_CASE(template109); // #9144
+        TEST_CASE(template110);
+        TEST_CASE(template111); // crash
         TEST_CASE(template_specialization_1);  // #7868 - template specialization template <typename T> struct S<C<T>> {..};
         TEST_CASE(template_specialization_2);  // #7868 - template specialization template <typename T> struct S<C<T>> {..};
         TEST_CASE(template_enum);  // #6299 Syntax error in complex enum declaration (including template)
@@ -195,6 +197,10 @@ private:
         TEST_CASE(simplifyTemplateArgs);
 
         TEST_CASE(template_variadic_1); // #9144
+
+        TEST_CASE(template_variable_1);
+        TEST_CASE(template_variable_2);
+        TEST_CASE(template_variable_3);
     }
 
     std::string tok(const char code[], bool debugwarnings = false, Settings::PlatformType type = Settings::Native) {
@@ -1356,12 +1362,19 @@ private:
                             "\n"
                             "void j() { h<int>(); }";
         const char exp[] = "struct S<int> ; "
-                           "template < typename T > void f ( ) { } " // <- TODO: This template is not expanded
+                           "void f<S<int>::type(0)> ( ) ; "
                            "void h<int> ( ) ; "
                            "void j ( ) { h<int> ( ) ; } "
-                           "void h<int> ( ) { f < S<int> :: type ( 0 ) > ( ) ; } "
+                           "void h<int> ( ) { f<S<int>::type(0)> ( ) ; } "
-                           "struct S<int> { } ;";
+                           "struct S<int> { } ; "
-        ASSERT_EQUALS(exp, tok(code));
+                           "void f<S<int>::type(0)> ( ) { }";
+        const char act[] = "template < typename T > struct S { } ; "
+                           "void f<S<int>::type(0)> ( ) ; "
+                           "void h<int> ( ) ; "
+                           "void j ( ) { h<int> ( ) ; } "
+                           "void h<int> ( ) { f<S<int>::type(0)> ( ) ; } "
+                           "void f<S<int>::type(0)> ( ) { }";
+        TODO_ASSERT_EQUALS(exp, act, tok(code));
     }
 
     void template61() { // hang in daca, code extracted from kodi
@@ -2254,8 +2267,7 @@ private:
                             "        delete a;\n"
                             "    });\n"
                             "}";
-        const char exp[] = "; "
+        const char exp[] = "class A { } ; "
-                           "class A { } ; "
                            "static void func ( ) { "
                            "std :: unique_ptr < A , std :: function < void ( A * ) > > tmp ( new A ( ) , [ ] ( A * a ) { "
                            "delete a ; "
@@ -2537,25 +2549,32 @@ private:
             const char code[] = "namespace a {\n"
                                 "template <typename b, bool = __is_empty(b) && __is_final(b)> struct c;\n"
                                 "}\n"
-                                "namespace boost {\n"
-                                "using a::c;\n"
-                                "}\n"
-                                "namespace d = boost;\n"
-                                "using d::c;\n"
                                 "template <typename...> struct e {};\n"
                                 "static_assert(sizeof(e<>) == sizeof(e<c<int>, c<int>, int>), \"\");\n";
             const char exp[] = "namespace a { "
                                "template < typename b , bool > struct c ; "
                                "} "
-                               "namespace boost { "
-                               "using a :: c ; "
-                               "} "
-                               "using boost :: c ; "
                                "struct e<> ; "
-                               "struct e<c<int,std::is_empty<int>{}&&std::is_final<int>{}>,c<int,std::is_empty<int>{}&&std::is_final<int>{}>,int> ; "
+                               "struct e<c<int>,c<int>,int> ; "
-                               "static_assert ( sizeof ( e<> ) == sizeof ( e<c<int,std::is_empty<int>{}&&std::is_final<int>{}>,c<int,std::is_empty<int>{}&&std::is_final<int>{}>,int> ) , \"\" ) ; "
+                               "static_assert ( sizeof ( e<> ) == sizeof ( e<c<int>,c<int>,int> ) , \"\" ) ; "
                                "struct e<> { } ; "
-                               "struct e<c<int,std::is_empty<int>{}&&std::is_final<int>{}>,c<int,std::is_empty<int>{}&&std::is_final<int>{}>,int> { } ;";
+                               "struct e<c<int>,c<int>,int> { } ;";
+            ASSERT_EQUALS(exp, tok(code));
+        }
+        {
+            const char code[] = "namespace a {\n"
+                                "template <typename b, bool = __is_empty(b) && __is_final(b)> struct c;\n"
+                                "}\n"
+                                "template <typename...> struct e {};\n"
+                                "static_assert(sizeof(e<>) == sizeof(e<a::c<int>, a::c<int>, int>), \"\");\n";
+            const char exp[] = "namespace a { "
+                               "template < typename b , bool > struct c ; "
+                               "} "
+                               "struct e<> ; "
+                               "struct e<a::c<int,std::is_empty<int>{}&&std::is_final<int>{}>,a::c<int,std::is_empty<int>{}&&std::is_final<int>{}>,int> ; "
+                               "static_assert ( sizeof ( e<> ) == sizeof ( e<a::c<int,std::is_empty<int>{}&&std::is_final<int>{}>,a::c<int,std::is_empty<int>{}&&std::is_final<int>{}>,int> ) , \"\" ) ; "
+                               "struct e<> { } ; "
+                               "struct e<a::c<int,std::is_empty<int>{}&&std::is_final<int>{}>,a::c<int,std::is_empty<int>{}&&std::is_final<int>{}>,int> { } ;";
             ASSERT_EQUALS(exp, tok(code));
         }
         {
@@ -2578,6 +2597,36 @@ private:
                                "struct c<int,std::is_empty<int>{}&&std::is_final<int>{}> { } ;";
             ASSERT_EQUALS(exp, tok(code));
         }
+        {
+            const char code[] = "template <typename b, bool = unknown1(b) && unknown2(b)> struct c{};\n"
+                                "c<int> cc;\n";
+            const char exp[] = "struct c<int,unknown1(int)&&unknown2(int)> ; "
+                               "c<int,unknown1(int)&&unknown2(int)> cc ; "
+                               "struct c<int,unknown1(int)&&unknown2(int)> { } ;";
+            ASSERT_EQUALS(exp, tok(code));
+        }
+    }
+
+    void template110() {
+        const char code[] = "template<typename T> using A = int;\n"
+                            "template<typename T> using A<T*> = char;\n"
+                            "template<> using A<char> = char;\n"
+                            "template using A<char> = char;\n"
+                            "using A<char> = char;";
+        const char exp[] = "template < typename T > using A = int ; "
+                           "template < typename T > using A < T * > = char ; "
+                           "template < > using A < char > = char ; "
+                           "template using A < char > = char ; "
+                           "using A < char > = char ;";
+        ASSERT_EQUALS(exp, tok(code));
+    }
+
+    void template111() { // crash
+        const char code[] = "template<typename T, typename U> struct pair;\n"
+                            "template<typename T> using cell = pair<T*, cell<T>*>;";
+        const char exp[] = "template < typename T , typename U > struct pair ; "
+                           "template < typename T > using cell = pair < T * , cell < T > * > ;";
+        ASSERT_EQUALS(exp, tok(code));
     }
 
     void template_specialization_1() {  // #7868 - template specialization template <typename T> struct S<C<T>> {..};
@@ -2648,8 +2697,6 @@ private:
                                 "    A<int,2> a1;\n"
                                 "    A<int> a2;\n"
                                 "}\n";
-
-            // The expected result..
             const char expected[] = "class A<int,2> ; "
                                     "class A<int,3> ; "
                                     "void f ( ) "
@@ -2674,7 +2721,6 @@ private:
                                 "    A<int,3> a2;\n"
                                 "}\n";
 
-            // The expected result..
             const char expected[] = "class A<int,3,2> ; "
                                     "void f ( ) "
                                     "{"
@@ -2695,29 +2741,18 @@ private:
                                 "    A<int,(int)2> a1;\n"
                                 "    A<int> a2;\n"
                                 "}\n";
-
+            const char expected[] = "class A<int,(int)2> ; "
-            const char wanted[] = "template < class T , int n >"
+                                    "class A<int,3> ; "
-                                  " class A"
+                                    "void f ( ) "
-                                  " { T ar [ n ] ; } ;"
+                                    "{ "
-                                  " void f ( )"
+                                    "A<int,(int)2> a1 ; "
-                                  " {"
+                                    "A<int,3> a2 ; "
-                                  " A<int,(int)2> a1 ;"
+                                    "} "
-                                  " A<int,3> a2 ;"
+                                    "class A<int,(int)2> "
-                                  " }"
+                                    "{ int ar [ ( int ) 2 ] ; } ; "
-                                  " class A<int,2>"
+                                    "class A<int,3> "
-                                  " { int ar [ 2 ] ; }"
+                                    "{ int ar [ 3 ] ; } ;";
-                                  " class A<int,3>"
+            ASSERT_EQUALS(expected, tok(code));
-                                  " { int ar [ 3 ] ; }";
-
-            const char current[] = "class A<int,3> ; "
-                                   "void f ( ) "
-                                   "{ "
-                                   "A < int , ( int ) 2 > a1 ; "
-                                   "A<int,3> a2 ; "
-                                   "} "
-                                   "class A<int,3> "
-                                   "{ int ar [ 3 ] ; } ;";
-            TODO_ASSERT_EQUALS(wanted, current, tok(code));
         }
         {
             const char code[] = "class A { }; "
@@ -3560,7 +3595,6 @@ private:
                             "Bar<int> b;\n";
 
         const char expected[] = "namespace A { struct Foo<int,3> ; } "
-                                "; "
                                 "A :: Foo<int,3> b ; "
                                 "struct A :: Foo<int,3> { } ;";
 
@@ -3582,8 +3616,7 @@ private:
                             "using IntrusivePtr = boost::intrusive_ptr<T>;\n"
                             "template <class T> class Vertex { };\n"
                             "IntrusivePtr<Vertex<int>> p;";
-        const char expected[] = "; "
+        const char expected[] = "class Vertex<int> ; "
-                                "class Vertex<int> ; "
                                 "boost :: intrusive_ptr < Vertex<int> > p ; "
                                 "class Vertex<int> { } ;";
         ASSERT_EQUALS(expected, tok(code));
@@ -3758,6 +3791,106 @@ private:
         ASSERT_EQUALS(expected, tok(code));
     }
 
+    void template_variable_1() {
+        {
+            const char code[] = "template <int N> const int foo = N*N;\n"
+                                "int x = foo<7>;";
+            const char expected[] = "const int foo<7> = 49 ; "
+                                    "int x ; x = foo<7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template <int> const int foo = 7;\n"
+                                "int x = foo<7>;";
+            const char expected[] = "const int foo<7> = 7 ; "
+                                    "int x ; x = foo<7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template <int N = 7> const int foo = N*N;\n"
+                                "int x = foo<7>;";
+            const char expected[] = "const int foo<7> = 49 ; "
+                                    "int x ; x = foo<7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template <int N = 7> const int foo = N*N;\n"
+                                "int x = foo<>;";
+            const char expected[] = "const int foo<7> = 49 ; "
+                                    "int x ; x = foo<7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+    }
+
+    void template_variable_2() {
+        {
+            const char code[] = "template<class T> constexpr T pi = T(3.1415926535897932385L);\n"
+                                "float x = pi<float>;";
+            const char expected[] = "const float pi<float> = float ( 3.1415926535897932385L ) ; "
+                                    "float x ; x = pi<float> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class> constexpr float pi = float(3.1415926535897932385L);\n"
+                                "float x = pi<float>;";
+            const char expected[] = "const float pi<float> = float ( 3.1415926535897932385L ) ; "
+                                    "float x ; x = pi<float> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class T = float> constexpr T pi = T(3.1415926535897932385L);\n"
+                                "float x = pi<float>;";
+            const char expected[] = "const float pi<float> = float ( 3.1415926535897932385L ) ; "
+                                    "float x ; x = pi<float> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class T = float> constexpr T pi = T(3.1415926535897932385L);\n"
+                                "float x = pi<>;";
+            const char expected[] = "const float pi<float> = float ( 3.1415926535897932385L ) ; "
+                                    "float x ; x = pi<float> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+    }
+
+    void template_variable_3() {
+        {
+            const char code[] = "template<class T, int N> constexpr T foo = T(N*N);\n"
+                                "float x = foo<float,7>;";
+            const char expected[] = "const float foo<float,7> = float ( 49 ) ; "
+                                    "float x ; x = foo<float,7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class,int> constexpr float foo = float(7);\n"
+                                "float x = foo<float,7>;";
+            const char expected[] = "const float foo<float,7> = float ( 7 ) ; "
+                                    "float x ; x = foo<float,7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class T = float, int N = 7> constexpr T foo = T(7);\n"
+                                "double x = foo<double, 14>;";
+            const char expected[] = "const double foo<double,14> = double ( 7 ) ; "
+                                    "double x ; x = foo<double,14> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class T = float, int N = 7> constexpr T foo = T(7);\n"
+                                "float x = foo<>;";
+            const char expected[] = "const float foo<float,7> = float ( 7 ) ; "
+                                    "float x ; x = foo<float,7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+        {
+            const char code[] = "template<class T = float, int N = 7> constexpr T foo = T(7);\n"
+                                "double x = foo<double>;";
+            const char expected[] = "const double foo<double,7> = double ( 7 ) ; "
+                                    "double x ; x = foo<double,7> ;";
+            ASSERT_EQUALS(expected, tok(code));
+        }
+    }
+
 };
 
 REGISTER_TEST(TestSimplifyTemplate)

test/testtokenize.cpp

@@ -5229,12 +5229,12 @@ private:
                            "struct S\n"
                            "{};\n"
                            "S<int> s;\n";
-        TODO_ASSERT_EQUALS("S<int,(int)0> s ; struct S<int,(int)0> { } ;",   // wanted result
+        ASSERT_EQUALS("struct S<int,(int)0> ;\n"
-                           "template < class T , T t >\n"
+                      "\n"
-                           "struct S\n"
+                      "\n"
-                           "{ } ;\n"
+                      "S<int,(int)0> s ; struct S<int,(int)0>\n"
-                           "S < int , ( int ) 0 > s ;",     // current result
+                      "{ } ;",
-                           tokenizeAndStringify(code));
+                      tokenizeAndStringify(code));
     }
 
     void cpp0xtemplate4() { // #6181, #6354, #6414

test/testvarid.cpp

@@ -2192,8 +2192,7 @@ private:
     void varid_templateUsing() { // #5781 #7273
         const char code[] = "template<class T> using X = Y<T,4>;\n"
                             "X<int> x;";
-        ASSERT_EQUALS("1: ;\n"
+        ASSERT_EQUALS("2: Y < int , 4 > x@1 ;\n",
-                      "2: Y < int , 4 > x@1 ;\n",
                       tokenize(code));
     }