template simplifier: only constant fold template instantiation arguments (#1721)

* template simplifier: only constant fold template instantiation arguments

* Fix travis build.

lib/templatesimplifier.cpp

@@ -647,6 +647,12 @@ bool TemplateSimplifier::getTemplateDeclarations()
     return codeWithTemplates;
 }
 
+void TemplateSimplifier::addInstantiation(Token *token, const std::string &scope)
+{
+    simplifyTemplateArgs(token->tokAt(2), token->next()->findClosingBracket());
+
+    mTemplateInstantiations.emplace_back(token, scope);
+}
 
 void TemplateSimplifier::getTemplateInstantiations()
 {
@@ -795,7 +801,7 @@ void TemplateSimplifier::getTemplateInstantiations()
             for (; tok2 && tok2 != tok; tok2 = tok2->previous()) {
                 if (Token::Match(tok2, ", %name% <") &&
                     templateParameters(tok2->tokAt(2))) {
-                    mTemplateInstantiations.emplace_back(tok2->next(), getScopeName(scopeList));
+                    addInstantiation(tok2->next(), getScopeName(scopeList));
                 } else if (Token::Match(tok2->next(), "class|struct"))
                     const_cast<Token *>(tok2)->deleteNext();
             }
@@ -809,7 +815,7 @@ void TemplateSimplifier::getTemplateInstantiations()
                     const std::list<TokenAndName>::const_iterator it = std::find_if(mTemplateDeclarations.begin(), mTemplateDeclarations.end(), FindFullName(fullName));
                     if (it != mTemplateDeclarations.end()) {
                         // full name matches
-                        mTemplateInstantiations.emplace_back(tok, it->scope);
+                        addInstantiation(tok, it->scope);
                         break;
                     } else {
                         // full name doesn't match so try with using namespaces if available
@@ -829,7 +835,7 @@ void TemplateSimplifier::getTemplateInstantiations()
                                 }
                                 qualificationTok->insertToken(nameSpace.substr(offset), "", true);
                                 qualificationTok->insertToken("::", "", true);
-                                mTemplateInstantiations.emplace_back(tok, it1->scope);
+                                addInstantiation(tok, it1->scope);
                                 found = true;
                                 break;
                             }
@@ -839,9 +845,9 @@ void TemplateSimplifier::getTemplateInstantiations()
 
                         if (scopeName.empty()) {
                             if (!qualification.empty())
-                                mTemplateInstantiations.emplace_back(tok, qualification);
+                                addInstantiation(tok, qualification);
                             else
-                                mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList));
+                                addInstantiation(tok, getScopeName(scopeList));
                             break;
                         }
                         const std::string::size_type pos = scopeName.rfind(" :: ");
@@ -1092,7 +1098,7 @@ void TemplateSimplifier::simplifyTemplateAliases()
                                 mTemplateInstantiations.end(),
                                 FindToken(tok1));
                         if (it != mTemplateInstantiations.end())
-                            mTemplateInstantiations.emplace_back(tok2, it->scope);
+                            addInstantiation(tok2, it->scope);
                     }
                     continue;
                 }
@@ -1317,6 +1323,12 @@ void TemplateSimplifier::expandTemplate(
     const bool isFunction = templateDeclaration.isFunction();
     const bool isSpecialization = templateDeclaration.isSpecialization();
     const bool isVariable = templateDeclaration.isVariable();
+    struct newInstantiation {
+        newInstantiation(Token *t, const std::string &s) : token(t), scope(s) { }
+        Token *token;
+        std::string scope;
+    };
+    std::vector<newInstantiation> newInstantiations;
 
     // add forward declarations
     if (copy && isClass) {
@@ -1411,7 +1423,7 @@ void TemplateSimplifier::expandTemplate(
                             start = start->next()->findClosingBracket();
                     } else {
                         dst->insertToken(start->str(), "", true);
-                        mTemplateInstantiations.emplace_back(dst->previous(), templateDeclaration.scope);
+                        newInstantiations.emplace_back(dst->previous(), templateDeclaration.scope);
                     }
                 } else {
                     // check if type is a template
@@ -1464,7 +1476,7 @@ void TemplateSimplifier::expandTemplate(
         dst->insertToken(";", "", true);
 
         if (isVariable)
-            simplifyCalculations(dstStart, dst);
+            simplifyTemplateArgs(dstStart, dst);
     }
 
     if (copy && (isClass || isFunction)) {
@@ -1688,9 +1700,9 @@ void TemplateSimplifier::expandTemplate(
                         scope = prev->str() + " :: " + scope;
                 }
                 if (copy)
-                    mTemplateInstantiations.emplace_back(mTokenList.back(), scope);
+                    newInstantiations.emplace_back(mTokenList.back(), scope);
                 else if (!inTemplateDefinition)
-                    mTemplateInstantiations.emplace_back(tok3, scope);
+                    newInstantiations.emplace_back(tok3, scope);
             }
 
             // link() newly tokens manually
@@ -1727,6 +1739,10 @@ void TemplateSimplifier::expandTemplate(
 
         assert(brackets.empty());
     }
+
+    // add new instantiations
+    for (const auto & inst : newInstantiations)
+        addInstantiation(inst.token, inst.scope);
 }
 
 static bool isLowerThanLogicalAnd(const Token *lower)
@@ -1854,6 +1870,153 @@ bool TemplateSimplifier::simplifyNumericCalculations(Token *tok)
     return ret;
 }
 
+static Token *skipTernaryOp(Token *tok, Token *backToken)
+{
+    unsigned int colonLevel = 1;
+    while (nullptr != (tok = tok->next())) {
+        if (tok->str() == "?") {
+            ++colonLevel;
+        } else if (tok->str() == ":") {
+            --colonLevel;
+            if (colonLevel == 0) {
+                tok = tok->next();
+                break;
+            }
+        }
+        if (tok->link() && tok->str() == "(")
+            tok = tok->link();
+        else if (Token::Match(tok->next(), "[{};)]") || tok->next() == backToken)
+            break;
+    }
+    if (colonLevel > 0) // Ticket #5214: Make sure the ':' matches the proper '?'
+        return nullptr;
+    return tok;
+}
+
+void TemplateSimplifier::simplifyTemplateArgs(Token *start, Token *end)
+{
+    bool again = true;
+
+    while (again) {
+        again = false;
+
+        for (Token *tok = start; tok && tok != end; tok = tok->next()) {
+            if (tok->str() == "sizeof") {
+                // sizeof('x')
+                if (Token::Match(tok->next(), "( %char% )")) {
+                    tok->deleteNext();
+                    tok->deleteThis();
+                    tok->deleteNext();
+                    std::ostringstream sz;
+                    sz << 1;
+                    tok->str(sz.str());
+                    again = true;
+                }
+
+                // sizeof ("text")
+                else if (Token::Match(tok->next(), "( %str% )")) {
+                    tok->deleteNext();
+                    tok->deleteThis();
+                    tok->deleteNext();
+                    std::ostringstream ostr;
+                    ostr << (Token::getStrLength(tok) + 1);
+                    tok->str(ostr.str());
+                    again = true;
+                }
+
+                else if (Token::Match(tok->next(), "( %type% * )")) {
+                    tok->str(MathLib::toString(mTokenizer->sizeOfType(tok->tokAt(3))));
+                    tok->deleteNext(4);
+                    again = true;
+                } else if (Token::simpleMatch(tok->next(), "( * )")) {
+                    tok->str(MathLib::toString(mTokenizer->sizeOfType(tok->tokAt(2))));
+                    tok->deleteNext(3);
+                    again = true;
+                } else if (Token::Match(tok->next(), "( %type% )")) {
+                    const unsigned int size = mTokenizer->sizeOfType(tok->tokAt(2));
+                    if (size > 0) {
+                        tok->str(MathLib::toString(size));
+                        tok->deleteNext(3);
+                        again = true;
+                    }
+                } else if (tok->strAt(1) == "(") {
+                    tok = tok->linkAt(1);
+                }
+            }
+        }
+
+        if (simplifyCalculations(start, end))
+            again = true;
+
+        for (Token *tok = start; tok && tok != end; tok = tok->next()) {
+            if (tok->str() == "?" && tok->previous()->isNumber()) {
+                const int offset = (tok->previous()->str() == ")") ? 2 : 1;
+
+                // Find the token ":" then go to the next token
+                Token *colon = skipTernaryOp(tok, end);
+                if (!colon || colon->previous()->str() != ":" || !colon->next())
+                    continue;
+
+                //handle the GNU extension: "x ? : y" <-> "x ? x : y"
+                if (colon->previous() == tok->next())
+                    tok->insertToken(tok->strAt(-offset));
+
+                // go back before the condition, if possible
+                tok = tok->tokAt(-2);
+                if (offset == 2) {
+                    // go further back before the "("
+                    tok = tok->tokAt(-2);
+                    //simplify the parentheses
+                    tok->deleteNext();
+                    tok->next()->deleteNext();
+                }
+
+                if (Token::Match(tok->next(), "false|0")) {
+                    // Use code after colon, remove code before it.
+                    Token::eraseTokens(tok, colon);
+
+                    tok = tok->next();
+                    again = true;
+                }
+
+                // The condition is true. Delete the operator after the ":"..
+                else {
+                    // delete the condition token and the "?"
+                    tok->deleteNext(2);
+
+                    unsigned int ternaryOplevel = 0;
+                    for (const Token *endTok = colon; endTok; endTok = endTok->next()) {
+                        if (Token::Match(endTok, "(|[|{")) {
+                            endTok = endTok->link();
+                        }
+
+                        else if (endTok->str() == "?")
+                            ++ternaryOplevel;
+                        else if (Token::Match(endTok, ")|}|]|;|,|:|>")) {
+                            if (endTok->str() == ":" && ternaryOplevel)
+                                --ternaryOplevel;
+                            else if (endTok->str() == ">" && !end)
+                                ;
+                            else {
+                                Token::eraseTokens(colon->tokAt(-2), endTok);
+                                again = true;
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        for (Token *tok = start; tok && tok != end; tok = tok->next()) {
+            if (Token::Match(tok, "( %num% )") && !Token::Match(tok->previous(), "%name%")) {
+                tok->deleteThis();
+                tok->deleteNext();
+            }
+        }
+    }
+}
+
 // TODO: This is not the correct class for simplifyCalculations(), so it
 // should be moved away.
 bool TemplateSimplifier::simplifyCalculations(Token* frontToken, Token *backToken)
@@ -2164,7 +2327,6 @@ bool TemplateSimplifier::simplifyTemplateInstantiations(
     for (const TokenAndName &instantiation : mTemplateInstantiations) {
         if (numberOfTemplateInstantiations != mTemplateInstantiations.size()) {
             numberOfTemplateInstantiations = mTemplateInstantiations.size();
-            simplifyCalculations(instantiation.token);
             ++recursiveCount;
             if (recursiveCount > 100) {
                 // bail out..
@@ -2172,10 +2334,6 @@ bool TemplateSimplifier::simplifyTemplateInstantiations(
             }
         }
 
-        // simplifyCalculations can erase an instantiation
-        if (!instantiation.token)
-            continue;
-
         // already simplified
         if (!Token::Match(instantiation.token, "%name% <"))
             continue;
@@ -2256,8 +2414,6 @@ bool TemplateSimplifier::simplifyTemplateInstantiations(
     // process uninstantiated templates
     // TODO: remove the specialized check and handle all uninstantiated templates someday.
     if (!instantiated && specialized) {
-        simplifyCalculations(templateDeclaration.token);
-
         Token * tok2 = const_cast<Token *>(templateDeclaration.nameToken);
         if (mErrorLogger && !mTokenList.getFiles().empty())
             mErrorLogger->reportProgress(mTokenList.getFiles()[0], "TemplateSimplifier::simplifyTemplateInstantiations()", tok2->progressValue());

lib/templatesimplifier.h

@@ -234,6 +234,12 @@ public:
      */
     bool simplifyCalculations(Token* frontToken = nullptr, Token *backToken = nullptr);
 
+    /** Simplify template instantiation arguments.
+     * @param start first token of arguments
+     * @param end token following last argument token
+     */
+    void simplifyTemplateArgs(Token *start, Token *end);
+
 private:
     /**
      * Get template declarations
@@ -241,6 +247,12 @@ private:
      */
     bool getTemplateDeclarations();
 
+    /** Add template instantiation.
+     * @param token first token of instantiation
+     * @param scope scope of instantiation
+     */
+    void addInstantiation(Token *token, const std::string &scope);
+
     /**
      * Get template instantiations
      */

lib/tokenize.cpp

@@ -2316,14 +2316,6 @@ void Tokenizer::simplifyTemplates()
         return;
 
     for (Token *tok = list.front(); tok; tok = tok->next()) {
-        // simple fix for sizeof used as template parameter
-        // TODO: this is a bit hardcoded. make a bit more generic
-        if (Token::Match(tok, "%name% < sizeof ( %type% ) >") && tok->tokAt(4)->isStandardType()) {
-            Token * const tok3 = tok->next();
-            const unsigned int sizeOfResult = sizeOfType(tok3->tokAt(3));
-            tok3->deleteNext(4);
-            tok3->insertToken(MathLib::toString(sizeOfResult));
-        }
         // Ticket #6181: normalize C++11 template parameter list closing syntax
         if (tok->str() == "<" && mTemplateSimplifier->templateParameters(tok)) {
             Token *endTok = tok->findClosingBracket();

test/testsimplifytemplate.cpp

@@ -432,14 +432,12 @@ private:
                             "{\n"
                             "    A<12,12,11> a;\n"
                             "}\n";
-
-        // The expected result..
         const char expected[] = "class A<12,12,11> ; "
                                 "void f ( ) "
                                 "{"
                                 " A<12,12,11> a ; "
                                 "} "
-                                "class A<12,12,11> : public B < 12 , 12 , ( 0 ) ? ( ( 0 ) ? 1 : -1 ) : 0 > "
+                                "class A<12,12,11> : public B < 12 , 12 , 0 > "
                                 "{ } ;";
         ASSERT_EQUALS(expected, tok(code));
     }
@@ -745,18 +743,12 @@ private:
                             "{};\n"
                             "\n"
                             "bitset<1> z;";
-
+        const char expected[] = "struct B<4> ; "
-        const char actual[] = "template < int n > struct B { int a [ n ] ; } ; "
+                                "class bitset<1> ; "
-                              "class bitset<1> ; "
-                              "bitset<1> z ; "
-                              "class bitset<1> : B < ( sizeof ( int ) ? : 1 ) > { } ;";
-
-        const char expected[] = "class bitset<1> ; "
                                 "bitset<1> z ; "
-                                "class bitset<1> : B < 4 > { } ; "
+                                "class bitset<1> : B<4> { } ; "
-                                "struct B < 4 > { int a [ 4 ] ; } ;";
+                                "struct B<4> { int a [ 4 ] ; } ;";
-
+        ASSERT_EQUALS(expected, tok(code));
-        TODO_ASSERT_EQUALS(expected, actual, tok(code));
     }
 
     void template26() {
@@ -1431,11 +1423,11 @@ private:
                             "} ; "
                             "void A :: t_func<0> ( ) "
                             "{ "
-                            "if ( 0 || foo<int> ( ) ) { ; } "
+                            "if ( 0 != 0 || foo<int> ( ) ) { ; } "
                             "} "
                             "void A :: t_func<1> ( ) "
                             "{ "
-                            "if ( 1 ) { ; } "
+                            "if ( 1 != 0 || foo<int> ( ) ) { ; } "
                             "} "
                             "bool foo<int> ( ) { return true ; }";
         ASSERT_EQUALS(exp, tok(code));