Tokenizer::simplifyTemplates: Better handling for multi-token template arguments such as 'Foo<std::string>'

lib/tokenize.cpp

@@ -2467,21 +2467,31 @@ void Tokenizer::labels()
  * is the token pointing at a template parameters block
  * < int , 3 > => yes
  * \param tok start token that must point at "<"
- * \return true if the tokens look like template parameters
+ * \return number of parameters (invalid parameters => 0)
  */
-static bool templateParameters(const Token *tok)
+static unsigned int templateParameters(const Token *tok)
 {
+    unsigned int numberOfParameters = 0;
+
     if (!tok)
-        return false;
+        return 0;
     if (tok->str() != "<")
-        return false;
+        return 0;
     tok = tok->next();
 
     while (tok)
     {
+        ++numberOfParameters;
+    
+        // skip std::
+        while (Token::Match(tok, "%var% ::"))
+            tok = tok->tokAt(2);
+        if (!tok)
+            return 0;
+
         // num/type ..
         if (!tok->isNumber() && !tok->isName())
-            return false;
+            return 0;
         tok = tok->next();
 
         // optional "*"
@@ -2490,12 +2500,12 @@ static bool templateParameters(const Token *tok)
 
         // ,/>
         if (tok->str() == ">")
-            return true;
+            return numberOfParameters;
         if (tok->str() != ",")
             break;
         tok = tok->next();
     }
-    return false;
+    return 0;
 }
 
 
@@ -2778,6 +2788,33 @@ void Tokenizer::simplifyTemplatesUseDefaultArgumentValues(const std::list<Token
     }
 }
 
+/**
+ * Match template declaration/instantiation
+ * @param instance template instantiation
+ * @param name name of template
+ * @param numberOfArguments number of template arguments
+ * @param patternAfter pattern that must match the tokens after the ">"
+ * @return match => true
+ */
+static bool simplifyTemplatesInstantiateMatch(const Token *instance, const std::string &name, unsigned int numberOfArguments, const char patternAfter[])
+{
+    if (!Token::simpleMatch(instance, (name + " <").c_str()))
+        return false;
+
+    if (numberOfArguments != templateParameters(instance->next()))
+        return false;
+
+    if (patternAfter)
+    {
+        const Token *tok = Token::findmatch(instance, ">");
+        if (!tok || !Token::Match(tok->next(), patternAfter))
+            return false;
+    }
+
+    // nothing mismatching was found..
+    return true;
+}
+
 void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
         std::list<Token *> &used,
         std::set<std::string> &expandedtemplates)
@@ -2836,16 +2873,6 @@ void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
     const bool isfunc(tok->strAt(namepos + 1) == "(");
 
     // locate template usage..
-
-    std::string s(name + " <");
-    for (unsigned int i = 0; i < type.size(); ++i)
-    {
-        if (i > 0)
-            s += ",";
-        s += " %any% ";
-    }
-    const std::string pattern(s + "> ");
-
     std::string::size_type sz1 = used.size();
     unsigned int recursiveCount = 0;
 
@@ -2865,17 +2892,16 @@ void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
         }
 
         Token * const tok2 = *iter2;
-
         if (tok2->str() != name)
             continue;
 
         if (Token::Match(tok2->previous(), "[;{}=]") &&
-            !Token::Match(tok2, (pattern + (isfunc ? "(" : "*| %var%")).c_str()))
+            !simplifyTemplatesInstantiateMatch(*iter2, name, type.size(), isfunc ? "(" : "*| %var%"))
             continue;
 
         // New type..
-        std::vector<Token> types2;
+        std::vector<const Token *> types2;
-        s = "";
+        std::string s;
         std::string s1(name + " < ");
         for (const Token *tok3 = tok2->tokAt(2); tok3 && tok3->str() != ">"; tok3 = tok3->next())
         {
@@ -2886,8 +2912,8 @@ void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
             }
             s1 += tok3->str();
             s1 += " ";
-            if (tok3->str() != ",")
+            if (Token::Match(tok3->previous(), "[<,]"))
-                types2.push_back(*tok3);
+                types2.push_back(tok3);
             // add additional type information
             if (tok3->isUnsigned())
                 s += "unsigned";
@@ -2949,7 +2975,9 @@ void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
                 }
 
                 // member function implemented outside class definition
-                else if (_indentlevel == 0 && _parlevel == 0 && Token::Match(tok3, (pattern + " :: ~| %var% (").c_str()))
+                else if (_indentlevel == 0 && 
+                         _parlevel == 0 && 
+                         simplifyTemplatesInstantiateMatch(tok3, name, type.size(), ":: ~| %var% ("))
                 {
                     addtoken(name2.c_str(), tok3->linenr(), tok3->fileIndex());
                     while (tok3->str() != "::")
@@ -2997,7 +3025,12 @@ void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
                         // replace type with given type..
                         if (itype < type.size())
                         {
-                            addtoken(&types2[itype], tok3->linenr(), tok3->fileIndex());
+                            for (const Token *typetok = types2[itype]; 
+                                 typetok && !Token::Match(typetok, "[,>]"); 
+                                 typetok = typetok->next())
+                            {
+                                addtoken(typetok, tok3->linenr(), tok3->fileIndex());
+                            }
                             continue;
                         }
                     }
@@ -3065,18 +3098,26 @@ void Tokenizer::simplifyTemplatesInstantiate(const Token *tok,
                 bool match = true;
                 Token * tok5 = tok4->tokAt(2);
                 unsigned int count = 0;
+                const Token *typetok = (!types2.empty()) ? types2[0] : 0;
                 while (tok5->str() != ">")
                 {
                     if (tok5->str() != ",")
                     {
-                        if (tok5->isUnsigned() != types2[count].isUnsigned() ||
+                        if (!typetok ||
-                            tok5->isSigned() != types2[count].isSigned() ||
+                            tok5->isUnsigned() != typetok->isUnsigned() ||
-                            tok5->isLong() != types2[count].isLong())
+                            tok5->isSigned() != typetok->isSigned() ||
+                            tok5->isLong() != typetok->isLong())
                         {
                             match = false;
                             break;
                         }
-                        count++;
+                        
+                        typetok = typetok ? typetok->next() : 0;
+                    }
+                    else
+                    {
+                        ++count;
+                        typetok = (count < types2.size()) ? types2[count] : 0;
                     }
                     tok5 = tok5->next();
                 }
@@ -9190,7 +9231,7 @@ void Tokenizer::simplifyBorland()
 void Tokenizer::simplifyQtSignalsSlots()
 {
     Token *tok = _tokens;
-    while ((tok = const_cast<Token *>(Token::findmatch(tok, "class %var% :"))))
+    while (NULL != (tok = const_cast<Token *>(Token::findmatch(tok, "class %var% :"))))
     {
         if (tok->previous() && tok->previous()->str() == "enum")
         {

test/testsimplifytokens.cpp

@@ -110,6 +110,7 @@ private:
         TEST_CASE(template19);
         TEST_CASE(template20);
         TEST_CASE(template21);
+        TEST_CASE(template22);
         TEST_CASE(template_unhandled);
         TEST_CASE(template_default_parameter);
         TEST_CASE(template_default_type);
@@ -1985,6 +1986,19 @@ private:
         }
     }
 
+    void template22()
+    {
+            const char code[] = "template <classname T> struct Fred { T a; };\n"
+                                "Fred<std::string> fred;";
+
+            const std::string expected("; "
+                                       "Fred<std::string> fred ; "
+                                       "struct Fred<std::string> { std :: string a ; }");
+
+            ASSERT_EQUALS(expected, sizeof_(code));
+        }
+
+
     void template_unhandled()
     {
         // An unhandled template usage should be simplified..