Tokenizer: Detect some syntax errors when trying to use templates. Ticket: #1961

lib/tokenize.cpp

@@ -1670,7 +1670,7 @@ bool Tokenizer::tokenize(std::istream &code, const char FileName[], const std::s
     }
 
     // check for simple syntax errors..
-    for (Token *tok = _tokens; tok; tok = tok->next())
+    for (const Token *tok = _tokens; tok; tok = tok->next())
     {
         if (Token::simpleMatch(tok, "> struct {") &&
             Token::simpleMatch(tok->tokAt(2)->link(), "} ;"))
@@ -1751,6 +1751,86 @@ bool Tokenizer::tokenize(std::istream &code, const char FileName[], const std::s
         }
     }
 
+    // check for more complicated syntax errors when using templates..
+    for (const Token *tok = _tokens; tok; tok = tok->next())
+    {
+        // skip executing scopes..
+        if (Token::Match(tok, ") const| {"))
+        {
+            while (tok->str() != "{")
+                tok = tok->next();
+            tok = tok->link();
+        }
+
+        // not start of statement?
+        if (tok->previous() && !Token::Match(tok, "[;{}]"))
+            continue;
+
+        // skip starting tokens.. ;;; typedef typename foo::bar::..
+        while (Token::Match(tok, "[;{}]"))
+            tok = tok->next();
+        while (Token::Match(tok, "typedef|typename"))
+            tok = tok->next();
+        while (Token::Match(tok, "%type% ::"))
+            tok = tok->tokAt(2);
+        if (!tok)
+            break;
+
+        // template variable or type..
+        if (Token::Match(tok, "%type% <"))
+        {
+            // these are used types..
+            std::set<std::string> usedtypes;
+
+            // parse this statement and see if the '<' and '>' are matching
+            unsigned int level = 0;
+            for (const Token *tok2 = tok; tok2 && !Token::Match(tok2, "[;{}]"); tok2 = tok2->next())
+            {
+                if (tok2->str() == "(")
+                    tok2 = tok2->link();
+                else if (tok2->str() == "<")
+                {
+                    bool inclevel = false;
+                    if (level == 0)
+                        inclevel = true;
+                    else if (tok2->next()->isStandardType())
+                        inclevel = true;
+                    else if (Token::simpleMatch(tok2, "< typename"))
+                        inclevel = true;
+                    else if (Token::Match(tok2->tokAt(-2), "<|, %type% <") && usedtypes.find(tok2->strAt(-1)) != usedtypes.end())
+                        inclevel = true;
+                    else if (Token::Match(tok2, "< %type%") && usedtypes.find(tok2->strAt(1)) != usedtypes.end())
+                        inclevel = true;
+                    else if (Token::Match(tok2, "< %type%"))
+                    {
+                        // is the next token a type and not a variable/constant?
+                        // assume it's a type if there comes another "<"
+                        const Token *tok3 = tok2->next();
+                        while (Token::Match(tok3, "%type% ::"))
+                            tok3 = tok3->tokAt(2);
+                        if (Token::Match(tok3, "%type% <"))
+                            inclevel = true;
+                    }
+
+                    if (inclevel)
+                    {
+                        ++level;
+                        if (Token::Match(tok2->tokAt(-2), "<|, %type% <"))
+                            usedtypes.insert(tok2->strAt(-1));
+                    }
+                }
+                else if (tok2->str() == ">")
+                {
+                    if (level > 0)
+                        --level;
+                }
+            }
+            if (level > 0)
+                syntaxError(tok);
+        }
+    }
+
+
     // Remove "= default|delete" inside class|struct definitions
     // Todo: Remove it if it is used "externally" too.
     for (Token *tok = _tokens; tok; tok = tok->next())

test/testtokenize.cpp

@@ -210,6 +210,7 @@ private:
         TEST_CASE(vardecl_template);
         TEST_CASE(volatile_variables);
         TEST_CASE(syntax_error);
+        TEST_CASE(syntax_error_templates);
 
         TEST_CASE(removeKeywords);
 
@@ -3665,6 +3666,42 @@ private:
         }
     }
 
+
+    void syntax_error_templates()
+    {
+        // ok code.. using ">" for a comparison
+        {
+            errout.str("");
+            std::istringstream istr("x<y>z> xyz;\n");
+            Tokenizer tokenizer(0, this);
+            tokenizer.tokenize(istr, "test.cpp");
+            ASSERT_EQUALS("", errout.str());
+        }
+
+        // bad code.. missing ">"
+        {
+            errout.str("");
+            std::istringstream istr("x<y<int> xyz;\n");
+            Tokenizer tokenizer(0, this);
+            tokenizer.tokenize(istr, "test.cpp");
+            ASSERT_EQUALS("[test.cpp:1]: (error) syntax error\n", errout.str());
+        }
+
+        // bad code
+        {
+            errout.str("");
+            std::istringstream istr("typedef\n"
+                                    "    typename boost::mpl::if_c<\n"
+                                    "          _visitableIndex < boost::mpl::size< typename _Visitables::ConcreteVisitables >::value\n"
+                                    "          , ConcreteVisitable\n"
+                                    "          , Dummy< _visitableIndex >\n"
+                                    "    >::type ConcreteVisitableOrDummy;\n");
+            Tokenizer tokenizer(0, this);
+            tokenizer.tokenize(istr, "test.cpp");
+            ASSERT_EQUALS("[test.cpp:2]: (error) syntax error\n", errout.str());
+        }
+    }
+
     void removeKeywords()
     {
         const char code[] = "if (__builtin_expect(!!(x), 1));";