Variable Id: Fixed various bugs related to templates and bitfields. Ticket: #1928

lib/tokenize.cpp

@@ -1679,6 +1679,9 @@ bool Tokenizer::tokenize(std::istream &code, const char FileName[], const std::s
     // specify array size..
     arraySize();
 
+    // simplify bit fields..
+    simplifyBitfields();
+
     // simplify labels..
     labels();
 
@@ -2740,6 +2743,9 @@ void Tokenizer::setVarId()
             {
                 again = false;
 
+                if (tok2->str() == "const")
+                    tok2 = tok2->next();
+
                 while (Token::Match(tok2, "%var% ::"))
                     tok2 = tok2->tokAt(2);
 
@@ -2749,26 +2755,28 @@ void Tokenizer::setVarId()
                     tok2 = tok2->tokAt(2);
                     again = true;
                 }
-                else if (Token::Match(tok2, "%type% *| ,"))
+                else if (Token::Match(tok2, "%type% *|&| ,"))
                 {
                     tok2 = tok2->tokAt(2);
                     if (tok2->str() == ",")
                         tok2 = tok2->next();
                     again = true;
                 }
-                else if (level > 1 && Token::Match(tok2, "%type% *| >"))
+                else if (level > 1 && Token::Match(tok2, "%type% *|&| >"))
                 {
                     --level;
                     while (tok2->str() != ">")
                         tok2 = tok2->next();
                     tok2 = tok2->next();
-                    if (level == 1 && tok->str() == ">")
+                    if (tok2->str() == ",")
+                        tok2 = tok2->next();
+                    if (level == 1 && tok2->str() == ">")
                         break;
                     again = true;
                 }
                 else
                 {
-                    while (tok2 && (tok2->isName() || tok2->isNumber() || tok2->str() == "*" || tok2->str() == ","))
+                    while (tok2 && (tok2->isName() || tok2->isNumber() || tok2->str() == "*" || tok2->str() == "&" || tok2->str() == ","))
                         tok2 = tok2->next();
                 }
             }
@@ -7849,6 +7857,18 @@ void Tokenizer::simplifyAsm()
     }
 }
 
+
+// Simplify bitfields
+void Tokenizer::simplifyBitfields()
+{
+    for (Token *tok = _tokens; tok; tok = tok->next())
+    {
+        if (Token::Match(tok, "[;{] int|signed|unsigned %var% : %num% ;"))
+            Token::eraseTokens(tok->tokAt(2), tok->tokAt(5));
+    }
+}
+
+
 // Remove __builtin_expect(...), likely(...), and unlikely(...)
 void Tokenizer::simplifyBuiltinExpect()
 {

lib/tokenize.h

@@ -463,6 +463,11 @@ public:
      */
     void simplifyAsm();
 
+    /**
+     * Simplify bitfields - the field width is removed as we don't use it.
+     */
+    void simplifyBitfields();
+
     /**
      * Remove __builtin_expect(...), likely(...), and unlikely(...)
      */

test/testtokenize.cpp

@@ -142,6 +142,8 @@ private:
         TEST_CASE(varid17); // ticket #1810
         TEST_CASE(varid18);
         TEST_CASE(varid19);
+        TEST_CASE(varid20);
+        TEST_CASE(varid21);
         TEST_CASE(varidStl);
         TEST_CASE(varid_delete);
         TEST_CASE(varid_functions);
@@ -234,6 +236,8 @@ private:
 
         TEST_CASE(labels);
         TEST_CASE(simplifyInitVar);
+
+        TEST_CASE(bitfields);
     }
 
 
@@ -2266,6 +2270,38 @@ private:
         ASSERT_EQUALS(expected, tokenizeDebugListing(code));
     }
 
+    void varid20()
+    {
+        const std::string code("void foo()\n"
+                               "{\n"
+                               "    pair<vector<int>, vector<double> > x;\n"
+                               "}\n");
+
+        const std::string expected("\n\n##file 0\n"
+                                   "1: void foo ( )\n"
+                                   "2: {\n"
+                                   "3: pair < vector < int > , vector < double > > x@1 ;\n"
+                                   "4: }\n");
+
+        ASSERT_EQUALS(expected, tokenizeDebugListing(code));
+    }
+
+    void varid21()
+    {
+        const std::string code("void foo()\n"
+                               "{\n"
+                               "    vector<const std::string &> x;\n"
+                               "}\n");
+
+        const std::string expected("\n\n##file 0\n"
+                                   "1: void foo ( )\n"
+                                   "2: {\n"
+                                   "3: vector < const std :: string & > x@1 ;\n"
+                                   "4: }\n");
+
+        ASSERT_EQUALS(expected, tokenizeDebugListing(code));
+    }
+
 
     void varidStl()
     {
@@ -4104,6 +4140,12 @@ private:
             ASSERT_EQUALS("", errout.str());
         }
     }
+
+    void bitfields()
+    {
+        const char code[] = "struct A { int x : 3; };";
+        ASSERT_EQUALS("struct A { int x ; } ;", tokenizeAndStringify(code,false));
+    }
 };
 
 REGISTER_TEST(TestTokenizer)