Refactoring the handling of typedefs

lib/tokenize.cpp

@@ -450,13 +450,14 @@ void Tokenizer::simplifyTypedef()
         Token *start = 0;
         Token *end = 0;
         Token *num = 0;
+        Token *typeDef = tok;
 
         if (Token::Match(tok->next(), "%type% <") ||
             Token::Match(tok->next(), "%type% :: %type% <") ||
-            Token::Match(tok->next(), "%type% *| %type% ;") ||
+            Token::Match(tok->next(), "%type% *| %type% ;|,") ||
-            Token::Match(tok->next(), "%type% %type% *| %type% ;") ||
+            Token::Match(tok->next(), "%type% %type% *| %type% ;|,") ||
-            Token::Match(tok->next(), "%type% *| %type% [ %num% ] ;") ||
+            Token::Match(tok->next(), "%type% *| %type% [ %num% ] ;|,") ||
-            Token::Match(tok->next(), "%type% %type% *| %type% [ %num% ] ;"))
+            Token::Match(tok->next(), "%type% %type% *| %type% [ %num% ] ;|,"))
         {
             if ((tok->tokAt(2)->str() == "<") ||
                 (tok->tokAt(4) && (tok->tokAt(4)->str() == "<")))
@@ -485,7 +486,7 @@ void Tokenizer::simplifyTypedef()
                 while (end && end->next() && Token::Match(end->next(), ":: %type%"))
                     end = end->tokAt(2);
 
-                if (end && end->next() && Token::Match(end->next(), "%type% ;"))
+                if (end && end->next() && Token::Match(end->next(), "%type% ;|,"))
                 {
                     typeName = end->strAt(1);
                     tok = end->tokAt(2);
@@ -535,7 +536,8 @@ void Tokenizer::simplifyTypedef()
                     }
                 }
             }
-            else if (tok->tokAt(3)->str() == ";")
+            else if (tok->tokAt(3)->str() == ";" ||
+                     tok->tokAt(3)->str() == ",")
             {
                 type1 = tok->strAt(1);
                 type2 = 0;
@@ -569,115 +571,172 @@ void Tokenizer::simplifyTypedef()
                 }
             }
 
-            const std::string pattern(className.empty() ? "" : (className + " :: " + typeName).c_str());
+            bool done = false;
-            int level = 0;
-            bool inScope = true;
 
-            bool exitThisScope = false;
+            while (!done)
-            int exitScope = 0;
-            bool simplifyType = false;
-            for (Token *tok2 = tok; tok2; tok2 = tok2->next())
             {
-                if (tok2->str() == "}")
+                const std::string pattern(className.empty() ? "" : (className + " :: " + typeName).c_str());
-                {
+                int level = 0;
-                    --level;
+                bool inScope = true;
-                    if (level < 0)
-                        inScope = false;
 
-                    if (exitThisScope)
+                bool exitThisScope = false;
-                    {
+                int exitScope = 0;
-                        if (level < exitScope)
+                bool simplifyType = false;
-                            exitThisScope = false;
+                for (Token *tok2 = tok; tok2; tok2 = tok2->next())
-                    }
-                }
-                else if (tok2->str() == "{")
-                    ++level;
-                else if (!pattern.empty() && Token::Match(tok2, pattern.c_str()))
                 {
-                    tok2->deleteNext();
+                    if (tok2->str() == "}")
-                    tok2->deleteNext();
-                    simplifyType = true;
-                }
-                else if (inScope && !exitThisScope && tok2->str() == typeName)
-                {
-                    if (Token::simpleMatch(tok2->previous(), "::"))
                     {
-                        // Don't replace this typename if it's preceded by "::"
+                        --level;
-                    }
+                        if (level < 0)
-                    else if (Token::Match(tok2->tokAt(-2), "!!typedef") &&
+                            inScope = false;
-                             Token::Match(tok2->tokAt(-3), "!!typedef"))
-                    {
-                        // Check for enum and typedef with same name.
-                        if (type1 && (tok2->tokAt(-1)->str() != type1))
-                            simplifyType = true;
-                        else if (!type1)
-                            simplifyType = true;
-                    }
-                    else
-                    {
-                        // Typedef with the same name.
-                        exitThisScope = true;
-                        exitScope = level;
-                    }
-                }
 
-                if (simplifyType)
+                        if (exitThisScope)
-                {
-                    if (start && end)
-                    {
-                        tok2->str(start->str());
-                        Token * nextToken;
-                        std::list<Token *> links;
-                        for (nextToken = start->next(); nextToken != end->next(); nextToken = nextToken->next())
                         {
-                            tok2->insertToken(nextToken->strAt(0));
+                            if (level < exitScope)
-                            tok2 = tok2->next();
+                                exitThisScope = false;
+                        }
+                    }
+                    else if (tok2->str() == "{")
+                        ++level;
+                    else if (!pattern.empty() && Token::Match(tok2, pattern.c_str()))
+                    {
+                        tok2->deleteNext();
+                        tok2->deleteNext();
+                        simplifyType = true;
+                    }
+                    else if (inScope && !exitThisScope && tok2->str() == typeName)
+                    {
+                        if (Token::simpleMatch(tok2->previous(), "::"))
+                        {
+                            // Don't replace this typename if it's preceded by "::"
+                        }
+                        else if (Token::Match(tok2->tokAt(-2), "!!typedef") &&
+                                 Token::Match(tok2->tokAt(-3), "!!typedef"))
+                        {
+                            // Check for enum and typedef with same name.
+                            if (type1 && (tok2->tokAt(-1)->str() != type1))
+                                simplifyType = true;
+                            else if (!type1)
+                                simplifyType = true;
+                        }
+                        else
+                        {
+                            // Typedef with the same name.
+                            exitThisScope = true;
+                            exitScope = level;
+                        }
+                    }
 
-                            // Check for links and fix them up
+                    if (simplifyType)
-                            if (tok2->str() == "(")
+                    {
-                                links.push_back(tok2);
+                        if (start && end)
-                            if (tok2->str() == ")")
+                        {
+                            tok2->str(start->str());
+                            Token * nextToken;
+                            std::list<Token *> links;
+                            for (nextToken = start->next(); nextToken != end->next(); nextToken = nextToken->next())
                             {
-                                Token * link = links.back();
+                                tok2->insertToken(nextToken->strAt(0));
+                                tok2 = tok2->next();
 
-                                tok2->link(link);
+                                // Check for links and fix them up
-                                link->link(tok2);
+                                if (tok2->str() == "(")
+                                    links.push_back(tok2);
+                                if (tok2->str() == ")")
+                                {
+                                    Token * link = links.back();
 
-                                links.pop_back();
+                                    tok2->link(link);
+                                    link->link(tok2);
+
+                                    links.pop_back();
+                                }
                             }
                         }
-                    }
+                        else
-                    else
-                    {
-                        tok2->str(type1);
-                        if (type2)
                         {
-                            tok2->insertToken(type2);
+                            tok2->str(type1);
-                            tok2 = tok2->next();
+                            if (type2)
+                            {
+                                tok2->insertToken(type2);
+                                tok2 = tok2->next();
+                            }
+                            if (pointer)
+                            {
+                                tok2->insertToken("*");
+                                tok2 = tok2->next();
+                            }
+
+                            if (num)
+                            {
+                                tok2 = tok2->next();
+                                tok2->insertToken("[");
+                                tok2 = tok2->next();
+                                Token *tok3 = tok2;
+                                tok2->insertToken(num->strAt(0));
+                                tok2 = tok2->next();
+                                tok2->insertToken("]");
+                                tok2 = tok2->next();
+                                Token::createMutualLinks(tok2, tok3);
+                            }
                         }
+
+                        simplifyType = false;
+                    }
+                }
+
+                if (tok->str() == ";")
+                    done = true;
+                else
+                {
+                    if (Token::Match(tok->next(), "*| %type% ;|,") ||
+                        Token::Match(tok->next(), "*| %type% [ %num% ] ;|,"))
+                    {
+                        num = 0;
+                        pointer = (tok->tokAt(1)->str() == "*");
+
                         if (pointer)
                         {
-                            tok2->insertToken("*");
+                            typeName = tok->strAt(2);
-                            tok2 = tok2->next();
-                        }
 
-                        if (num)
+                            if (tok->tokAt(3)->str() == "[")
+                            {
+                                num = tok->tokAt(4);
+                                tok = tok->tokAt(6);
+                            }
+                            else
+                                tok = tok->tokAt(3);
+                        }
+                        else
                         {
-                            tok2 = tok2->next();
+                            typeName = tok->strAt(1);
-                            tok2->insertToken("[");
+
-                            tok2 = tok2->next();
+                            if (tok->tokAt(2)->str() == "[")
-                            Token *tok3 = tok2;
+                            {
-                            tok2->insertToken(num->strAt(0));
+                                num = tok->tokAt(3);
-                            tok2 = tok2->next();
+                                tok = tok->tokAt(5);
-                            tok2->insertToken("]");
+                            }
-                            tok2 = tok2->next();
+                            else
-                            Token::createMutualLinks(tok2, tok3);
+                                tok = tok->tokAt(2);
                         }
                     }
-
-                    simplifyType = false;
                 }
             }
+
+            // remove typedef but leave ;
+            while (typeDef->next() && typeDef->next() != tok)
+                typeDef->deleteNext();
+
+            if (typeDef != _tokens)
+            {
+                tok = typeDef->previous();
+                tok->deleteNext();
+            }
+            else
+            {
+                _tokens->deleteThis();
+                tok = _tokens;
+            }
         }
     }
 }
@@ -819,9 +878,6 @@ bool Tokenizer::tokenize(std::istream &code, const char FileName[])
     // Split up variable declarations.
     simplifyVarDecl();
 
-    // typedef..
-    simplifyTypedef();
-
     // Handle templates..
     simplifyTemplates();
 

test/testsimplifytokens.cpp

@@ -151,6 +151,7 @@ private:
         TEST_CASE(simplifyTypedef14);
         TEST_CASE(simplifyTypedef15);
         TEST_CASE(simplifyTypedef16);
+        TEST_CASE(simplifyTypedef17);
         TEST_CASE(reverseArraySyntax)
         TEST_CASE(simplify_numeric_condition)
 
@@ -1241,7 +1242,7 @@ private:
                             "\n"
                             "template<typename T> inline B<T> h() { return B<T>(); }\n";
 
-        ASSERT_EQUALS("; ; typedef A < int > x ; typedef B < int > y ; ; ; ;", sizeof_(code));
+        ASSERT_EQUALS("; ; ; ; ; ; ;", sizeof_(code));
 
         ASSERT_EQUALS("class A { ; } ;", sizeof_("class A{ template<typename T> int foo(T d);};"));
     }
@@ -2172,10 +2173,10 @@ private:
             "class A "
             "{ "
             "public: "
-            "typedef wchar_t duplicate ; "
+            "; "
             "void foo ( ) { } "
             "} ; "
-            "typedef A duplicate ; "
+            "; "
             "int main ( ) "
             "{ "
             "A a ; "
@@ -2198,11 +2199,11 @@ private:
 
         const std::string expected =
             "class A ; "
-            "typedef A duplicate ; "
+            "; "
             "class A "
             "{ "
             "public: "
-            "typedef wchar_t duplicate ; "
+            "; "
             "wchar_t foo ( ) { wchar_t b ; return b ; } "
             "} ;";
         ASSERT_EQUALS(expected, tok(code));
@@ -2225,10 +2226,10 @@ private:
 
         const std::string expected =
             "class A { } ; "
-            "typedef A duplicate ; "
+            "; "
             "wchar_t foo ( ) "
             "{ "
-            "typedef wchar_t duplicate ; "
+            "; "
             "wchar_t b ; "
             "return b ; "
             "} "
@@ -2251,8 +2252,8 @@ private:
                             "}\n";
 
         const std::string expected =
-            "typedef int s32 ; "
+            "; "
-            "typedef unsigned int u32 ; "
+            "; "
             "void f ( ) "
             "{ "
             "int ivar ; ivar = -2 ; "
@@ -2273,7 +2274,7 @@ private:
             "}\n";
 
         const char expected[] =
-            "typedef struct yy_buffer_state * YY_BUFFER_STATE ; "
+            "; "
             "void f ( ) "
             "{ "
             "struct yy_buffer_state * state ; "
@@ -2293,7 +2294,7 @@ private:
 
         const char expected[] =
             "namespace VL { "
-            "typedef float float_t ; "
+            "; "
             "inline float fast_atan2 ( float y , float x ) { } "
             "}";
 
@@ -2304,7 +2305,10 @@ private:
     {
         const char code[] = "typedef int abc ; "
                             "Fred :: abc f ;";
-        ASSERT_EQUALS(code, tok(code, false));
+        const char expected[] =
+            "; "
+            "Fred :: abc f ;";
+        ASSERT_EQUALS(expected, tok(code, false));
     }
 
     void simplifyTypedef8()
@@ -2319,10 +2323,10 @@ private:
                             "PUINT tpui;";
 
         const char expected[] =
-            "typedef int INT ; "
+            "; "
-            "typedef unsigned int UINT ; "
+            "; "
-            "typedef int * PINT ; "
+            "; "
-            "typedef unsigned int * PUINT ; "
+            "; "
             "int ti ; "
             "unsigned int tui ; "
             "int * tpi ; "
@@ -2333,7 +2337,7 @@ private:
 
     void simplifyTypedef9()
     {
-        const char code[] = "typedef struct s S, * PS\n;"
+        const char code[] = "typedef struct s S, * PS;\n"
                             "typedef struct t { int a; } T, *TP;\n"
                             "typedef struct { int a; } U;\n"
                             "typedef struct { int a; } * V;\n"
@@ -2345,10 +2349,10 @@ private:
                             "V v;";
 
         const char expected[] =
-            "typedef struct s S ; typedef struct s * PS ; "
+            "; "
-            "struct t { int a ; } ; typedef struct t T ; typedef struct t * TP ; "
+            "struct t { int a ; } ; ; "
-            "struct U { int a ; } ; typedef struct U U ; "
+            "struct U { int a ; } ; ; "
-            "struct Unnamed0 { int a ; } ; typedef struct Unnamed0 * V ; "
+            "struct Unnamed0 { int a ; } ; ; "
             "struct s s ; "
             "struct s * ps ; "
             "struct t t ; "
@@ -2361,7 +2365,7 @@ private:
 
     void simplifyTypedef10()
     {
-        const char code[] = "typedef union s S, * PS\n;"
+        const char code[] = "typedef union s S, * PS;\n"
                             "typedef union t { int a; float b ; } T, *TP;\n"
                             "typedef union { int a; float b; } U;\n"
                             "typedef union { int a; float b; } * V;\n"
@@ -2373,10 +2377,10 @@ private:
                             "V v;";
 
         const char expected[] =
-            "typedef union s S ; typedef union s * PS ; "
+            "; "
-            "union t { int a ; float b ; } ; typedef union t T ; typedef union t * TP ; "
+            "union t { int a ; float b ; } ; ; "
-            "union U { int a ; float b ; } ; typedef union U U ; "
+            "union U { int a ; float b ; } ; ; "
-            "union Unnamed1 { int a ; float b ; } ; typedef union Unnamed1 * V ; "
+            "union Unnamed1 { int a ; float b ; } ; ; "
             "union s s ; "
             "union s * ps ; "
             "union t t ; "
@@ -2395,8 +2399,8 @@ private:
                             "XYZ e2;";
 
         const char expected[] =
-            "enum abc { a = 0 , b = 1 , c = 2 } ; typedef enum abc abc ; "
+            "enum abc { a = 0 , b = 1 , c = 2 } ; ; "
-            "enum xyz { x = 0 , y = 1 , z = 2 } ; typedef enum xyz XYZ ; "
+            "enum xyz { x = 0 , y = 1 , z = 2 } ; ; "
             "enum abc e1 ; "
             "enum xyz e2 ;";
 
@@ -2415,10 +2419,10 @@ private:
                             "IntListIterator iter;";
 
         const char expected[] =
-            "typedef vector < int > V1 ; "
+            "; "
-            "typedef std :: vector < int > V2 ; "
+            "; "
-            "typedef std :: vector < std :: vector < int > > V3 ; "
+            "; "
-            "typedef std :: list < int > :: iterator IntListIterator ; "
+            "; "
             "vector < int > v1 ; "
             "std :: vector < int > v2 ; "
             "std :: vector < std :: vector < int > > v3 ; "
@@ -2480,7 +2484,7 @@ private:
                                 "frame f;";
 
             const char expected[] =
-                "typedef char frame [ 10 ] ; "
+                "; "
                 "char f [ 10 ] ;";
 
             ASSERT_EQUALS(expected, tok(code, false));
@@ -2491,7 +2495,7 @@ private:
                                 "frame f;";
 
             const char expected[] =
-                "typedef unsigned char frame [ 10 ] ; "
+                "; "
                 "unsigned char f [ 10 ] ;";
 
             ASSERT_EQUALS(expected, tok(code, false));
@@ -2519,6 +2523,20 @@ private:
         ASSERT_EQUALS("", errout.str());
     }
 
+    void simplifyTypedef17()
+    {
+        const char code[] = "typedef char * PCHAR, CHAR;\n"
+                            "PCHAR pc;\n"
+                            "CHAR c;";
+
+        const char expected[] =
+            "; "
+            "char * pc ; "
+            "char c ;";
+
+        ASSERT_EQUALS(expected, tok(code, false));
+    }
+
     void reverseArraySyntax()
     {
         ASSERT_EQUALS("a [ 13 ]", tok("13[a]"));

test/testtokenize.cpp

@@ -1370,7 +1370,7 @@ private:
         // result..
         const std::string actual(tokenizer.tokens()->stringifyList(true));
         const std::string expected("\n\n##file 0\n"
-                                   "1: typedef int INT32 ;\n");
+                                   "1: ;\n");
 
         ASSERT_EQUALS(expected, actual);
     }