Fixed #1816 (Tokenizer: remove restrict keyword)

lib/tokenize.cpp

@@ -1684,101 +1684,13 @@ bool Tokenizer::tokenize(std::istream &code, const char FileName[], const std::s
     simplifyEnum();
 
     // Remove __asm..
-    for (Token *tok = _tokens; tok; tok = tok->next())
+    simplifyAsm();
-    {
-        if (Token::Match(tok->next(), "__asm|_asm|asm {") &&
-            tok->tokAt(2)->link() &&
-            tok->tokAt(2)->link()->next())
-        {
-            Token::eraseTokens(tok, tok->tokAt(2)->link()->next());
-        }
 
-        else if (Token::Match(tok->next(), "__asm__ __volatile__ (") &&
+    // Remove "volatile", "inline", "register", and "restrict"
-                 tok->tokAt(3)->link() &&
+    simplifyKeyword();
-                 tok->tokAt(3)->link()->next())
-        {
-            Token::eraseTokens(tok, tok->tokAt(3)->link()->next());
-        }
-
-        else if (Token::Match(tok->next(), "asm volatile (") &&
-                 tok->tokAt(3)->link() &&
-                 tok->tokAt(3)->link()->next())
-        {
-            Token::eraseTokens(tok, tok->tokAt(3)->link()->next());
-        }
-
-        else if (Token::simpleMatch(tok->next(), "__asm"))
-        {
-            const Token *tok2 = tok->next();
-            while (tok2 && (tok2->isNumber() || tok2->isName() || tok2->str() == ","))
-                tok2 = tok2->next();
-            if (tok2 && tok2->str() == ";")
-                Token::eraseTokens(tok, tok2);
-            else
-                continue;
-        }
-
-        else
-            continue;
-
-        // insert "asm ( )"
-        tok->insertToken(")");
-        tok->insertToken("(");
-        tok->insertToken("asm");
-
-        Token::createMutualLinks(tok->tokAt(2), tok->tokAt(3));
-    }
-
-    // Remove "volatile", "inline" and "register"
-    while (Token::Match(_tokens, "volatile|inline|__inline|__forceinline|register"))
-    {
-        _tokens->deleteThis();
-    }
-    for (Token *tok = _tokens; tok; tok = tok->next())
-    {
-        while (Token::Match(tok->next(), "volatile|inline|__inline|__forceinline|register"))
-        {
-            tok->deleteNext();
-        }
-    }
 
     // Remove __builtin_expect, likely and unlikely
-    for (Token *tok = _tokens; tok; tok = tok->next())
+    simplifyBuiltinExpect();
-    {
-        if (Token::simpleMatch(tok->next(), "__builtin_expect ("))
-        {
-            unsigned int parlevel = 0;
-            for (Token *tok2 = tok->next(); tok2; tok2 = tok2->next())
-            {
-                if (tok2->str() == "(")
-                    ++parlevel;
-                else if (tok2->str() == ")")
-                {
-                    if (parlevel <= 1)
-                        break;
-                    --parlevel;
-                }
-                if (parlevel == 1 && tok2->str() == ",")
-                {
-                    if (Token::Match(tok2, ", %num% )"))
-                    {
-                        tok->deleteNext();
-                        Token::eraseTokens(tok2->previous(), tok2->tokAt(2));
-                    }
-                    break;
-                }
-            }
-        }
-        else if (Token::Match(tok->next(), "likely|unlikely ("))
-        {
-            // remove closing ')'
-            tok->tokAt(2)->link()->previous()->deleteNext();
-
-            // remove "likely|unlikely ("
-            tok->deleteNext();
-            tok->deleteNext();
-        }
-    }
 
     // colapse compound standard types into a single token
     // unsigned long long int => long _isUnsigned=true,_isLong=true
@@ -7746,3 +7658,110 @@ void Tokenizer::simplifyAttribute()
     }
 }
 
+// Remove "volatile", "inline", "register", and "restrict"
+void Tokenizer::simplifyKeyword()
+{
+    const char pattern[] = "volatile|inline|__inline|__forceinline|register|restrict|__restrict__";
+    while (Token::Match(_tokens, pattern))
+    {
+        _tokens->deleteThis();
+    }
+    for (Token *tok = _tokens; tok; tok = tok->next())
+    {
+        while (Token::Match(tok->next(), pattern))
+        {
+            tok->deleteNext();
+        }
+    }
+}
+
+// Remove __asm..
+void Tokenizer::simplifyAsm()
+{
+    for (Token *tok = _tokens; tok; tok = tok->next())
+    {
+        if (Token::Match(tok->next(), "__asm|_asm|asm {") &&
+            tok->tokAt(2)->link() &&
+            tok->tokAt(2)->link()->next())
+        {
+            Token::eraseTokens(tok, tok->tokAt(2)->link()->next());
+        }
+
+        else if (Token::Match(tok->next(), "__asm__ __volatile__ (") &&
+                 tok->tokAt(3)->link() &&
+                 tok->tokAt(3)->link()->next())
+        {
+            Token::eraseTokens(tok, tok->tokAt(3)->link()->next());
+        }
+
+        else if (Token::Match(tok->next(), "asm volatile (") &&
+                 tok->tokAt(3)->link() &&
+                 tok->tokAt(3)->link()->next())
+        {
+            Token::eraseTokens(tok, tok->tokAt(3)->link()->next());
+        }
+
+        else if (Token::simpleMatch(tok->next(), "__asm"))
+        {
+            const Token *tok2 = tok->next();
+            while (tok2 && (tok2->isNumber() || tok2->isName() || tok2->str() == ","))
+                tok2 = tok2->next();
+            if (tok2 && tok2->str() == ";")
+                Token::eraseTokens(tok, tok2);
+            else
+                continue;
+        }
+
+        else
+            continue;
+
+        // insert "asm ( )"
+        tok->insertToken(")");
+        tok->insertToken("(");
+        tok->insertToken("asm");
+
+        Token::createMutualLinks(tok->tokAt(2), tok->tokAt(3));
+    }
+}
+
+// Remove __builtin_expect(...), likely(...), and unlikely(...)
+void Tokenizer::simplifyBuiltinExpect()
+{
+    for (Token *tok = _tokens; tok; tok = tok->next())
+    {
+        if (Token::simpleMatch(tok->next(), "__builtin_expect ("))
+        {
+            unsigned int parlevel = 0;
+            for (Token *tok2 = tok->next(); tok2; tok2 = tok2->next())
+            {
+                if (tok2->str() == "(")
+                    ++parlevel;
+                else if (tok2->str() == ")")
+                {
+                    if (parlevel <= 1)
+                        break;
+                    --parlevel;
+                }
+                if (parlevel == 1 && tok2->str() == ",")
+                {
+                    if (Token::Match(tok2, ", %num% )"))
+                    {
+                        tok->deleteNext();
+                        Token::eraseTokens(tok2->previous(), tok2->tokAt(2));
+                    }
+                    break;
+                }
+            }
+        }
+        else if (Token::Match(tok->next(), "likely|unlikely ("))
+        {
+            // remove closing ')'
+            tok->tokAt(2)->link()->previous()->deleteNext();
+
+            // remove "likely|unlikely ("
+            tok->deleteNext();
+            tok->deleteNext();
+        }
+    }
+}
+

lib/tokenize.h

@@ -448,6 +448,21 @@ public:
      */
     void simplifyAttribute();
 
+    /**
+     * Remove keywords "volatile", "inline", "register", and "restrict"
+     */
+    void simplifyKeyword();
+
+    /**
+     * Remove __asm
+     */
+    void simplifyAsm();
+
+    /**
+     * Remove __builtin_expect(...), likely(...), and unlikely(...)
+     */
+    void simplifyBuiltinExpect();
+
     /**
      * This will return a short name describing function parameters
      * e.g. parameters: (int a, char b) should get name "int,char,".

test/testsimplifytokens.cpp

@@ -5295,6 +5295,12 @@ private:
         ASSERT_EQUALS("int foo ( ) { }", tok("__forceinline int foo ( ) { }", true));
         ASSERT_EQUALS("if ( a ) { }", tok("if ( likely ( a ) ) { }", true));
         ASSERT_EQUALS("if ( a ) { }", tok("if ( unlikely ( a ) ) { }", true));
+        ASSERT_EQUALS("int * p ;", tok("int * __restrict__ p;", true));
+        ASSERT_EQUALS("int * * p ;", tok("int * __restrict__ * p;", true));
+        ASSERT_EQUALS("void foo ( float * a , float * b ) ;", tok("void foo(float * __restrict__ a, float * __restrict__ b);", true));
+        ASSERT_EQUALS("int * p ;", tok("int * restrict p;", true));
+        ASSERT_EQUALS("int * * p ;", tok("int * restrict * p;", true));
+        ASSERT_EQUALS("void foo ( float * a , float * b ) ;", tok("void foo(float * restrict a, float * restrict b);", true));
     }
 
     void simplifyCallingConvention()