Tokenizer: improve code and remove redundant checks.

In particular: improve 'double sharp' concatenation by handling also code which begins without the [{};]. Do the same with C# code. Also, in the labels simplification function, skip also '{}' when the open bracket is after a '='.

lib/tokenize.cpp

@@ -1935,26 +1935,35 @@ bool Tokenizer::tokenize(std::istream &code,
             tok->str(MathLib::toString(tok->linenr()));
     }
 
-    // token concatenation
+    // 'double sharp' token concatenation
-    for (Token *tok = _tokens; tok; tok = tok->next()) {
+    {
-        // TODO: pattern should be "%var%|%num% ## %var%|%num%"
+        bool goback = false;
-        if (Token::Match(tok, "%any% ## %any%") &&
+        for (Token *tok = _tokens; tok; tok = tok->next()) {
-            (tok->isName() || tok->isNumber()) &&
+            if (goback) {
-            (tok->tokAt(2)->isName() || tok->tokAt(2)->isNumber())) {
+                goback = false;
-            tok->str(tok->str() + tok->strAt(2));
-            tok->deleteNext(2);
-            if (tok->previous())
                 tok = tok->previous();
+            }
+            // TODO: pattern should be "%var%|%num% ## %var%|%num%"
+            if (Token::Match(tok, "%any% ## %any%") &&
+                (tok->isName() || tok->isNumber()) &&
+                (tok->tokAt(2)->isName() || tok->tokAt(2)->isNumber())) {
+                tok->str(tok->str() + tok->strAt(2));
+                tok->deleteNext(2);
+                goback = true;
+            }
         }
     }
 
     // Convert C# code
     if (_files[0].find(".cs")) {
         for (Token *tok = _tokens; tok; tok = tok->next()) {
-            if (Token::Match(tok, "[;{}] %type% [ ] %var% [=;]")) {
+            if (Token::Match(tok, "%type% [ ] %var% [=;]") &&
+                (!tok->previous() || Token::Match(tok->previous(), "[;{}]"))) {
+                tok->deleteNext(2);
+                tok->insertToken("*");
                 tok = tok->tokAt(2);
-                tok->str("*");
+                if (tok->next()->str() == "=")
-                tok->deleteNext();
+                    tok = tok->next();
             }
         }
     }
@@ -2620,9 +2629,12 @@ void Tokenizer::simplifyLabelsCaseDefault()
             // Simplify labels in the executable scope..
             unsigned int indentlevel = 0;
             while (NULL != (tok = tok->next())) {
-                if (tok->str() == "{")
+                if (tok->str() == "{") {
-                    ++indentlevel;
+                    if (tok->previous() && tok->previous()->str() == "=")
-                else if (tok->str() == "}") {
+                        tok = tok->link();
+                    else
+                        ++indentlevel;
+                } else if (tok->str() == "}") {
                     --indentlevel;
                     if (!indentlevel)
                         break;
@@ -5703,18 +5715,15 @@ void Tokenizer::simplifyVarDecl(bool only_k_r_fpar)
     for (Token *tok = _tokens; tok; tok = tok->next()) {
         if (Token::simpleMatch(tok, "= {")) {
             tok = tok->next()->link();
-            if (!tok)
-                break;
         }
 
         if (only_k_r_fpar) {
-            if (tok->str() == "(" || tok->str() == "{") {
+            if (tok->link()) {
                 tok = tok->link();
-                if (!tok)
-                    break;
                 if (tok->next() && Token::Match(tok, ") !!{"))
                     tok = tok->next();
-                else continue;
+                else
+                    continue;
             } else
                 continue;
         }
@@ -5768,10 +5777,9 @@ void Tokenizer::simplifyVarDecl(bool only_k_r_fpar)
         if (Token::Match(tok2, "%type% *| %var% ,|=")) {
             const bool isPointer = (tok2->next()->str() == "*");
             const Token *varName = tok2->tokAt((isPointer ? 2 : 1));
-            Token *endDeclaration = varName->next();
 
             if (varName->str() != "operator") {
-                tok2 = endDeclaration; // The ',' or '=' token
+                tok2 = varName->next(); // The ',' or '=' token
 
                 if (isstatic && tok2->str() == "=") {
                     if (Token::Match(tok2->next(), "%num% ,"))
@@ -5866,70 +5874,69 @@ void Tokenizer::simplifyVarDecl(bool only_k_r_fpar)
                 --typelen;
             } else {
                 tok2 = NULL;
-                typelen = 0;
             }
         } else {
             tok2 = NULL;
-            typelen = 0;
         }
 
-        if (tok2) {
+        if (!tok2)
-            if (tok2->str() == ",") {
+            continue;
-                tok2->str(";");
-                insertTokens(tok2, type0, typelen);
-                std::stack<Token *> link1;
-                std::stack<Token *> link2;
-                while (((typelen--) > 0) && (NULL != (tok2 = tok2->next()))) {
-                    if (tok2->str() == "(")
-                        link1.push(tok2);
-                    else if (tok2->str() == ")" && !link1.empty()) {
-                        Token::createMutualLinks(tok2, link1.top());
-                        link1.pop();
-                    }
 
-                    else if (tok2->str() == "[")
+        if (tok2->str() == ",") {
-                        link2.push(tok2);
+            tok2->str(";");
-                    else if (tok2->str() == "]" && !link2.empty()) {
+            insertTokens(tok2, type0, typelen);
-                        Token::createMutualLinks(tok2, link2.top());
+            std::stack<Token *> link1;
-                        link2.pop();
+            std::stack<Token *> link2;
-                    }
+            while (((typelen--) > 0) && (NULL != (tok2 = tok2->next()))) {
+                if (tok2->str() == "(")
+                    link1.push(tok2);
+                else if (tok2->str() == ")" && !link1.empty()) {
+                    Token::createMutualLinks(tok2, link1.top());
+                    link1.pop();
+                }
+
+                else if (tok2->str() == "[")
+                    link2.push(tok2);
+                else if (tok2->str() == "]" && !link2.empty()) {
+                    Token::createMutualLinks(tok2, link2.top());
+                    link2.pop();
                 }
             }
+        }
 
-            else {
+        else {
-                Token *eq = tok2;
+            Token *eq = tok2;
 
-                unsigned int level = 0;
+            unsigned int level = 0;
-                while (tok2) {
+            while (tok2) {
-                    if (Token::Match(tok2, "[{(]"))
+                if (Token::Match(tok2, "[{(]"))
-                        tok2 = tok2->link();
+                    tok2 = tok2->link();
 
-                    else if (tok2->str() == "<") {
+                else if (tok2->str() == "<") {
-                        if (tok2->previous()->isName() && !tok2->previous()->varId())
+                    if (tok2->previous()->isName() && !tok2->previous()->varId())
-                            ++level;
+                        ++level;
-                    }
-
-                    else if (level > 0 && tok2->str() == ">")
-                        --level;
-
-                    else if (level == 0 && strchr(";,", tok2->str()[0])) {
-                        // "type var ="   =>   "type var; var ="
-                        Token *VarTok = type0->tokAt((int)typelen);
-                        while (Token::Match(VarTok, "*|&|const"))
-                            VarTok = VarTok->next();
-                        insertTokens(eq, VarTok, 2);
-                        eq->str(";");
-
-                        // "= x, "   =>   "= x; type "
-                        if (tok2->str() == ",") {
-                            tok2->str(";");
-                            insertTokens(tok2, type0, typelen);
-                        }
-                        break;
-                    }
-
-                    tok2 = tok2->next();
                 }
+
+                else if (level > 0 && tok2->str() == ">")
+                    --level;
+
+                else if (level == 0 && strchr(";,", tok2->str()[0])) {
+                    // "type var ="   =>   "type var; var ="
+                    Token *VarTok = type0->tokAt((int)typelen);
+                    while (Token::Match(VarTok, "*|&|const"))
+                        VarTok = VarTok->next();
+                    insertTokens(eq, VarTok, 2);
+                    eq->str(";");
+
+                    // "= x, "   =>   "= x; type "
+                    if (tok2->str() == ",") {
+                        tok2->str(";");
+                        insertTokens(tok2, type0, typelen);
+                    }
+                    break;
+                }
+
+                tok2 = tok2->next();
             }
         }
     }

test/testtokenize.cpp

@@ -3882,7 +3882,7 @@ private:
 
 
     void doublesharp() {
-        const char code[] = "TEST(var,val) var##_##val = val\n";
+        const char code[] = "a##_##b TEST(var,val) var##_##val = val\n";
 
         errout.str("");
 
@@ -3898,7 +3898,7 @@ private:
         for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next())
             ostr << tok->str() << " ";
 
-        ASSERT_EQUALS("TEST ( var , val ) var_val = val ", ostr.str());
+        ASSERT_EQUALS("a_b TEST ( var , val ) var_val = val ", ostr.str());
     }
 
     void macrodoublesharp() {
@@ -5876,6 +5876,7 @@ private:
     }
 
     void cs() {
+        ASSERT_EQUALS("int * i ;", tokenizeAndStringify("int [] i;"));
         ASSERT_EQUALS("; int * i ;", tokenizeAndStringify("; int [] i;"));
     }