Improve tokenizer: remove more redundant parenthesis and add a prototype code for: '&str[num]'=>'(str+num)' with reporting non obvious test failures.

lib/tokenize.cpp

@@ -4154,38 +4154,55 @@ bool Tokenizer::simplifyTokenList()
 
     // Replace "*(str + num)" => "str[num]"
     for (Token *tok = _tokens; tok; tok = tok->next()) {
-        if (! strchr(";{}(=<>", tok->str()[0]))
+        if (Token::Match(tok->next(), "* ( %var% + %num% )") ||
-            continue;
+            Token::Match(tok->next(), "* ( %var% + %var% )")) {
+            // remove '* ('
+            tok->deleteNext();
+            tok->deleteNext();
 
-        Token *next = tok->next();
+            tok = tok->tokAt(2);
-        if (! next)
+            // '+'->'['
-            break;
-
-        if (Token::Match(next, "* ( %var% + %num% )") ||
-            Token::Match(next, "* ( %var% + %var% )")) {
-            // var
-            tok = tok->next();
-            tok->str(tok->strAt(2));
-
-            // [
-            tok = tok->next();
             tok->str("[");
 
-            // num
+            tok = tok->tokAt(2);
-            tok = tok->next();
-            tok->str(tok->strAt(2));
-
-            // ]
-            tok = tok->next();
             tok->str("]");
-
+            Token::createMutualLinks(tok->tokAt(-2), tok);
-            tok->deleteNext();
-            tok->deleteNext();
-
-            Token::createMutualLinks(next->next(), next->tokAt(3));
         }
     }
 
+/*    // Replace "&str[num]" => "(str + num)"
+    //TODO: fix the fails testrunner reports:
+    //1)
+    //test/teststl.cpp:805: Assertion failed.
+    //Expected:
+    //"[test.cpp:7]: (error) Invalid pointer 'first' after push_back / push_front\n".
+    //Actual:
+    //"".
+    //2)
+    //test/testautovariables.cpp:279: Assertion failed.
+    //Expected:
+    //"[test.cpp:4]: (error) Return of the address of an auto-variable\n".
+    //Actual:
+    //"".
+    for (Token *tok = _tokens; tok; tok = tok->next()) {
+        if ((Token::Match(tok->next(), "& %var% [ %num% ]") ||
+             Token::Match(tok->next(), "& %var% [ %var% ]"))) {
+            tok = tok->next();
+            // '&' => '('
+            tok->str("(");
+
+            tok = tok->next();
+            // '[' => '+'
+            tok->deleteNext();
+            tok->insertToken("+");
+
+            tok = tok->tokAt(3);
+            //remove ']'
+            tok->str(")");
+            Token::createMutualLinks(tok->tokAt(-4), tok);
+        }
+    }*/
+
     // simplify "x=realloc(y,0);" => "free(y); x=0;"..
     // and "x = realloc (0, n);" => "x = malloc(n);"
     for (Token *tok = _tokens; tok; tok = tok->next()) {
@@ -7219,7 +7236,7 @@ bool Tokenizer::simplifyRedundantParenthesis()
             continue;
         }
 
-        if (Token::Match(tok->previous(), "[(!] ( %var% . %var% )")) {
+        if (Token::Match(tok->previous(), "[(,!] ( %var% . %var% )")) {
             // We have "( var . var )", remove the parenthesis
             tok->deleteThis();
             tok = tok->tokAt(2);
@@ -7228,6 +7245,16 @@ bool Tokenizer::simplifyRedundantParenthesis()
             continue;
         }
 
+        if (Token::Match(tok->previous(), "(|[|,|! ( %var% %op% %var% ) ,|]|)") ||
+            Token::Match(tok->previous(), "(|[|,|! ( %var% %op% %num% ) ,|]|)")) {
+            // We have "( var %op% var )", remove the parenthesis
+            tok->deleteThis();
+            tok = tok->tokAt(2);
+            tok->deleteNext();
+            ret = true;
+            continue;
+        }
+
         if (Token::Match(tok, "( ( %bool% )") ||
             Token::Match(tok, "( ( %num% )")) {
             tok->tokAt(2)->deleteNext();

test/testtokenize.cpp

@@ -240,6 +240,7 @@ private:
         TEST_CASE(removeParentheses10);      // Ticket #2320
         TEST_CASE(removeParentheses11);      // Ticket #2505
         TEST_CASE(removeParentheses12);      // Ticket #2760 ',(b)='
+        TEST_CASE(removeParentheses13);
 
         TEST_CASE(tokenize_double);
         TEST_CASE(tokenize_strings);
@@ -3951,6 +3952,11 @@ private:
         ASSERT_EQUALS(", x = 0 ;", tokenizeAndStringify(",(x)=0;", false));
     }
 
+    void removeParentheses13() {
+        ASSERT_EQUALS("; f ( a + b , c ) ;", tokenizeAndStringify(";f((a+b),c);", false));
+        ASSERT_EQUALS("; x = y [ a + b ] ;", tokenizeAndStringify(";x=y[(a+b)];", false));
+    }
+
     void tokenize_double() {
         const char code[] = "void f()\n"
                             "{\n"