TOKEN: Match() function improved, %any% and const|volatile kind of patterns are now accepted. Simplified comparing on tokenize.cpp.

testtokenize.cpp

@@ -45,8 +45,10 @@ private:
         TEST_CASE( dupfuncname );
 
         TEST_CASE( const_and_volatile_functions );
-        
+
-        TEST_CASE( numeric_true_condition );
+        TEST_CASE( numeric_true_condition );
+
+        TEST_CASE( multi_compare );
     }
 
 
@@ -188,7 +190,7 @@ private:
         }
     }
 
-    
+
     void numeric_true_condition()
     {
         const char code[] = "void f()\n"
@@ -207,6 +209,23 @@ private:
         for (const TOKEN *tok = tokenizer.tokens(); tok; tok = tok->next)
             ostr << " " << tok->str();
         ASSERT_EQUALS( std::string(" void f ( ) { if ( true ) ; }"), ostr.str() );
+    }
+
+    void multi_compare()
+    {
+        // Test for found
+        ASSERT_EQUALS( TOKEN::multiCompare( "one|two", "one" ), 1 );
+        ASSERT_EQUALS( TOKEN::multiCompare( "one|two", "two" ), 1 );
+        ASSERT_EQUALS( TOKEN::multiCompare( "verybig|two|", "two" ), 1 );
+
+        // Test for empty string found
+        ASSERT_EQUALS( TOKEN::multiCompare( "|one|two", "notfound" ), 0 );
+        ASSERT_EQUALS( TOKEN::multiCompare( "one||two", "notfound" ), 0 );
+        ASSERT_EQUALS( TOKEN::multiCompare( "one|two|", "notfound" ), 0 );
+
+        // Test for not found
+        ASSERT_EQUALS( TOKEN::multiCompare( "one|two", "notfound" ), -1 );
+        ASSERT_EQUALS( TOKEN::multiCompare( "verybig|two", "s" ), -1 );
     }
 };
 

token.cpp

@@ -51,7 +51,7 @@ void TOKEN::setstr( const char s[] )
     _cstr = _strdup(s);
 #endif
     _isName = bool(_str[0]=='_' || isalpha(_str[0]));
-    _isNumber = bool(isdigit(_str[0]) != 0);    
+    _isNumber = bool(isdigit(_str[0]) != 0);
 }
 
 void TOKEN::combineWithNext(const char str1[], const char str2[])
@@ -89,6 +89,37 @@ const char *TOKEN::strAt(int index) const
     const TOKEN *tok = this->tokAt(index);
     return tok ? tok->_cstr : "";
 }
+
+int TOKEN::multiCompare( const char *needle, const char *haystack )
+{
+    bool emptyStringFound = false;
+    bool findNextOr = false;
+    for( ; *needle; ++needle )
+    {
+        if( *needle == '|' )
+        {
+            if( findNextOr )
+                findNextOr = false;
+            else
+                emptyStringFound = true;
+
+            continue;
+        }
+
+        // If this part of needle equals to the haystack
+        else if( strncmp( haystack, needle, strlen( haystack ) ) == 0 )
+            return 1;
+        else
+            findNextOr = true;
+    }
+
+    // If empty string was found or if last character in needle was '|'
+    if( emptyStringFound || findNextOr == false )
+        return 0;
+
+    return -1;
+}
+
 
 bool TOKEN::Match(const TOKEN *tok, const char pattern[], const char *varname1[], const char *varname2[])
 {
@@ -122,6 +153,12 @@ bool TOKEN::Match(const TOKEN *tok, const char pattern[], const char *varname1[]
         {
             if (!tok->isName())
                 return false;
+        }
+
+        // Accept any token
+        else if (strcmp(str,"%any%")==0 )
+        {
+
         }
 
         // Variable name..
@@ -170,6 +207,22 @@ bool TOKEN::Match(const TOKEN *tok, const char pattern[], const char *varname1[]
             if ( strchr( str + 1, tok->_str[0] ) == 0 )
                 return false;
         }
+
+        // Parse multi options, such as void|int|char (accept token which is one of these 3)
+        else if ( strchr(str, '|') && strlen( str ) > 2 )
+        {
+            int res = multiCompare( str, tok->_cstr );
+            if( res == 0 )
+            {
+                // Empty alternative matches, use the same token on next round
+                continue;
+            }
+            else if( res == 2 )
+            {
+                // No match
+                return false;
+            }
+        }
 
         else if (str != tok->_str)
             return false;

token.h

@@ -30,7 +30,7 @@ public:
 
     const std::string &str() const
     { return _str; }
-    
+
     const char *aaaa() const
     { return _cstr; }
 
@@ -39,7 +39,7 @@ public:
 
     char aaaa1() const
     { return _cstr[1]; }
-    
+
 
     /**
      * Combine two tokens that belong to each other.
@@ -61,13 +61,56 @@ public:
 
     const char *strAt(int index) const;
 
+    /**
+     * Match given token (or list of tokens) to a pattern list.
+     *
+     * Possible patterns
+     * "%any%" any token
+     * "%var%" any token which is a name or type e.g. "hello" or "int"
+     * "%num%" Any numeric token, e.g. "23"
+     * "%str%" Any token starting with "-character (C-string).
+     * "[abc]" Any of the characters 'a' or 'b' or 'c'
+     * "int|void|char" Any of the strings, int, void or char
+     * "int|void|char|" Any of the strings, int, void or char or empty string
+     * "someRandomText" If token contains "someRandomText".
+     *
+     * The patterns can be also combined to compare to multiple tokens at once
+     * by separating tokens with a space, e.g.
+     * ") const|void {" will return true if first token is ')' next token is either
+     * "const" or "void" and token after that is '{'. If even one of the tokens does not
+     * match its pattern, false is returned.
+     *
+     * @param tok List of tokens to be compared to the pattern
+     * @param pattern The pattern where tokens are compared, e.g. "const"
+     * or ") const|volatile| {".
+     * @param varname1 Used with pattern "%var1%" and "%var2%"
+     * @param varname2 Used with pattern "%var1%" and "%var2%"
+     * @return true if given token matches with given pattern
+     *         false if given token does not match with given pattern
+     */
     static bool Match(const TOKEN *tok, const char pattern[], const char *varname1[]=0, const char *varname2[]=0);
+
     bool isName() const;
     bool isNumber() const;
     bool isStandardType() const;
     static const TOKEN *findmatch(const TOKEN *tok, const char pattern[], const char *varname1[]=0, const char *varname2[]=0);
     static const TOKEN *findtoken(const TOKEN *tok1, const char *tokenstr[]);
 
+    /**
+     * Needle is build from multiple alternatives. If one of
+     * them is equal to haystack, return value is 1. If there
+     * are no matches, but one alternative to needle is empty
+     * string, return value is 0. If needle was not found, return
+     * value is -1.
+     *
+     * @param needle e.g. "one|two" or "|one|two"
+     * @param haystack e.g. "one", "two" or "invalid"
+     * @return 1 if needle is found from the haystack
+     *         0 if needle was empty string
+     *        -1 if needle was not found
+     */
+    static int multiCompare( const char *needle, const char *haystack );
+
     unsigned int FileIndex;
     unsigned int linenr;
     TOKEN *next;

tokenize.cpp

@@ -1005,7 +1005,7 @@ bool Tokenizer::simplifyConditions()
             tok->next->setstr((strcmp(tok->next->aaaa(), "0")!=0) ? "true" : "false");
             ret = false;
         }
-        
+
         // Reduce "(%num% == %num%)" => "(true)"/"(false)"
         if ( (TOKEN::Match(tok, "&&") || TOKEN::Match(tok, "||") || TOKEN::Match(tok, "(")) &&
              TOKEN::Match(tok->tokAt(1), "%num%") &&
@@ -1031,13 +1031,13 @@ bool Tokenizer::simplifyConditions()
                 result = (op1 < op2);
             else
                 cmp = "";
-            
+
             if ( ! cmp.empty() )
             {
                 tok = tok->next;
                 tok->deleteNext();
                 tok->deleteNext();
-            
+
                 tok->setstr( result ? "true" : "false" );
                 ret = false;
             }
@@ -1116,22 +1116,12 @@ void Tokenizer::fillFunctionList()
                 }
 
                 else if ( tok2->aaaa0() == ')' )
-                {
+                {
-                    if ( TOKEN::Match(tok2, ") {") )
+                    if ( TOKEN::Match(tok2, ") const|volatile| {") )
-                    {
-                        _functionList.push_back( tok );
-                        tok = tok2;
-                    }
-                    else if ( TOKEN::Match(tok2, ") const {") )
                     {
                         _functionList.push_back( tok );
                         tok = tok2;
                     }
-                    else if ( TOKEN::Match(tok2, ") volatile {") )
-                    {
-                        _functionList.push_back( tok );
-                        tok = tok2;
-                    }
                     else
                     {
                         tok = tok2;