Optimised simplifyKnownVariables (#1205)

* Optimised simplifyKnownVariables

Changed the check for references to constant variables so instead of iterating through all tokens looking for references (which is very slow for large files), usages of each known variable are recorded so each usage can be checked for whether or not it is a reference.  Just checking known usages is a lot quicker than checking through all tokens.

* Fixed test error caused by incorrect code

* Fixes to constant variable simplification optimisation

Indexing variables by varId is easier than by Token pointer, but means we also have to store a reference to the constant variable Token in another collection.
Switched to using unordered_map over map as it has slightly better find performance.

* Fixed incorrect simplification behaviour

This should remove constant variables correctly and efficiently.  Requires additional functions, Token::deleteThisInPlace() and TokenList::front(Token*).

* Added setter for TokenList::first

This allows code that adds and removes Tokens from the list to do so without copying nodes into other nodes, which sometimes create difficulties.

* Added deleteThisInPlace function

This allows a token to delete itself without invalidating pointers to the node after it.  If this token is the first or last in a list the calling code will have to remember to change the list's front or back.

* Added declaration for deleteThisInPlace()

* Removed premature MatchCompiler optimisation

* Added and removed some functions

Added declaration for deleteToken(Token*) which deletes a single token from the list
Removed public front(Token*) because it broke encapsulation

* Implemented deleteToken(Token*) function

* Removed 'delete this' from deleteThisInPlace

* Switched to using safer function

TokenList::deleteToken is better than calling straight into Token::deleteThisInPlace because it doesn't call delete this and doesn't break the TokenList's encapsulation.

* Replace constant variables in reverse order

This fixes the problem where you have two constant value assignment statements in a row.  Replacing and deleting them in reverse order means we avoid the problem of deleteThis() potentially invalidating the pointer to the start of the next assignment statement

* Removed unneeded and unsafe deleteThisInPlace

* Removed unneeded and unsafe deleteThisInPlace

* Removed unneeded deleteToken

* Removed unneeded deleteToken

* Removed extra whitespace

lib/tokenize.cpp

@@ -40,6 +40,7 @@
 #include <ctime>
 #include <iostream>
 #include <stack>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 //---------------------------------------------------------------------------
@@ -6323,7 +6324,9 @@ bool Tokenizer::simplifyKnownVariables()
 
     // constants..
     {
-        std::map<unsigned int, std::string> constantValues;
+        std::unordered_map<unsigned int, std::string> constantValues;
+        std::map<unsigned int, Token*> constantVars;
+        std::unordered_map<unsigned int, std::list<Token*>> constantValueUsages;
         bool goback = false;
         for (Token *tok = list.front(); tok; tok = tok->next()) {
             if (goback) {
@@ -6380,36 +6383,53 @@ bool Tokenizer::simplifyKnownVariables()
                 if (!tok->isStandardType())
                     continue;
 
-                const Token * const vartok = (tok->next() && tok->next()->str() == "const") ? tok->tokAt(2) : tok->next();
+                Token * const vartok = (tok->next() && tok->next()->str() == "const") ? tok->tokAt(2) : tok->next();
                 const Token * const valuetok = vartok->tokAt(2);
                 if (Token::Match(valuetok, "%bool%|%char%|%num%|%str% )| ;")) {
-                    //check if there's not a reference usage inside the code
+                    // record a constant value for this variable
-                    bool withreference = false;
-                    for (const Token *tok2 = valuetok->tokAt(2); tok2; tok2 = tok2->next()) {
-                        if (Token::Match(tok2,"(|[|,|{|return|%op% & %varid%", vartok->varId())) {
-                            withreference = true;
-                            break;
-                        }
-                    }
-                    //don't simplify 'f(&x)' to 'f(&100)'
-                    if (withreference)
-                        continue;
-
                     constantValues[vartok->varId()] = valuetok->str();
+                    constantVars[vartok->varId()] = tok1;
+                }
+            }
+            else if (tok->varId()) {
+                // find the entry for the known variable, if any.  Exclude the location where the variable is assigned with next == "="
+                if (constantValues.find(tok->varId()) != constantValues.end() && tok->next()->str() != "=") {
+                    constantValueUsages[tok->varId()].push_back(tok);
+                }
+            }
+        }
 
-                    // remove statement
+        for (auto constantVar = constantVars.rbegin(); constantVar != constantVars.rend(); constantVar++)
-                    while (tok1->next()->str() != ";")
+        {
-                        tok1->deleteNext();
+            bool referenceFound = false;
-                    tok1->deleteNext();
+            std::list<Token*> usageList = constantValueUsages[constantVar->first];
-                    tok1->deleteThis();
+            for (Token* usage : usageList)
-                    tok = tok1;
+            {
-                    goback = true;
+                // check if any usages of each known variable are a reference
-                    ret = true;
+                if (Token::Match(usage->tokAt(-2), "(|[|,|{|return|%op% & %varid%", constantVar->first))
+                {
+                    referenceFound = true;
+                    break;
                 }
             }
             
-            else if (tok->varId() && constantValues.find(tok->varId()) != constantValues.end()) {
+            if (!referenceFound)
-                tok->str(constantValues[tok->varId()]);
+            {
+                // replace all usages of non-referenced known variables with their value
+                for (Token* usage : usageList)
+                {
+                    usage->str(constantValues[constantVar->first]);
+                }
+
+                Token* startTok = constantVar->second;
+                // remove variable assignment statement
+                while (startTok->next()->str() != ";")
+                    startTok->deleteNext();
+                startTok->deleteNext();
+				startTok->deleteThis();
+                
+                constantVar->second = nullptr;
+                ret = true;
             }
         }
     }