From cccf0355351a5e8c768bb242b57b1ffa5fe24b6c Mon Sep 17 00:00:00 2001
From: Florin Iucha <florin@signbit.net>
Date: Sat, 14 Oct 2017 19:27:47 -0400
Subject: [PATCH] Memoize token nature computation

---
 lib/token.cpp | 40 ++++++++++++++++++++--------------------
 lib/token.h   | 38 ++++++++++++++++++++++++--------------
 2 files changed, 44 insertions(+), 34 deletions(-)

diff --git a/lib/token.cpp b/lib/token.cpp
index f934b7eea..e6a73925a 100644
--- a/lib/token.cpp
+++ b/lib/token.cpp
@@ -69,32 +69,32 @@ void Token::update_property_info()
 {
     if (!_str.empty()) {
         if (_str == "true" || _str == "false")
-            _tokType = eBoolean;
+            tokType(eBoolean);
         else if (std::isalpha((unsigned char)_str[0]) || _str[0] == '_' || _str[0] == '$') { // Name
             if (_varId)
-                _tokType = eVariable;
+                tokType(eVariable);
             else if (_tokType != eVariable && _tokType != eFunction && _tokType != eType && _tokType != eKeyword)
-                _tokType = eName;
+                tokType(eName);
         } else if (std::isdigit((unsigned char)_str[0]) || (_str.length() > 1 && _str[0] == '-' && std::isdigit((unsigned char)_str[1])))
-            _tokType = eNumber;
+            tokType(eNumber);
         else if (_str.length() > 1 && _str[0] == '"' && endsWith(_str,'"'))
-            _tokType = eString;
+            tokType(eString);
         else if (_str.length() > 1 && _str[0] == '\'' && endsWith(_str,'\''))
-            _tokType = eChar;
+            tokType(eChar);
         else if (_str == "=" || _str == "<<=" || _str == ">>=" ||
                  (_str.size() == 2U && _str[1] == '=' && std::strchr("+-*/%&^|", _str[0])))
-            _tokType = eAssignmentOp;
+            tokType(eAssignmentOp);
         else if (_str.size() == 1 && _str.find_first_of(",[]()?:") != std::string::npos)
-            _tokType = eExtendedOp;
+            tokType(eExtendedOp);
         else if (_str=="<<" || _str==">>" || (_str.size()==1 && _str.find_first_of("+-*/%") != std::string::npos))
-            _tokType = eArithmeticalOp;
+            tokType(eArithmeticalOp);
         else if (_str.size() == 1 && _str.find_first_of("&|^~") != std::string::npos)
-            _tokType = eBitOp;
+            tokType(eBitOp);
         else if (_str.size() <= 2 &&
                  (_str == "&&" ||
                   _str == "||" ||
                   _str == "!"))
-            _tokType = eLogicalOp;
+            tokType(eLogicalOp);
         else if (_str.size() <= 2 && !_link &&
                  (_str == "==" ||
                   _str == "!=" ||
@@ -102,17 +102,17 @@ void Token::update_property_info()
                   _str == "<=" ||
                   _str == ">"  ||
                   _str == ">="))
-            _tokType = eComparisonOp;
+            tokType(eComparisonOp);
         else if (_str.size() == 2 &&
                  (_str == "++" ||
                   _str == "--"))
-            _tokType = eIncDecOp;
+            tokType(eIncDecOp);
         else if (_str.size() == 1 && (_str.find_first_of("{}") != std::string::npos || (_link && _str.find_first_of("<>") != std::string::npos)))
-            _tokType = eBracket;
+            tokType(eBracket);
         else
-            _tokType = eOther;
+            tokType(eOther);
     } else {
-        _tokType = eNone;
+        tokType(eNone);
     }
 
     update_property_isStandardType();
@@ -140,7 +140,7 @@ void Token::update_property_isStandardType()
 
     if (stdTypes.find(_str)!=stdTypes.end()) {
         isStandardType(true);
-        _tokType = eType;
+        tokType(eType);
     }
 }
 
@@ -232,7 +232,7 @@ void Token::swapWithNext()
 void Token::takeData(Token *fromToken)
 {
     _str = fromToken->_str;
-    _tokType = fromToken->_tokType;
+    tokType(fromToken->_tokType);
     _flags = fromToken->_flags;
     _varId = fromToken->_varId;
     _fileIndex = fromToken->_fileIndex;
@@ -1627,9 +1627,9 @@ void Token::type(const ::Type *t)
 {
     _type = t;
     if (t) {
-        _tokType = eType;
+        tokType(eType);
         isEnumType(_type->isEnumType());
     } else if (_tokType == eType)
-        _tokType = eName;
+        tokType(eName);
 }
 
diff --git a/lib/token.h b/lib/token.h
index 8c4b64766..58431095c 100644
--- a/lib/token.h
+++ b/lib/token.h
@@ -238,24 +238,31 @@ public:
     }
     void tokType(Token::Type t) {
         _tokType = t;
+
+        bool memoizedIsName = (_tokType == eName || _tokType == eType || _tokType == eVariable ||
+              _tokType == eFunction || _tokType == eKeyword || _tokType == eBoolean ||
+              _tokType == eEnumerator); // TODO: "true"/"false" aren't really a name...
+        setFlag(fIsName, memoizedIsName);
+
+        bool memoizedIsLiteral = (_tokType == eNumber || _tokType == eString || _tokType == eChar ||
+               _tokType == eBoolean || _tokType == eLiteral || _tokType == eEnumerator);
+        setFlag(fIsLiteral, memoizedIsLiteral);
     }
     void isKeyword(bool kwd) {
         if (kwd)
-            _tokType = eKeyword;
+            tokType(eKeyword);
         else if (_tokType == eKeyword)
-            _tokType = eName;
+            tokType(eName);
     }
     bool isKeyword() const {
         return _tokType == eKeyword;
     }
     bool isName() const {
-        return _tokType == eName || _tokType == eType || _tokType == eVariable || _tokType == eFunction || _tokType == eKeyword ||
-               _tokType == eBoolean || _tokType == eEnumerator; // TODO: "true"/"false" aren't really a name...
+        return getFlag(fIsName);
     }
     bool isUpperCaseName() const;
     bool isLiteral() const {
-        return _tokType == eNumber || _tokType == eString || _tokType == eChar ||
-               _tokType == eBoolean || _tokType == eLiteral || _tokType == eEnumerator;
+        return getFlag(fIsLiteral);
     }
     bool isNumber() const {
         return _tokType == eNumber;
@@ -502,7 +509,7 @@ public:
     void varId(unsigned int id) {
         _varId = id;
         if (id != 0) {
-            _tokType = eVariable;
+            tokType(eVariable);
             isStandardType(false);
         } else {
             update_property_info();
@@ -616,9 +623,9 @@ public:
     void function(const Function *f) {
         _function = f;
         if (f)
-            _tokType = eFunction;
+            tokType(eFunction);
         else if (_tokType == eFunction)
-            _tokType = eName;
+            tokType(eName);
     }
 
     /**
@@ -635,9 +642,9 @@ public:
     void variable(const Variable *v) {
         _variable = v;
         if (v || _varId)
-            _tokType = eVariable;
+            tokType(eVariable);
         else if (_tokType == eVariable)
-            _tokType = eName;
+            tokType(eName);
     }
 
     /**
@@ -674,9 +681,9 @@ public:
     void enumerator(const Enumerator *e) {
         _enumerator = e;
         if (e)
-            _tokType = eEnumerator;
+            tokType(eEnumerator);
         else if (_tokType == eEnumerator)
-            _tokType = eName;
+            tokType(eName);
     }
 
     /**
@@ -884,7 +891,10 @@ private:
         fIsAttributePacked      = (1 << 15), // __attribute__((packed))
         fIsOperatorKeyword      = (1 << 16), // operator=, etc
         fIsComplex              = (1 << 17), // complex/_Complex type
-        fIsEnumType             = (1 << 18)  // enumeration type
+        fIsEnumType             = (1 << 18), // enumeration type
+
+        fIsName                 = (1 << 19),
+        fIsLiteral              = (1 << 20),
     };
 
     unsigned int _flags;