From 2a0b2f538e3605467dcdf0b18cb1dfb8d1f559ba Mon Sep 17 00:00:00 2001
From: Paul Fultz II <pfultz2@yahoo.com>
Date: Sat, 18 Jun 2022 14:30:42 -0500
Subject: [PATCH] Check tokType in match compiler (#4219)

* Check tokType in match compiler

* Set keyword when tokenlist is missing
---
 lib/token.cpp          | 33 +++++++++++++++++++
 lib/tokenlist.cpp      |  1 +
 tools/matchcompiler.py | 75 +++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/lib/token.cpp b/lib/token.cpp
index da8e0940d..1b212eed3 100644
--- a/lib/token.cpp
+++ b/lib/token.cpp
@@ -86,6 +86,37 @@ static const std::unordered_set<std::string> controlFlowKeywords = {
     "return"
 };
 
+// Another list of keywords
+static const std::unordered_set<std::string> baseKeywords = {
+    "asm",
+    "auto",
+    "break",
+    "case",
+    "const",
+    "continue",
+    "default",
+    "do",
+    "else",
+    "enum",
+    "extern",
+    "for",
+    "goto",
+    "if",
+    "inline",
+    "register",
+    "restrict",
+    "return",
+    "sizeof",
+    "static",
+    "struct",
+    "switch",
+    "typedef",
+    "union",
+    "volatile",
+    "while",
+    "void"
+};
+
 void Token::update_property_info()
 {
     setFlag(fIsControlFlowKeyword, controlFlowKeywords.find(mStr) != controlFlowKeywords.end());
@@ -102,6 +133,8 @@ void Token::update_property_info()
                 tokType(eVariable);
             else if (mTokensFrontBack && mTokensFrontBack->list && mTokensFrontBack->list->isKeyword(mStr))
                 tokType(eKeyword);
+            else if (baseKeywords.count(mStr) > 0)
+                tokType(eKeyword);
             else if (mTokType != eVariable && mTokType != eFunction && mTokType != eType && mTokType != eKeyword)
                 tokType(eName);
         } else if (std::isdigit((unsigned char)mStr[0]) || (mStr.length() > 1 && mStr[0] == '-' && std::isdigit((unsigned char)mStr[1])))
diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp
index a74cb2cce..1a6feb1de 100644
--- a/lib/tokenlist.cpp
+++ b/lib/tokenlist.cpp
@@ -52,6 +52,7 @@ TokenList::TokenList(const Settings* settings) :
     mIsCpp(false)
 {
     mTokensFrontBack.list = this;
+    mKeywords.insert("asm");
     mKeywords.insert("auto");
     mKeywords.insert("break");
     mKeywords.insert("case");
diff --git a/tools/matchcompiler.py b/tools/matchcompiler.py
index 29ca714ae..026fe8412 100755
--- a/tools/matchcompiler.py
+++ b/tools/matchcompiler.py
@@ -24,6 +24,77 @@ import glob
 import argparse
 import errno
 
+tokTypes = {
+    '+': ['eArithmeticalOp'],
+    '-': ['eArithmeticalOp'],
+    '*': ['eArithmeticalOp'],
+    '/': ['eArithmeticalOp'],
+    '%': ['eArithmeticalOp'],
+    '>>': ['eArithmeticalOp'],
+    '<<': ['eArithmeticalOp'],
+    '=': ['eAssignmentOp'],
+    '+=': ['eAssignmentOp'],
+    '-=': ['eAssignmentOp'],
+    '*=': ['eAssignmentOp'],
+    '/=': ['eAssignmentOp'],
+    '%=': ['eAssignmentOp'],
+    '&=': ['eAssignmentOp'],
+    '|=': ['eAssignmentOp'],
+    '^=': ['eAssignmentOp'],
+    '&': ['eBitOp'],
+    '^': ['eBitOp'],
+    '~': ['eBitOp'],
+    'true': ['eBoolean'],
+    'false': ['eBoolean'],
+    '{': ['eBracket'],
+    '}': ['eBracket'],
+    '<': ['eBracket', 'eComparisonOp'],
+    '>': ['eBracket', 'eComparisonOp'],
+    '==': ['eComparisonOp'],
+    '!=': ['eComparisonOp'],
+    '<=': ['eComparisonOp'],
+    '>=': ['eComparisonOp'],
+    '<=>': ['eComparisonOp'],
+    '...': ['eEllipsis'],
+    ',': ['eExtendedOp'],
+    '?': ['eExtendedOp'],
+    ':': ['eExtendedOp'],
+    '(': ['eExtendedOp'],
+    ')': ['eExtendedOp'],
+    '[': ['eExtendedOp', 'eLambda'],
+    ']': ['eExtendedOp', 'eLambda'],
+    '++': ['eIncDecOp'],
+    '--': ['eIncDecOp'],
+    'asm': ['eKeyword'],
+    'auto': ['eKeyword', 'eType'],
+    'break': ['eKeyword'],
+    'case': ['eKeyword'],
+    'const': ['eKeyword'],
+    'continue': ['eKeyword'],
+    'default': ['eKeyword'],
+    'do': ['eKeyword'],
+    'else': ['eKeyword'],
+    'enum': ['eKeyword'],
+    'extern': ['eKeyword'],
+    'for': ['eKeyword'],
+    'goto': ['eKeyword'],
+    'if': ['eKeyword'],
+    'inline': ['eKeyword'],
+    'register': ['eKeyword'],
+    'restrict': ['eKeyword'],
+    'return': ['eKeyword'],
+    'sizeof': ['eKeyword'],
+    'static': ['eKeyword'],
+    'struct': ['eKeyword'],
+    'switch': ['eKeyword'],
+    'typedef': ['eKeyword'],
+    'union': ['eKeyword'],
+    'volatile': ['eKeyword'],
+    'while': ['eKeyword'],
+    'void': ['eKeyword', 'eType'],
+    '&&': ['eLogicalOp'],
+    '!': ['eLogicalOp']
+}
 
 class MatchCompiler:
 
@@ -117,7 +188,9 @@ class MatchCompiler:
             return '(tok->isName() && tok->varId() == varid)'
         elif (len(tok) > 2) and (tok[0] == "%"):
             print("unhandled:" + tok)
-
+        elif tok in tokTypes:
+            cond = ' || '.join(['tok->tokType() == Token::{}'.format(tokType) for tokType in tokTypes[tok]])
+            return '(({cond}) && tok->str() == MatchCompiler::makeConstString("{tok}"))'.format(cond=cond, tok=tok)
         return (
             '(tok->str() == MatchCompiler::makeConstString("' + tok + '"))'
         )