From d49f4a611a0fb8605f5c447b3c5d6bede8ae4926 Mon Sep 17 00:00:00 2001
From: PKEuS <philipp.kloke@web.de>
Date: Sat, 2 Mar 2013 09:19:53 -0800
Subject: [PATCH] Reduced code duplication in tokenizer

---
 lib/tokenize.cpp | 252 +++++++++++++++++++----------------------------
 lib/tokenize.h   |   6 ++
 2 files changed, 105 insertions(+), 153 deletions(-)

diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
index 92e0ec24f..9d2c74426 100644
--- a/lib/tokenize.cpp
+++ b/lib/tokenize.cpp
@@ -1560,17 +1560,7 @@ bool Tokenizer::tokenize(std::istream &code,
     assert(_settings);
 
     // Fill the map _typeSize..
-    _typeSize.clear();
-    _typeSize["char"] = 1;
-    _typeSize["bool"] = _settings->sizeof_bool;
-    _typeSize["short"] = _settings->sizeof_short;
-    _typeSize["int"] = _settings->sizeof_int;
-    _typeSize["long"] = _settings->sizeof_long;
-    _typeSize["float"] = _settings->sizeof_float;
-    _typeSize["double"] = _settings->sizeof_double;
-    _typeSize["wchar_t"] = _settings->sizeof_wchar_t;
-    _typeSize["size_t"] = _settings->sizeof_size_t;
-    _typeSize["*"] = _settings->sizeof_pointer;
+    fillTypeSizes();
 
     _configuration = configuration;
 
@@ -1595,30 +1585,8 @@ bool Tokenizer::tokenize(std::istream &code,
     // remove MACRO in variable declaration: MACRO int x;
     removeMacroInVarDecl();
 
-    // Combine wide strings
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
-            // Combine 'L "string"'
-            tok->str(tok->next()->str());
-            tok->deleteNext();
-            tok->isLong(true);
-        }
-    }
-
     // Combine strings
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        if (tok->str()[0] != '"')
-            continue;
-
-        tok->str(simplifyString(tok->str()));
-        while (tok->next() && tok->next()->type() == Token::eString) {
-            tok->next()->str(simplifyString(tok->next()->str()));
-
-            // Two strings after each other, combine them
-            tok->concatStr(tok->next()->str());
-            tok->deleteNext();
-        }
-    }
+    combineStrings();
 
     // replace inline SQL with "asm()" (Oracle PRO*C). Ticket: #1959
     simplifySQL();
@@ -1670,45 +1638,7 @@ bool Tokenizer::tokenize(std::istream &code,
         return false;
 
     // Combine tokens..
-    for (Token *tok = list.front(); tok && tok->next(); tok = tok->next()) {
-        const char c1 = tok->str()[0];
-
-        if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
-            const char c2 = tok->next()->str()[0];
-
-            // combine +-*/ and =
-            if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) {
-                tok->str(tok->str() + c2);
-                tok->deleteNext();
-                continue;
-            }
-
-            // replace "->" with "."
-            else if (c1 == '-' && c2 == '>') {
-                tok->str(".");
-                tok->deleteNext();
-                continue;
-            }
-        }
-
-        else if (tok->str() == ">>" && tok->next()->str() == "=") {
-            tok->str(">>=");
-            tok->deleteNext();
-        }
-
-        else if (tok->str() == "<<" && tok->next()->str() == "=") {
-            tok->str("<<=");
-            tok->deleteNext();
-        }
-
-        else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
-            if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
-                tok->str(tok->str() + ":");
-                tok->deleteNext();
-                continue;
-            }
-        }
-    }
+    combineOperators();
 
     // Simplify: 0[foo] -> *(foo)
     for (Token* tok = list.front(); tok; tok = tok->next()) {
@@ -2037,17 +1967,7 @@ bool Tokenizer::tokenizeCondition(const std::string &code)
     assert(_settings);
 
     // Fill the map _typeSize..
-    _typeSize.clear();
-    _typeSize["char"] = 1;
-    _typeSize["bool"] = _settings->sizeof_bool;
-    _typeSize["short"] = _settings->sizeof_short;
-    _typeSize["int"] = _settings->sizeof_int;
-    _typeSize["long"] = _settings->sizeof_long;
-    _typeSize["float"] = _settings->sizeof_float;
-    _typeSize["double"] = _settings->sizeof_double;
-    _typeSize["wchar_t"] = _settings->sizeof_wchar_t;
-    _typeSize["size_t"] = _settings->sizeof_size_t;
-    _typeSize["*"] = _settings->sizeof_pointer;
+    fillTypeSizes();
 
     {
         std::istringstream istr(code);
@@ -2057,34 +1977,8 @@ bool Tokenizer::tokenizeCondition(const std::string &code)
         }
     }
 
-    // Combine wide strings
-    for (Token *tok = list.front();
-         tok;
-         tok = tok->next()) {
-        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
-            // Combine 'L "string"'
-            tok->str(tok->next()->str());
-            tok->deleteNext();
-            tok->isLong(true);
-        }
-    }
-
     // Combine strings
-    for (Token *tok = list.front();
-         tok;
-         tok = tok->next()) {
-        if (tok->str()[0] != '"')
-            continue;
-
-        tok->str(simplifyString(tok->str()));
-        while (tok->next() && tok->next()->type() == Token::eString) {
-            tok->next()->str(simplifyString(tok->next()->str()));
-
-            // Two strings after each other, combine them
-            tok->concatStr(tok->next()->str());
-            tok->deleteNext();
-        }
-    }
+    combineStrings();
 
     // Remove "volatile", "inline", "register", and "restrict"
     simplifyKeyword();
@@ -2123,48 +2017,7 @@ bool Tokenizer::tokenizeCondition(const std::string &code)
             TemplateSimplifier::simplifyNumericCalculations(tok->previous());
     }
 
-    // Combine tokens..
-    for (Token *tok = list.front();
-         tok && tok->next();
-         tok = tok->next()) {
-        const char c1 = tok->str()[0];
-
-        if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
-            const char c2 = tok->next()->str()[0];
-
-            // combine +-*/ and =
-            if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) {
-                tok->str(tok->str() + c2);
-                tok->deleteNext();
-                continue;
-            }
-
-            // replace "->" with "."
-            else if (c1 == '-' && c2 == '>') {
-                tok->str(".");
-                tok->deleteNext();
-                continue;
-            }
-        }
-
-        else if (tok->str() == ">>" && tok->next()->str() == "=") {
-            tok->str(">>=");
-            tok->deleteNext();
-        }
-
-        else if (tok->str() == "<<" && tok->next()->str() == "=") {
-            tok->str("<<=");
-            tok->deleteNext();
-        }
-
-        else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
-            if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
-                tok->str(tok->str() + ":");
-                tok->deleteNext();
-                continue;
-            }
-        }
-    }
+    combineOperators();
 
     simplifyRedundantParentheses();
     for (Token *tok = list.front();
@@ -2210,6 +2063,99 @@ bool Tokenizer::hasEnumsWithTypedef()
     return false;
 }
 
+void Tokenizer::fillTypeSizes()
+{
+    _typeSize.clear();
+    _typeSize["char"] = 1;
+    _typeSize["bool"] = _settings->sizeof_bool;
+    _typeSize["short"] = _settings->sizeof_short;
+    _typeSize["int"] = _settings->sizeof_int;
+    _typeSize["long"] = _settings->sizeof_long;
+    _typeSize["float"] = _settings->sizeof_float;
+    _typeSize["double"] = _settings->sizeof_double;
+    _typeSize["wchar_t"] = _settings->sizeof_wchar_t;
+    _typeSize["size_t"] = _settings->sizeof_size_t;
+    _typeSize["*"] = _settings->sizeof_pointer;
+}
+
+void Tokenizer::combineOperators()
+{
+    // Combine tokens..
+    for (Token *tok = list.front();
+         tok && tok->next();
+         tok = tok->next()) {
+        const char c1 = tok->str()[0];
+
+        if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
+            const char c2 = tok->next()->str()[0];
+
+            // combine +-*/ and =
+            if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) {
+                tok->str(tok->str() + c2);
+                tok->deleteNext();
+                continue;
+            }
+
+            // replace "->" with "."
+            else if (c1 == '-' && c2 == '>') {
+                tok->str(".");
+                tok->deleteNext();
+                continue;
+            }
+        }
+
+        else if (tok->str() == ">>" && tok->next()->str() == "=") {
+            tok->str(">>=");
+            tok->deleteNext();
+        }
+
+        else if (tok->str() == "<<" && tok->next()->str() == "=") {
+            tok->str("<<=");
+            tok->deleteNext();
+        }
+
+        else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
+            if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
+                tok->str(tok->str() + ":");
+                tok->deleteNext();
+                continue;
+            }
+        }
+    }
+}
+
+void Tokenizer::combineStrings()
+{
+    // Combine wide strings
+    for (Token *tok = list.front();
+         tok;
+         tok = tok->next()) {
+        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
+            // Combine 'L "string"'
+            tok->str(tok->next()->str());
+            tok->deleteNext();
+            tok->isLong(true);
+        }
+    }
+
+    // Combine strings
+    for (Token *tok = list.front();
+         tok;
+         tok = tok->next()) {
+        if (tok->str()[0] != '"')
+            continue;
+
+        tok->str(simplifyString(tok->str()));
+        while (tok->next() && tok->next()->type() == Token::eString) {
+            tok->next()->str(simplifyString(tok->next()->str()));
+
+            // Two strings after each other, combine them
+            tok->concatStr(tok->next()->str());
+            tok->deleteNext();
+        }
+    }
+}
+
 void Tokenizer::concatenateDoubleSharp()
 {
     for (Token *tok = list.front(); tok; tok = tok->next()) {
diff --git a/lib/tokenize.h b/lib/tokenize.h
index d03d61580..fce4aedfc 100644
--- a/lib/tokenize.h
+++ b/lib/tokenize.h
@@ -441,6 +441,12 @@ public:
 
     void simplifyParameterVoid();
 
+    void fillTypeSizes();
+
+    void combineOperators();
+
+    void combineStrings();
+
     void concatenateDoubleSharp();
 
     void simplifyFileAndLineMacro();