From c20adf91bf56ede26b48dfb75e5f8cfedff0126c Mon Sep 17 00:00:00 2001
From: PKEuS <philipp.kloke@web.de>
Date: Sat, 1 Sep 2012 13:12:47 +0200
Subject: [PATCH] Moved simplification of wide character string literals
 (L"foo") to tokenize(). Token::isLong flag used to indicate that string is a
 wchar_t literal.

---
 lib/tokenize.cpp            | 49 +++++++++++++++++++------------------
 test/testsimplifytokens.cpp | 10 +++++++-
 2 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
index d1fe942fe..946063404 100644
--- a/lib/tokenize.cpp
+++ b/lib/tokenize.cpp
@@ -1600,6 +1600,31 @@ bool Tokenizer::tokenize(std::istream &code,
         }
     }
 
+    // Combine wide strings
+    for (Token *tok = list.front(); tok; tok = tok->next()) {
+        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
+            // Combine 'L "string"'
+            tok->str(tok->next()->str());
+            tok->deleteNext();
+            tok->isLong(true);
+        }
+    }
+
+    // Combine strings
+    for (Token *tok = list.front(); tok; tok = tok->next()) {
+        if (tok->str()[0] != '"')
+            continue;
+
+        tok->str(simplifyString(tok->str()));
+        while (tok->next() && tok->next()->type() == Token::eString) {
+            tok->next()->str(simplifyString(tok->next()->str()));
+
+            // Two strings after each other, combine them
+            tok->concatStr(tok->next()->str());
+            tok->deleteNext();
+        }
+    }
+
     // replace inline SQL with "asm()" (Oracle PRO*C). Ticket: #1959
     simplifySQL();
 
@@ -3243,30 +3268,6 @@ bool Tokenizer::simplifyTokenList()
 
     simplifyGoto();
 
-    // Combine wide strings
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
-            // Combine 'L "string"'
-            tok->str(tok->next()->str());
-            tok->deleteNext();
-        }
-    }
-
-    // Combine strings
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        if (tok->str()[0] != '"')
-            continue;
-
-        tok->str(simplifyString(tok->str()));
-        while (tok->next() && tok->next()->type() == Token::eString) {
-            tok->next()->str(simplifyString(tok->next()->str()));
-
-            // Two strings after each other, combine them
-            tok->concatStr(tok->next()->str());
-            tok->deleteNext();
-        }
-    }
-
     simplifySizeof();
 
     simplifyUndefinedSizeArray();
diff --git a/test/testsimplifytokens.cpp b/test/testsimplifytokens.cpp
index c644390c7..296b3cd47 100644
--- a/test/testsimplifytokens.cpp
+++ b/test/testsimplifytokens.cpp
@@ -613,7 +613,15 @@ private:
                               "\"hello world\"\n"
                               "};\n"
                               "}\n";
-        ASSERT_EQUALS(tok(code2), tok(code1));
+
+        Settings settings;
+        settings.platform(Settings::Unspecified);
+        Tokenizer tokenizer(&settings, this);
+        std::istringstream istr(code1);
+        tokenizer.tokenize(istr, "test.cpp");
+
+        ASSERT_EQUALS(tok(code2), tokenizer.tokens()->stringifyList(0, false));
+        ASSERT_EQUALS(true, tokenizer.tokens()->tokAt(13) && tokenizer.tokens()->tokAt(13)->isLong());
     }
 
     void double_plus() {