Moved simplification of wide character string literals (L"foo") to tokenize(). Token::isLong flag used to indicate that string is a wchar_t literal.

lib/tokenize.cpp

@@ -1600,6 +1600,31 @@ bool Tokenizer::tokenize(std::istream &code,
         }
     }
 
+    // Combine wide strings
+    for (Token *tok = list.front(); tok; tok = tok->next()) {
+        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
+            // Combine 'L "string"'
+            tok->str(tok->next()->str());
+            tok->deleteNext();
+            tok->isLong(true);
+        }
+    }
+
+    // Combine strings
+    for (Token *tok = list.front(); tok; tok = tok->next()) {
+        if (tok->str()[0] != '"')
+            continue;
+
+        tok->str(simplifyString(tok->str()));
+        while (tok->next() && tok->next()->type() == Token::eString) {
+            tok->next()->str(simplifyString(tok->next()->str()));
+
+            // Two strings after each other, combine them
+            tok->concatStr(tok->next()->str());
+            tok->deleteNext();
+        }
+    }
+
     // replace inline SQL with "asm()" (Oracle PRO*C). Ticket: #1959
     simplifySQL();
 
@@ -3243,30 +3268,6 @@ bool Tokenizer::simplifyTokenList()
 
     simplifyGoto();
 
-    // Combine wide strings
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
-            // Combine 'L "string"'
-            tok->str(tok->next()->str());
-            tok->deleteNext();
-        }
-    }
-
-    // Combine strings
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        if (tok->str()[0] != '"')
-            continue;
-
-        tok->str(simplifyString(tok->str()));
-        while (tok->next() && tok->next()->type() == Token::eString) {
-            tok->next()->str(simplifyString(tok->next()->str()));
-
-            // Two strings after each other, combine them
-            tok->concatStr(tok->next()->str());
-            tok->deleteNext();
-        }
-    }
-
     simplifySizeof();
 
     simplifyUndefinedSizeArray();

test/testsimplifytokens.cpp

@@ -613,7 +613,15 @@ private:
                               "\"hello world\"\n"
                               "};\n"
                               "}\n";
-        ASSERT_EQUALS(tok(code2), tok(code1));
+
+        Settings settings;
+        settings.platform(Settings::Unspecified);
+        Tokenizer tokenizer(&settings, this);
+        std::istringstream istr(code1);
+        tokenizer.tokenize(istr, "test.cpp");
+
+        ASSERT_EQUALS(tok(code2), tokenizer.tokens()->stringifyList(0, false));
+        ASSERT_EQUALS(true, tokenizer.tokens()->tokAt(13) && tokenizer.tokens()->tokAt(13)->isLong());
     }
 
     void double_plus() {