Handle prefixed strings and characters in Token (#1742)

This makes it possible to call getStrLength() and similar functions
before the tokenizer is called.

lib/token.cpp

@@ -34,14 +34,17 @@
 #include <stack>
 #include <utility>
 
+static const std::string literal_prefix[4] = {"u8", "u", "U", "L"};
+
 static bool isStringCharLiteral(const std::string &str, char q)
 {
 
     if (!endsWith(str, q))
         return false;
+    if (str[0] == q && str.length() > 1)
+        return true;
 
-    const std::string prefix[5] = { "", "u8", "L", "U", "u" };
+    for (const std::string & p: literal_prefix) {
-    for (const std::string & p: prefix) {
         if ((str.length() + 1) > p.length() && (str.compare(0, p.size() + 1, (p + q)) == 0))
             return true;
     }
@@ -131,6 +134,7 @@ void Token::update_property_info()
         tokType(eNone);
     }
 
+    update_property_char_string_literal();
     update_property_isStandardType();
 }
 
@@ -160,6 +164,20 @@ void Token::update_property_isStandardType()
     }
 }
 
+void Token::update_property_char_string_literal()
+{
+    if (!(mTokType == Token::eString || mTokType == Token::eChar)) // Token has already been updated
+        return;
+
+    for (const std::string & p : literal_prefix) {
+        if (((mTokType == Token::eString) && mStr.compare(0, p.size() + 1, p + "\"") == 0) ||
+            ((mTokType == Token::eChar) && (mStr.compare(0, p.size() +  1, p + "\'") == 0))) {
+            mStr = mStr.substr(p.size());
+            isLong(p != "u8");
+            break;
+        }
+    }
+}
 
 bool Token::isUpperCaseName() const
 {

lib/token.h

@@ -1069,6 +1069,9 @@ private:
     /** Update internal property cache about isStandardType() */
     void update_property_isStandardType();
 
+    /** Update internal property cache about string and char literals */
+    void update_property_char_string_literal();
+
 public:
     void astOperand1(Token *tok);
     void astOperand2(Token *tok);

lib/tokenize.cpp

@@ -1928,18 +1928,6 @@ void Tokenizer::combineOperators()
 
 void Tokenizer::combineStringAndCharLiterals()
 {
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        const std::string prefix[4] = {"u8", "L", "U", "u"};
-        for (const std::string & p : prefix) {
-            if (((tok->tokType() == Token::eString) && (tok->str().find(p + "\"") == 0)) ||
-                ((tok->tokType() == Token::eChar) && (tok->str().find(p + "\'") == 0))) {
-                tok->str(tok->str().substr(p.size()));
-                tok->isLong(p != "u8");
-                break;
-            }
-        }
-    }
-
     // Combine strings
     for (Token *tok = list.front();
          tok;

test/testtoken.cpp

@@ -267,16 +267,28 @@ private:
         Token tok;
 
         tok.str("\"\"");
-        ASSERT_EQUALS(0, (int)Token::getStrLength(&tok));
+        ASSERT_EQUALS(0, Token::getStrLength(&tok));
 
         tok.str("\"test\"");
-        ASSERT_EQUALS(4, (int)Token::getStrLength(&tok));
+        ASSERT_EQUALS(4, Token::getStrLength(&tok));
 
         tok.str("\"test \\\\test\"");
-        ASSERT_EQUALS(10, (int)Token::getStrLength(&tok));
+        ASSERT_EQUALS(10, Token::getStrLength(&tok));
 
         tok.str("\"a\\0\"");
-        ASSERT_EQUALS(1, (int)Token::getStrLength(&tok));
+        ASSERT_EQUALS(1, Token::getStrLength(&tok));
+
+        tok.str("L\"\"");
+        ASSERT_EQUALS(0, Token::getStrLength(&tok));
+
+        tok.str("u8\"test\"");
+        ASSERT_EQUALS(4, Token::getStrLength(&tok));
+
+        tok.str("U\"test \\\\test\"");
+        ASSERT_EQUALS(10, Token::getStrLength(&tok));
+
+        tok.str("u\"a\\0\"");
+        ASSERT_EQUALS(1, Token::getStrLength(&tok));
     }
 
     void getStrSize() const {