tokenizer: tokenize ## better

src/tokenize.cpp

@@ -243,6 +243,13 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
 
         if (ch == '#' && CurrentToken.empty())
         {
+            // If previous token was "#" then append this to create a "##" token
+            if (Token::simpleMatch(_tokensBack, "#"))
+            {
+                _tokensBack->str("##");
+                continue;
+            }
+
             std::string line("#");
             {
                 char chPrev = '#';
@@ -254,7 +261,9 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
                     if (ch != ' ')
                         chPrev = ch;
                     if (ch != '\\' && ch != '\n')
+                    {
                         line += ch;
+                    }
                     if (ch == '\n')
                         ++lineno;
                 }

test/testtokenize.cpp

@@ -69,6 +69,8 @@ private:
 
         TEST_CASE(file1);
         // TODO TEST_CASE(file2);
+
+        TEST_CASE(doublesharp);
     }
 
 
@@ -687,6 +689,25 @@ private:
             ASSERT_EQUALS(tok->str(), ostr.str());
         }
     }
+
+
+
+    void doublesharp()
+    {
+        const char code[] = "TEST(var,val) var##_##val = val\n";
+
+        // Tokenize..
+        Tokenizer tokenizer;
+        std::istringstream istr(code);
+        tokenizer.tokenize(istr, "");
+
+        // Stringify the tokens..
+        std::ostringstream ostr;
+        for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next())
+            ostr << tok->str() << " ";
+
+        ASSERT_EQUALS("TEST ( var , val ) var ## _ ## val = val ", ostr.str());
+    }
 };
 
 REGISTER_TEST(TestTokenizer)