Tokenizer: fixed ## tokenization.

src/tokenize.cpp

@@ -249,11 +249,18 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
             std::string line("#");
             {
                 char chPrev = '#';
+                bool skip = false;
                 while (code.good())
                 {
                     ch = (char)code.get();
                     if (chPrev != '\\' && ch == '\n')
                         break;
+                    if (chPrev == '#' && ch == '#')
+                    {
+                        addtoken("##", lineno, FileIndex);
+                        skip = true;
+                        break;
+                    }
                     if (ch != ' ')
                         chPrev = ch;
                     if (ch != '\\' && ch != '\n')
@@ -263,6 +270,8 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
                     if (ch == '\n')
                         ++lineno;
                 }
+                if (skip)
+                    continue;
             }
             if (strncmp(line.c_str(), "#file", 5) == 0 &&
                 line.find("\"") != std::string::npos)

test/testtokenize.cpp

@@ -99,6 +99,8 @@ private:
 
         TEST_CASE(doublesharp);
 
+        TEST_CASE(macrodoublesharp);
+
         TEST_CASE(simplify_function_parameters);
 
         TEST_CASE(reduce_redundant_paranthesis);        // Ticket #61
@@ -801,6 +803,23 @@ private:
         ASSERT_EQUALS("TEST ( var , val ) var ## _ ## val = val ", ostr.str());
     }
 
+    void macrodoublesharp()
+    {
+        const char code[] = "DBG(fmt,args...) printf(fmt, ## args)\n";
+
+        // Tokenize..
+        Tokenizer tokenizer;
+        std::istringstream istr(code);
+        tokenizer.tokenize(istr, "");
+
+        // Stringify the tokens..
+        std::ostringstream ostr;
+        for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next())
+            ostr << tok->str() << " ";
+
+        ASSERT_EQUALS("DBG ( fmt , args . . . ) printf ( fmt , ## args ) ", ostr.str());
+    }
+
     void simplify_function_parameters()
     {
         {