From 127a910516c6560674b9429b6b2ed1f8da66f6e4 Mon Sep 17 00:00:00 2001
From: Leandro Penz <lpenz@users.sourceforge.net>
Date: Fri, 13 Feb 2009 13:33:12 +0000
Subject: [PATCH] Tokenizer: fixed ## tokenization.

---
 src/tokenize.cpp      |  9 +++++++++
 test/testtokenize.cpp | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/src/tokenize.cpp b/src/tokenize.cpp
index 13e8f11ba..a9fee5e40 100644
--- a/src/tokenize.cpp
+++ b/src/tokenize.cpp
@@ -249,11 +249,18 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
             std::string line("#");
             {
                 char chPrev = '#';
+                bool skip = false;
                 while (code.good())
                 {
                     ch = (char)code.get();
                     if (chPrev != '\\' && ch == '\n')
                         break;
+                    if (chPrev == '#' && ch == '#')
+                    {
+                        addtoken("##", lineno, FileIndex);
+                        skip = true;
+                        break;
+                    }
                     if (ch != ' ')
                         chPrev = ch;
                     if (ch != '\\' && ch != '\n')
@@ -263,6 +270,8 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
                     if (ch == '\n')
                         ++lineno;
                 }
+                if (skip)
+                    continue;
             }
             if (strncmp(line.c_str(), "#file", 5) == 0 &&
                 line.find("\"") != std::string::npos)
diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp
index cc477594a..5d26d79f7 100644
--- a/test/testtokenize.cpp
+++ b/test/testtokenize.cpp
@@ -99,6 +99,8 @@ private:
 
         TEST_CASE(doublesharp);
 
+        TEST_CASE(macrodoublesharp);
+
         TEST_CASE(simplify_function_parameters);
 
         TEST_CASE(reduce_redundant_paranthesis);        // Ticket #61
@@ -801,6 +803,23 @@ private:
         ASSERT_EQUALS("TEST ( var , val ) var ## _ ## val = val ", ostr.str());
     }
 
+    void macrodoublesharp()
+    {
+        const char code[] = "DBG(fmt,args...) printf(fmt, ## args)\n";
+
+        // Tokenize..
+        Tokenizer tokenizer;
+        std::istringstream istr(code);
+        tokenizer.tokenize(istr, "");
+
+        // Stringify the tokens..
+        std::ostringstream ostr;
+        for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next())
+            ostr << tok->str() << " ";
+
+        ASSERT_EQUALS("DBG ( fmt , args . . . ) printf ( fmt , ## args ) ", ostr.str());
+    }
+
     void simplify_function_parameters()
     {
         {