From 88abb32ddf65602eb9e9de38002acb8f0236d581 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= <danielm77@spray.se>
Date: Sun, 9 Jan 2011 10:09:54 +0100
Subject: [PATCH] Fixed #2429 (Tokenizer: Wrong simplification of 'sizeof
 .1250E+04')

---
 lib/tokenize.cpp      | 5 +++++
 test/testtokenize.cpp | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
index 6c963dbec..d00e70bcc 100644
--- a/lib/tokenize.cpp
+++ b/lib/tokenize.cpp
@@ -361,6 +361,11 @@ void Tokenizer::createTokens(std::istream &code)
             {
                 // Don't separate doubles "4.2e+10"
             }
+            else if (CurrentToken.empty() && ch == '.' && std::isdigit(code.peek()))
+            {
+                // tokenize .125 into 0.125
+                CurrentToken = "0";
+            }
             else if (ch=='&' && CurrentToken.empty() && code.peek() == '&')
             {
                 // &&
diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp
index 18db0f6fb..590d7ebb9 100644
--- a/test/testtokenize.cpp
+++ b/test/testtokenize.cpp
@@ -52,6 +52,7 @@ private:
         TEST_CASE(tokenize12);
         TEST_CASE(tokenize13);  // bailout if the code contains "@" - that is not handled well.
         TEST_CASE(tokenize14);  // tokenize "0X10" => 16
+        TEST_CASE(tokenize15);  // tokenize ".123"
 
         // don't freak out when the syntax is wrong
         TEST_CASE(wrong_syntax);
@@ -490,6 +491,12 @@ private:
         ASSERT_EQUALS("; 16 ;", tokenizeAndStringify(";0x10;"));
         ASSERT_EQUALS("; 16 ;", tokenizeAndStringify(";0X10;"));
     }
+    
+    // Ticket #2429: 0.125
+    void tokenize15()
+    {
+        ASSERT_EQUALS("0.125", tokenizeAndStringify(".125"));
+    }
 
     void wrong_syntax()
     {