From ff38bbf468fd0cd68bc85f2cb0c8969cd0c62cf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= <danielm77@spray.se>
Date: Fri, 2 Apr 2010 20:48:32 +0200
Subject: [PATCH] Cleanup: The Tokenizer doesn't need to handle UTF in code nor
 multiline string. That is handled in the Preprocessor.

---
 lib/tokenize.cpp | 8 +-------
 lib/tokenize.h   | 6 +++++-
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
index 2edae2696..b8d7a81f8 100644
--- a/lib/tokenize.cpp
+++ b/lib/tokenize.cpp
@@ -230,11 +230,8 @@ void Tokenizer::createTokens(std::istream &code)
     // Read one byte at a time from code and create tokens
     for (char ch = (char)code.get(); code.good(); ch = (char)code.get())
     {
-        // We are not handling UTF and stuff like that. Code is supposed to plain simple text.
-        if (ch < 0)
-            continue;
-
         // char/string..
+        // multiline strings are not handled. The preprocessor should handle that for us.
         if (ch == '\'' || ch == '\"')
         {
             std::string line;
@@ -247,9 +244,6 @@ void Tokenizer::createTokens(std::istream &code)
                 // Append token..
                 line += c;
 
-                if (c == '\n')
-                    ++lineno;
-
                 // Special sequence '\.'
                 if (special)
                     special = false;
diff --git a/lib/tokenize.h b/lib/tokenize.h
index 8664d0854..c38587464 100644
--- a/lib/tokenize.h
+++ b/lib/tokenize.h
@@ -71,7 +71,11 @@ public:
 
     /**
      * Create tokens from code.
-     * @param code input stream for code, same as what tokenize()
+     * The code must be preprocessed first:
+     * - multiline strings are not handled.
+     * - UTF in the code are not handled.
+     * - comments are not handled.
+     * @param code input stream for code
      */
     void createTokens(std::istream &code);