Cleanup: The Tokenizer doesn't need to handle UTF in code nor multiline string. That is handled in the Preprocessor.

lib/tokenize.cpp

@@ -230,11 +230,8 @@ void Tokenizer::createTokens(std::istream &code)
     // Read one byte at a time from code and create tokens
     for (char ch = (char)code.get(); code.good(); ch = (char)code.get())
     {
-        // We are not handling UTF and stuff like that. Code is supposed to plain simple text.
-        if (ch < 0)
-            continue;
-
         // char/string..
+        // multiline strings are not handled. The preprocessor should handle that for us.
         if (ch == '\'' || ch == '\"')
         {
             std::string line;
@@ -247,9 +244,6 @@ void Tokenizer::createTokens(std::istream &code)
                 // Append token..
                 line += c;
 
-                if (c == '\n')
-                    ++lineno;
-
                 // Special sequence '\.'
                 if (special)
                     special = false;

lib/tokenize.h

@@ -71,7 +71,11 @@ public:
 
     /**
      * Create tokens from code.
-     * @param code input stream for code, same as what tokenize()
+     * The code must be preprocessed first:
+     * - multiline strings are not handled.
+     * - UTF in the code are not handled.
+     * - comments are not handled.
+     * @param code input stream for code
      */
     void createTokens(std::istream &code);