From 46bf23aa6db39b9a4c93ac78b415d34ffdcc7bc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= <daniel.marjamaki@gmail.com>
Date: Sat, 31 May 2014 18:02:19 +0200
Subject: [PATCH] TokenList: added assertion that makes sure the Preprocessor
 doesn't let through extended ascii wrongly.

---
 lib/tokenlist.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp
index cf6974c0f..79df478d3 100644
--- a/lib/tokenlist.cpp
+++ b/lib/tokenlist.cpp
@@ -29,7 +29,7 @@
 #include <sstream>
 #include <cctype>
 #include <stack>
-
+#include <cassert>
 
 // How many compileExpression recursions are allowed?
 // For practical code this could be endless. But in some special torture test
@@ -270,8 +270,11 @@ bool TokenList::createTokens(std::istream &code, const std::string& file0)
             continue;
         }
 
+        // Preprocessor should ensure code doesn't contain any extended ascii / utf / etc.
+        assert(CurrentToken.empty() || (CurrentToken[0] & 0x80) == 0);
+
         if (ch == '.' &&
-            CurrentToken.length() > 0 &&
+            !CurrentToken.empty() &&
             std::isdigit(CurrentToken[0])) {
             // Don't separate doubles "5.4"
         } else if (std::strchr("+-", ch) &&