TokenList: added assertion that makes sure the Preprocessor doesn't let through extended ascii wrongly.

This commit is contained in:
Daniel Marjamäki 2014-05-31 18:02:19 +02:00
parent 98ea1397b3
commit 46bf23aa6d
1 changed files with 5 additions and 2 deletions

View File

@ -29,7 +29,7 @@
#include <sstream>
#include <cctype>
#include <stack>
#include <cassert>
// How many compileExpression recursions are allowed?
// For practical code this could be endless. But in some special torture test
@ -270,8 +270,11 @@ bool TokenList::createTokens(std::istream &code, const std::string& file0)
continue;
}
// Preprocessor should ensure code doesn't contain any extended ascii / utf / etc.
assert(CurrentToken.empty() || (CurrentToken[0] & 0x80) == 0);
if (ch == '.' &&
CurrentToken.length() > 0 &&
!CurrentToken.empty() &&
std::isdigit(CurrentToken[0])) {
// Don't separate doubles "5.4"
} else if (std::strchr("+-", ch) &&