TokenList: added assertion that makes sure the Preprocessor doesn't let through extended ascii wrongly.
This commit is contained in:
parent
98ea1397b3
commit
46bf23aa6d
|
@ -29,7 +29,7 @@
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <cctype>
|
#include <cctype>
|
||||||
#include <stack>
|
#include <stack>
|
||||||
|
#include <cassert>
|
||||||
|
|
||||||
// How many compileExpression recursions are allowed?
|
// How many compileExpression recursions are allowed?
|
||||||
// For practical code this could be endless. But in some special torture test
|
// For practical code this could be endless. But in some special torture test
|
||||||
|
@ -270,8 +270,11 @@ bool TokenList::createTokens(std::istream &code, const std::string& file0)
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Preprocessor should ensure code doesn't contain any extended ascii / utf / etc.
|
||||||
|
assert(CurrentToken.empty() || (CurrentToken[0] & 0x80) == 0);
|
||||||
|
|
||||||
if (ch == '.' &&
|
if (ch == '.' &&
|
||||||
CurrentToken.length() > 0 &&
|
!CurrentToken.empty() &&
|
||||||
std::isdigit(CurrentToken[0])) {
|
std::isdigit(CurrentToken[0])) {
|
||||||
// Don't separate doubles "5.4"
|
// Don't separate doubles "5.4"
|
||||||
} else if (std::strchr("+-", ch) &&
|
} else if (std::strchr("+-", ch) &&
|
||||||
|
|
Loading…
Reference in New Issue