tokenize: "~Base" should generate two tokens. "~" and "Base"
This commit is contained in:
parent
4493400c49
commit
93c04efc95
|
@ -454,7 +454,10 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
// Append token..
|
// Append token..
|
||||||
CurrentToken += c;
|
CurrentToken += c;
|
||||||
|
|
||||||
|
if ( c == '\n' )
|
||||||
|
++lineno;
|
||||||
|
|
||||||
// Special sequence '\.'
|
// Special sequence '\.'
|
||||||
if (special)
|
if (special)
|
||||||
|
@ -472,7 +475,7 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (strchr("+-*/%&|^?!=<>[](){};:,.",ch))
|
if (strchr("+-*/%&|^?!=<>[](){};:,.~",ch))
|
||||||
{
|
{
|
||||||
addtoken(CurrentToken.c_str(), lineno, FileIndex);
|
addtoken(CurrentToken.c_str(), lineno, FileIndex);
|
||||||
CurrentToken.clear();
|
CurrentToken.clear();
|
||||||
|
|
Loading…
Reference in New Issue