tokenize: "~Base" should generate two tokens. "~" and "Base"

This commit is contained in:
Daniel Marjamäki 2008-12-06 17:53:30 +00:00
parent 4493400c49
commit 93c04efc95
1 changed files with 5 additions and 2 deletions

View File

@ -456,6 +456,9 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
// Append token..
CurrentToken += c;
if ( c == '\n' )
++lineno;
// Special sequence '\.'
if (special)
special = false;
@ -472,7 +475,7 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
continue;
}
if (strchr("+-*/%&|^?!=<>[](){};:,.",ch))
if (strchr("+-*/%&|^?!=<>[](){};:,.~",ch))
{
addtoken(CurrentToken.c_str(), lineno, FileIndex);
CurrentToken.clear();