tokenize: "~Base" should generate two tokens. "~" and "Base"

This commit is contained in:
Daniel Marjamäki 2008-12-06 17:53:30 +00:00
parent 4493400c49
commit 93c04efc95
1 changed files with 5 additions and 2 deletions

View File

@ -454,7 +454,10 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
do
{
// Append token..
CurrentToken += c;
CurrentToken += c;
if ( c == '\n' )
++lineno;
// Special sequence '\.'
if (special)
@ -472,7 +475,7 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
continue;
}
if (strchr("+-*/%&|^?!=<>[](){};:,.",ch))
if (strchr("+-*/%&|^?!=<>[](){};:,.~",ch))
{
addtoken(CurrentToken.c_str(), lineno, FileIndex);
CurrentToken.clear();