tokenize: "~Base" should generate two tokens. "~" and "Base"
This commit is contained in:
parent
4493400c49
commit
93c04efc95
|
@ -454,7 +454,10 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
|
|||
do
|
||||
{
|
||||
// Append token..
|
||||
CurrentToken += c;
|
||||
CurrentToken += c;
|
||||
|
||||
if ( c == '\n' )
|
||||
++lineno;
|
||||
|
||||
// Special sequence '\.'
|
||||
if (special)
|
||||
|
@ -472,7 +475,7 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
|
|||
continue;
|
||||
}
|
||||
|
||||
if (strchr("+-*/%&|^?!=<>[](){};:,.",ch))
|
||||
if (strchr("+-*/%&|^?!=<>[](){};:,.~",ch))
|
||||
{
|
||||
addtoken(CurrentToken.c_str(), lineno, FileIndex);
|
||||
CurrentToken.clear();
|
||||
|
|
Loading…
Reference in New Issue