Tokenizer: fixed ## tokenization.

This commit is contained in:
Leandro Penz 2009-02-13 13:33:12 +00:00
parent d511863b57
commit 127a910516
2 changed files with 28 additions and 0 deletions

View File

@ -249,11 +249,18 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
std::string line("#"); std::string line("#");
{ {
char chPrev = '#'; char chPrev = '#';
bool skip = false;
while (code.good()) while (code.good())
{ {
ch = (char)code.get(); ch = (char)code.get();
if (chPrev != '\\' && ch == '\n') if (chPrev != '\\' && ch == '\n')
break; break;
if (chPrev == '#' && ch == '#')
{
addtoken("##", lineno, FileIndex);
skip = true;
break;
}
if (ch != ' ') if (ch != ' ')
chPrev = ch; chPrev = ch;
if (ch != '\\' && ch != '\n') if (ch != '\\' && ch != '\n')
@ -263,6 +270,8 @@ void Tokenizer::tokenize(std::istream &code, const char FileName[])
if (ch == '\n') if (ch == '\n')
++lineno; ++lineno;
} }
if (skip)
continue;
} }
if (strncmp(line.c_str(), "#file", 5) == 0 && if (strncmp(line.c_str(), "#file", 5) == 0 &&
line.find("\"") != std::string::npos) line.find("\"") != std::string::npos)

View File

@ -99,6 +99,8 @@ private:
TEST_CASE(doublesharp); TEST_CASE(doublesharp);
TEST_CASE(macrodoublesharp);
TEST_CASE(simplify_function_parameters); TEST_CASE(simplify_function_parameters);
TEST_CASE(reduce_redundant_paranthesis); // Ticket #61 TEST_CASE(reduce_redundant_paranthesis); // Ticket #61
@ -801,6 +803,23 @@ private:
ASSERT_EQUALS("TEST ( var , val ) var ## _ ## val = val ", ostr.str()); ASSERT_EQUALS("TEST ( var , val ) var ## _ ## val = val ", ostr.str());
} }
void macrodoublesharp()
{
const char code[] = "DBG(fmt,args...) printf(fmt, ## args)\n";
// Tokenize..
Tokenizer tokenizer;
std::istringstream istr(code);
tokenizer.tokenize(istr, "");
// Stringify the tokens..
std::ostringstream ostr;
for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next())
ostr << tok->str() << " ";
ASSERT_EQUALS("DBG ( fmt , args . . . ) printf ( fmt , ## args ) ", ostr.str());
}
void simplify_function_parameters() void simplify_function_parameters()
{ {
{ {