walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

tokenizer: The tokenizer shouldn't handle comments nor preprocessor directives. The preprocessor will take care of 

that
This commit is contained in:
Daniel Marjamäki 2009-01-18 08:38:25 +00:00
parent 86c95c27d2
commit a269cfc8d9
2 changed files with 1 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
src/tokenize.cpp
View File

@ -257,101 +257,6 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
if (ch < 0) if (ch < 0)
continue; continue;
// Preprocessor stuff?
if (ch == '#' && CurrentToken.empty())
{
std::string line("#");
{
char chPrev = '#';
while (code.good())
{
ch = (char)code.get();
if (chPrev != '\\' && ch == '\n')
break;
if (ch != ' ')
chPrev = ch;
if (ch != '\\' && ch != '\n')
line += ch;
if (ch == '\n')
++lineno;
}
}
if (strncmp(line.c_str(), "#include", 8) == 0 &&
line.find("\"") != std::string::npos)
{
// Extract the filename
line.erase(0, line.find("\"") + 1);
if (line.find("\"") != std::string::npos)
line.erase(line.find("\""));
// Relative path..
if (_files.back().find_first_of("\\/") != std::string::npos)
{
std::string path = _files.back();
path.erase(1 + path.find_last_of("\\/"));
line = path + line;
}
addtoken("#include", lineno, FileIndex);
addtoken(line.c_str(), lineno, FileIndex);
std::ifstream fin(line.c_str());
tokenize(fin, line.c_str());
}
else if (strncmp(line.c_str(), "#define", 7) == 0)
{
std::string strId;
enum {Space1, Id, Space2, Value} State;
State = Space1;
for (unsigned int i = 8; i < line.length(); i++)
{
if (State == Space1 || State == Space2)
{
if (isspace(line[i]))
continue;
State = (State == Space1) ? Id : Value;
}
else if (State == Id)
{
if (isspace(line[i]))
{
strId = CurrentToken;
CurrentToken.clear();
State = Space2;
continue;
}
else if (! isalnum(line[i]))
{
break;
}
}
CurrentToken += line[i];
}
if (State == Value)
{
addtoken("def", lineno, FileIndex);
addtoken(strId.c_str(), lineno, FileIndex);
addtoken(";", lineno, FileIndex);
Define(strId.c_str(), CurrentToken.c_str());
}
CurrentToken.clear();
}
else
{
addtoken("#", lineno, FileIndex);
addtoken(";", lineno, FileIndex);
}
++lineno;
continue;
}
if (ch == '\n') if (ch == '\n')
{ {
// Add current token.. // Add current token..
@ -360,64 +265,6 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
continue; continue;
} }
// Comments..
if (ch == '/' && code.good())
{
bool newstatement = bool(strchr(";{}", CurrentToken.empty() ? '\0' : CurrentToken[0]) != NULL);
// Add current token..
addtoken(CurrentToken.c_str(), lineno, FileIndex);
CurrentToken.clear();
// Read next character..
ch = (char)code.get();
// If '//'..
if (ch == '/')
{
std::string comment;
getline(code, comment); // Parse in the whole comment
// If the comment says something like "fred is deleted" then generate appropriate tokens for that
comment = comment + " ";
if (newstatement && comment.find(" deleted ") != std::string::npos)
{
// delete
addtoken("delete", lineno, FileIndex);
// fred
std::string::size_type pos1 = comment.find_first_not_of(" \t");
std::string::size_type pos2 = comment.find(" ", pos1);
std::string firstWord = comment.substr(pos1, pos2 - pos1);
addtoken(firstWord.c_str(), lineno, FileIndex);
// ;
addtoken(";", lineno, FileIndex);
}
++lineno;
continue;
}
// If '/*'..
if (ch == '*')
{
char chPrev;
ch = chPrev = 'A';
while (code.good() && (chPrev != '*' || ch != '/'))
{
chPrev = ch;
ch = (char)code.get();
if (ch == '\n')
++lineno;
}
continue;
}
// Not a comment.. add token..
addtoken("/", lineno, FileIndex);
}
// char.. // char..
if (ch == '\'') if (ch == '\'')
{ {
@ -470,7 +317,7 @@ void Tokenizer::tokenizeCode(std::istream &code, const unsigned int FileIndex)
continue; continue;
} }
if (strchr("+-*/%&|^?!=<>[](){};:,.~", ch)) if (strchr("#+-*/%&|^?!=<>[](){};:,.~", ch))
{ {
addtoken(CurrentToken.c_str(), lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
CurrentToken.clear(); CurrentToken.clear();

25
test/testtokenize.cpp
View File

@ -37,7 +37,6 @@ private:
void run() void run()
{ {
TEST_CASE(multiline);
TEST_CASE(longtok); TEST_CASE(longtok);
TEST_CASE(inlineasm); TEST_CASE(inlineasm);
@ -82,30 +81,6 @@ private:
} }
void multiline()
{
const char filedata[] = "#define str \"abc\" \\\n"
" \"def\"\n";
// tokenize..
Tokenizer tokenizer;
std::istringstream istr(filedata);
tokenizer.tokenize(istr, "test.cpp");
// Expected result..
const char *expected[] =
{
"def",
"str",
";",
0
};
// Compare..
ASSERT_EQUALS(true, cmptok(expected, tokenizer.tokens()));
}
void longtok() void longtok()
{ {
std::string filedata(10000, 'a'); std::string filedata(10000, 'a');