walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Tokenize: Changed the array CurrentToken to a std::string

This commit is contained in:
Daniel Marjamäki 2008-11-04 19:09:31 +00:00
parent 80ce75216d
commit 5e4a7fefe5
2 changed files with 55 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
testtokenize.cpp
View File

@ -35,6 +35,7 @@ private:
void run() void run()
{ {
TEST_CASE( multiline ); TEST_CASE( multiline );
TEST_CASE( longtok );
} }
@ -75,6 +76,22 @@ private:
DeallocateTokens(); DeallocateTokens();
} }
void longtok()
{
std::string filedata(10000,'a');
// tokenize..
tokens = tokens_back = NULL;
std::istringstream istr(filedata);
TokenizeCode(istr, 0);
// Expected result..
ASSERT_EQUALS( std::string(10000,'a'), std::string(tokens->str) );
DeallocateTokens();
}
}; };
REGISTER_TEST( TestTokenizer ) REGISTER_TEST( TestTokenizer )

92
tokenize.cpp
View File

@ -1,4 +1,4 @@
/* /*
* c++check - c/c++ syntax checking * c++check - c/c++ syntax checking
* Copyright (C) 2007 Daniel Marjamäki * Copyright (C) 2007 Daniel Marjamäki
* *
@ -315,8 +315,7 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
{ {
// Tokenize the file. // Tokenize the file.
unsigned int lineno = 1; unsigned int lineno = 1;
char CurrentToken[1000] = {0}; std::string CurrentToken;
char *pToken = CurrentToken;
for (char ch = (char)code.get(); code.good(); ch = (char)code.get()) for (char ch = (char)code.get(); code.good(); ch = (char)code.get())
{ {
// Todo // Todo
@ -324,7 +323,7 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
continue; continue;
// Preprocessor stuff? // Preprocessor stuff?
if (ch == '#' && !CurrentToken[0]) if (ch == '#' && CurrentToken.empty())
{ {
std::string line("#"); std::string line("#");
{ {
@ -366,7 +365,7 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
else if (strncmp(line.c_str(), "#define", 7) == 0) else if (strncmp(line.c_str(), "#define", 7) == 0)
{ {
char *strId = NULL; std::string strId;
enum {Space1, Id, Space2, Value} State; enum {Space1, Id, Space2, Value} State;
State = Space1; State = Space1;
for (unsigned int i = 8; i < line.length(); i++) for (unsigned int i = 8; i < line.length(); i++)
@ -382,9 +381,8 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
{ {
if ( isspace( line[i] ) ) if ( isspace( line[i] ) )
{ {
strId = _strdup(CurrentToken); strId = CurrentToken;
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
State = Space2; State = Space2;
continue; continue;
} }
@ -394,21 +392,18 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
} }
} }
*pToken = line[i]; CurrentToken += line[i];
pToken++;
} }
if (State==Value) if (State==Value)
{ {
addtoken("def", lineno, FileIndex); addtoken("def", lineno, FileIndex);
addtoken(strId, lineno, FileIndex); addtoken(strId.c_str(), lineno, FileIndex);
addtoken(";", lineno, FileIndex); addtoken(";", lineno, FileIndex);
Define(strId, CurrentToken); Define(strId.c_str(), CurrentToken.c_str());
} }
pToken = CurrentToken; CurrentToken.clear();
memset(CurrentToken, 0, sizeof(CurrentToken));
free(strId);
} }
else else
@ -424,21 +419,19 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
if (ch == '\n') if (ch == '\n')
{ {
// Add current token.. // Add current token..
addtoken(CurrentToken, lineno++, FileIndex); addtoken(CurrentToken.c_str(), lineno++, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
continue; continue;
} }
// Comments.. // Comments..
if (ch == '/' && code.good()) if (ch == '/' && code.good())
{ {
bool newstatement = bool( strchr(";{}", CurrentToken[0]) != NULL ); bool newstatement = bool( strchr(";{}", CurrentToken.empty() ? '\0' : CurrentToken[0]) != NULL );
// Add current token.. // Add current token..
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
// Read next character.. // Read next character..
ch = (char)code.get(); ch = (char)code.get();
@ -493,20 +486,19 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
if (ch == '\'') if (ch == '\'')
{ {
// Add previous token // Add previous token
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
// Read this .. // Read this ..
CurrentToken[0] = ch; CurrentToken += ch;
CurrentToken[1] = (char)code.get(); CurrentToken += (char)code.get();
CurrentToken[2] = (char)code.get(); CurrentToken += (char)code.get();
if (CurrentToken[1] == '\\') if (CurrentToken[1] == '\\')
CurrentToken[3] = (char)code.get(); CurrentToken += (char)code.get();
// Add token and start on next.. // Add token and start on next..
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
continue; continue;
} }
@ -514,19 +506,14 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
// String.. // String..
if (ch == '\"') if (ch == '\"')
{ {
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
bool special = false; bool special = false;
char c = ch; char c = ch;
do do
{ {
// Append token.. // Append token..
if ( pToken < &CurrentToken[sizeof(CurrentToken)-10] ) CurrentToken += c;
{
*pToken = c;
pToken++;
}
// Special sequence '\.' // Special sequence '\.'
if (special) if (special)
@ -538,36 +525,33 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
c = (char)code.get(); c = (char)code.get();
} }
while (code.good() && (special || c != '\"')); while (code.good() && (special || c != '\"'));
*pToken = '\"'; CurrentToken += '\"';
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
continue; continue;
} }
if (strchr("+-*/%&|^?!=<>[](){};:,.",ch)) if (strchr("+-*/%&|^?!=<>[](){};:,.",ch))
{ {
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
CurrentToken[0] = ch; CurrentToken += ch;
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
memset(CurrentToken, 0, sizeof(CurrentToken)); CurrentToken.clear();
pToken = CurrentToken;
continue; continue;
} }
if (isspace(ch) || iscntrl(ch)) if (isspace(ch) || iscntrl(ch))
{ {
addtoken(CurrentToken, lineno, FileIndex); addtoken(CurrentToken.c_str(), lineno, FileIndex);
pToken = CurrentToken; CurrentToken.clear();
memset(CurrentToken, 0, sizeof(CurrentToken));
continue; continue;
} }
*pToken = ch; CurrentToken += ch;
pToken++;
} }
addtoken( CurrentToken.c_str(), lineno, FileIndex );
// Combine tokens.. // Combine tokens..
for (TOKEN *tok = tokens; tok && tok->next; tok = tok->next) for (TOKEN *tok = tokens; tok && tok->next; tok = tok->next)