Refactoring TOKEN: Changed from struct to class
This commit is contained in:
parent
69a3595f19
commit
c220b061dc
|
@ -78,7 +78,7 @@ static void CheckBufferOverrun_CheckScope( const TOKEN *tok, const char *varname
|
||||||
int indentlevel = 0;
|
int indentlevel = 0;
|
||||||
for ( ; tok; tok = tok->next )
|
for ( ; tok; tok = tok->next )
|
||||||
{
|
{
|
||||||
if (tok->str[0]=='{')
|
if (tok->str[0] == '{')
|
||||||
{
|
{
|
||||||
indentlevel++;
|
indentlevel++;
|
||||||
}
|
}
|
||||||
|
|
|
@ -195,8 +195,8 @@ static void MemoryLeak( const TOKEN *tok, const char varname[] )
|
||||||
static void instoken(TOKEN *tok, const char str[])
|
static void instoken(TOKEN *tok, const char str[])
|
||||||
{
|
{
|
||||||
TOKEN *newtok = new TOKEN;
|
TOKEN *newtok = new TOKEN;
|
||||||
memcpy( newtok, tok, sizeof(TOKEN) );
|
newtok->setstr(str);
|
||||||
newtok->str = _strdup(str);
|
newtok->next = tok->next;
|
||||||
tok->next = newtok;
|
tok->next = newtok;
|
||||||
}
|
}
|
||||||
//---------------------------------------------------------------------------
|
//---------------------------------------------------------------------------
|
||||||
|
@ -220,7 +220,7 @@ static TOKEN *getcode(const TOKEN *tok, const char varname[])
|
||||||
#define addtoken(_str) \
|
#define addtoken(_str) \
|
||||||
{ \
|
{ \
|
||||||
TOKEN *newtok = new TOKEN; \
|
TOKEN *newtok = new TOKEN; \
|
||||||
newtok->str = _strdup(_str); \
|
newtok->setstr(_str); \
|
||||||
newtok->linenr = tok->linenr; \
|
newtok->linenr = tok->linenr; \
|
||||||
newtok->FileIndex = tok->FileIndex; \
|
newtok->FileIndex = tok->FileIndex; \
|
||||||
newtok->next = 0; \
|
newtok->next = 0; \
|
||||||
|
@ -381,7 +381,6 @@ static void erase(TOKEN *begin, const TOKEN *end)
|
||||||
{
|
{
|
||||||
TOKEN *next = begin->next;
|
TOKEN *next = begin->next;
|
||||||
begin->next = begin->next->next;
|
begin->next = begin->next->next;
|
||||||
free(next->str);
|
|
||||||
delete next;
|
delete next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -406,7 +405,7 @@ static void CheckMemoryLeak_CheckScope( const TOKEN *Tok1, const char varname[]
|
||||||
{
|
{
|
||||||
done = true;
|
done = true;
|
||||||
|
|
||||||
for (TOKEN *tok2 = tok ; tok2; tok2 = tok2->next )
|
for (TOKEN *tok2 = tok; tok2; tok2 = tok2 ? tok2->next : NULL )
|
||||||
{
|
{
|
||||||
// Delete extra ";"
|
// Delete extra ";"
|
||||||
while (Match(tok2,"[;{}] ;"))
|
while (Match(tok2,"[;{}] ;"))
|
||||||
|
@ -418,7 +417,7 @@ static void CheckMemoryLeak_CheckScope( const TOKEN *Tok1, const char varname[]
|
||||||
// Replace "{ }" with ";"
|
// Replace "{ }" with ";"
|
||||||
if ( Match(tok2->next, "{ }") )
|
if ( Match(tok2->next, "{ }") )
|
||||||
{
|
{
|
||||||
tok2->next->str[0] = ';';
|
tok2->next->setstr(";");
|
||||||
erase(tok2->next, gettok(tok2,3));
|
erase(tok2->next, gettok(tok2,3));
|
||||||
done = false;
|
done = false;
|
||||||
}
|
}
|
||||||
|
@ -548,16 +547,14 @@ static void CheckMemoryLeak_CheckScope( const TOKEN *Tok1, const char varname[]
|
||||||
if ( !incase && valid )
|
if ( !incase && valid )
|
||||||
{
|
{
|
||||||
done = false;
|
done = false;
|
||||||
free(tok2->str);
|
tok2->setstr(";");
|
||||||
tok2->str = _strdup(";");
|
|
||||||
erase( tok2, gettok(tok2, 2) );
|
erase( tok2, gettok(tok2, 2) );
|
||||||
tok2 = tok2->next;
|
tok2 = tok2->next;
|
||||||
bool first = true;
|
bool first = true;
|
||||||
while (Match(tok2,"case") || Match(tok2,"default"))
|
while (Match(tok2,"case") || Match(tok2,"default"))
|
||||||
{
|
{
|
||||||
bool def = Match(tok2, "default");
|
bool def = Match(tok2, "default");
|
||||||
free(tok2->str);
|
tok2->setstr(first ? "if" : "}");
|
||||||
tok2->str = _strdup(first ? "if" : "}");
|
|
||||||
if ( first )
|
if ( first )
|
||||||
{
|
{
|
||||||
first = false;
|
first = false;
|
||||||
|
@ -575,14 +572,12 @@ static void CheckMemoryLeak_CheckScope( const TOKEN *Tok1, const char varname[]
|
||||||
tok2 = tok2->next;
|
tok2 = tok2->next;
|
||||||
if (Match(tok2,"break ;"))
|
if (Match(tok2,"break ;"))
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
tok2->setstr(";");
|
||||||
tok2->str = _strdup(";");
|
|
||||||
tok2 = tok2->next->next;
|
tok2 = tok2->next->next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -413,7 +413,6 @@ void deleteTokens(TOKEN *tok)
|
||||||
while (tok)
|
while (tok)
|
||||||
{
|
{
|
||||||
TOKEN *next = tok->next;
|
TOKEN *next = tok->next;
|
||||||
free(tok->str);
|
|
||||||
delete tok;
|
delete tok;
|
||||||
tok = next;
|
tok = next;
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,7 @@
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
struct TOKEN;
|
class TOKEN;
|
||||||
|
|
||||||
std::string FileLine(const TOKEN *tok);
|
std::string FileLine(const TOKEN *tok);
|
||||||
|
|
||||||
|
|
64
tokenize.cpp
64
tokenize.cpp
|
@ -68,7 +68,7 @@ static TOKEN *_gettok(TOKEN *tok, int index)
|
||||||
//---------------------------------------------------------------------------
|
//---------------------------------------------------------------------------
|
||||||
|
|
||||||
std::vector<std::string> Files;
|
std::vector<std::string> Files;
|
||||||
struct TOKEN *tokens, *tokens_back;
|
TOKEN *tokens, *tokens_back;
|
||||||
|
|
||||||
//---------------------------------------------------------------------------
|
//---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -161,8 +161,7 @@ static void addtoken(const char str[], const unsigned int lineno, const unsigned
|
||||||
}
|
}
|
||||||
|
|
||||||
TOKEN *newtoken = new TOKEN;
|
TOKEN *newtoken = new TOKEN;
|
||||||
memset(newtoken, 0, sizeof(TOKEN));
|
newtoken->setstr(str2.str().c_str());
|
||||||
newtoken->str = _strdup(str2.str().c_str());
|
|
||||||
newtoken->linenr = lineno;
|
newtoken->linenr = lineno;
|
||||||
newtoken->FileIndex = fileno;
|
newtoken->FileIndex = fileno;
|
||||||
if (tokens_back)
|
if (tokens_back)
|
||||||
|
@ -180,8 +179,7 @@ static void addtoken(const char str[], const unsigned int lineno, const unsigned
|
||||||
{
|
{
|
||||||
if (strcmp(str,sym->name)==0)
|
if (strcmp(str,sym->name)==0)
|
||||||
{
|
{
|
||||||
free(newtoken->str);
|
newtoken->setstr(sym->value);
|
||||||
newtoken->str = _strdup(sym->value);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -206,9 +204,8 @@ static void combine_2tokens(TOKEN *tok, const char str1[], const char str2[])
|
||||||
if (strcmp(tok->str,str1) || strcmp(tok->next->str,str2))
|
if (strcmp(tok->str,str1) || strcmp(tok->next->str,str2))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
free(tok->str);
|
|
||||||
std::string newstr(std::string(str1) + std::string(str2));
|
std::string newstr(std::string(str1) + std::string(str2));
|
||||||
tok->str = _strdup( newstr.c_str() );
|
tok->setstr( newstr.c_str() );
|
||||||
|
|
||||||
DeleteNextToken(tok);
|
DeleteNextToken(tok);
|
||||||
}
|
}
|
||||||
|
@ -245,7 +242,6 @@ static void DeleteNextToken(TOKEN *tok)
|
||||||
{
|
{
|
||||||
TOKEN *next = tok->next;
|
TOKEN *next = tok->next;
|
||||||
tok->next = next->next;
|
tok->next = next->next;
|
||||||
free(next->str);
|
|
||||||
delete next;
|
delete next;
|
||||||
}
|
}
|
||||||
//---------------------------------------------------------------------------
|
//---------------------------------------------------------------------------
|
||||||
|
@ -266,7 +262,7 @@ static void InsertTokens(TOKEN *dest, TOKEN *src, unsigned int n)
|
||||||
TOKEN *NewToken = new TOKEN;
|
TOKEN *NewToken = new TOKEN;
|
||||||
NewToken->FileIndex = src->FileIndex;
|
NewToken->FileIndex = src->FileIndex;
|
||||||
NewToken->linenr = src->linenr;
|
NewToken->linenr = src->linenr;
|
||||||
NewToken->str = _strdup(src->str);
|
NewToken->setstr(src->str);
|
||||||
|
|
||||||
NewToken->next = dest->next;
|
NewToken->next = dest->next;
|
||||||
dest->next = NewToken;
|
dest->next = NewToken;
|
||||||
|
@ -587,8 +583,7 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
|
||||||
{
|
{
|
||||||
if ( strcmp(tok->str, "->") == 0 )
|
if ( strcmp(tok->str, "->") == 0 )
|
||||||
{
|
{
|
||||||
tok->str[0] = '.';
|
tok->setstr(".");
|
||||||
tok->str[1] = 0;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -603,8 +598,7 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
|
||||||
{
|
{
|
||||||
if (tok2->str!=type1 && tok2->str!=type2 && strcmp(tok2->str,type2)==0)
|
if (tok2->str!=type1 && tok2->str!=type2 && strcmp(tok2->str,type2)==0)
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
tok2->setstr(type1);
|
||||||
tok2->str = _strdup(type1);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -623,11 +617,10 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
|
||||||
{
|
{
|
||||||
if (tok2->str!=type3 && strcmp(tok2->str,type3)==0)
|
if (tok2->str!=type3 && strcmp(tok2->str,type3)==0)
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
tok2->setstr(type1);
|
||||||
tok2->str = _strdup(type1);
|
|
||||||
|
|
||||||
TOKEN *newtok = new TOKEN;
|
TOKEN *newtok = new TOKEN;
|
||||||
newtok->str = _strdup(type2);
|
newtok->setstr(type2);
|
||||||
newtok->FileIndex = tok2->FileIndex;
|
newtok->FileIndex = tok2->FileIndex;
|
||||||
newtok->linenr = tok2->linenr;
|
newtok->linenr = tok2->linenr;
|
||||||
newtok->next = tok2->next;
|
newtok->next = tok2->next;
|
||||||
|
@ -651,7 +644,6 @@ void TokenizeCode(std::istream &code, const unsigned int FileIndex)
|
||||||
// Unlink and delete tok->next
|
// Unlink and delete tok->next
|
||||||
TOKEN *next = tok->next;
|
TOKEN *next = tok->next;
|
||||||
tok->next = tok->next->next;
|
tok->next = tok->next->next;
|
||||||
free(next->str);
|
|
||||||
delete next;
|
delete next;
|
||||||
|
|
||||||
// break if this was the last token to delete..
|
// break if this was the last token to delete..
|
||||||
|
@ -699,8 +691,7 @@ void SimplifyTokenList()
|
||||||
{
|
{
|
||||||
if (strcmp(tok2->str,sym) == 0)
|
if (strcmp(tok2->str,sym) == 0)
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
tok2->setstr(num);
|
||||||
tok2->str = _strdup(num);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -737,11 +728,10 @@ void SimplifyTokenList()
|
||||||
|
|
||||||
if (Match(tok, "sizeof ( %type% * )"))
|
if (Match(tok, "sizeof ( %type% * )"))
|
||||||
{
|
{
|
||||||
free(tok->str);
|
std::ostringstream str;
|
||||||
std::ostringstream str;
|
|
||||||
// 'sizeof(type *)' has the same size as 'sizeof(char *)'
|
// 'sizeof(type *)' has the same size as 'sizeof(char *)'
|
||||||
str << sizeof(char *);
|
str << sizeof(char *);
|
||||||
tok->str = _strdup( str.str().c_str() );
|
tok->setstr( str.str().c_str() );
|
||||||
|
|
||||||
for (int i = 0; i < 4; i++)
|
for (int i = 0; i < 4; i++)
|
||||||
{
|
{
|
||||||
|
@ -755,10 +745,9 @@ void SimplifyTokenList()
|
||||||
int size = SizeOfType(type);
|
int size = SizeOfType(type);
|
||||||
if (size > 0)
|
if (size > 0)
|
||||||
{
|
{
|
||||||
free(tok->str);
|
std::ostringstream str;
|
||||||
std::ostringstream str;
|
str << size;
|
||||||
str << size;
|
tok->setstr( str.str().c_str() );
|
||||||
tok->str = _strdup( str.str().c_str() );
|
|
||||||
for (int i = 0; i < 3; i++)
|
for (int i = 0; i < 3; i++)
|
||||||
{
|
{
|
||||||
DeleteNextToken(tok);
|
DeleteNextToken(tok);
|
||||||
|
@ -802,10 +791,9 @@ void SimplifyTokenList()
|
||||||
{
|
{
|
||||||
if (strcmp(getstr(tok2,2), varname) == 0)
|
if (strcmp(getstr(tok2,2), varname) == 0)
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
std::ostringstream str;
|
||||||
std::ostringstream str;
|
|
||||||
str << total_size;
|
str << total_size;
|
||||||
tok2->str = _strdup(str.str().c_str());
|
tok2->setstr(str.str().c_str());
|
||||||
// Delete the other tokens..
|
// Delete the other tokens..
|
||||||
for (int i = 0; i < 3; i++)
|
for (int i = 0; i < 3; i++)
|
||||||
{
|
{
|
||||||
|
@ -857,10 +845,9 @@ void SimplifyTokenList()
|
||||||
case '/': i1 /= i2; break;
|
case '/': i1 /= i2; break;
|
||||||
}
|
}
|
||||||
tok = tok->next;
|
tok = tok->next;
|
||||||
free(tok->str);
|
std::ostringstream str;
|
||||||
std::ostringstream str;
|
|
||||||
str << i1;
|
str << i1;
|
||||||
tok->str = _strdup(str.str().c_str());
|
tok->setstr(str.str().c_str());
|
||||||
for (int i = 0; i < 2; i++)
|
for (int i = 0; i < 2; i++)
|
||||||
{
|
{
|
||||||
DeleteNextToken(tok);
|
DeleteNextToken(tok);
|
||||||
|
@ -891,8 +878,7 @@ void SimplifyTokenList()
|
||||||
for (int i = 0; i < 4; i++)
|
for (int i = 0; i < 4; i++)
|
||||||
{
|
{
|
||||||
tok = tok->next;
|
tok = tok->next;
|
||||||
free(tok->str);
|
tok->setstr(str[i]);
|
||||||
tok->str = _strdup(str[i]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
DeleteNextToken(tok);
|
DeleteNextToken(tok);
|
||||||
|
@ -976,8 +962,7 @@ void SimplifyTokenList()
|
||||||
{
|
{
|
||||||
if (tok2->str[0] == ',')
|
if (tok2->str[0] == ',')
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
tok2->setstr(";");
|
||||||
tok2->str = _strdup(";");
|
|
||||||
InsertTokens(tok2, type0, typelen);
|
InsertTokens(tok2, type0, typelen);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1007,14 +992,12 @@ void SimplifyTokenList()
|
||||||
if (VarTok->str[0]=='*')
|
if (VarTok->str[0]=='*')
|
||||||
VarTok = VarTok->next;
|
VarTok = VarTok->next;
|
||||||
InsertTokens(eq, VarTok, 2);
|
InsertTokens(eq, VarTok, 2);
|
||||||
free(eq->str);
|
eq->setstr(";");
|
||||||
eq->str = _strdup(";");
|
|
||||||
|
|
||||||
// "= x, " => "= x; type "
|
// "= x, " => "= x; type "
|
||||||
if (tok2->str[0] == ',')
|
if (tok2->str[0] == ',')
|
||||||
{
|
{
|
||||||
free(tok2->str);
|
tok2->setstr(";");
|
||||||
tok2->str = _strdup(";");
|
|
||||||
InsertTokens( tok2, type0, typelen );
|
InsertTokens( tok2, type0, typelen );
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
@ -1090,7 +1073,6 @@ void DeallocateTokens()
|
||||||
while (tokens)
|
while (tokens)
|
||||||
{
|
{
|
||||||
TOKEN *next = tokens->next;
|
TOKEN *next = tokens->next;
|
||||||
free(tokens->str);
|
|
||||||
delete tokens;
|
delete tokens;
|
||||||
tokens = next;
|
tokens = next;
|
||||||
}
|
}
|
||||||
|
|
22
tokenize.h
22
tokenize.h
|
@ -27,14 +27,28 @@
|
||||||
|
|
||||||
extern std::vector<std::string> Files;
|
extern std::vector<std::string> Files;
|
||||||
|
|
||||||
struct TOKEN
|
class TOKEN
|
||||||
{
|
{
|
||||||
|
private:
|
||||||
|
char * _str;
|
||||||
|
|
||||||
|
public:
|
||||||
|
TOKEN()
|
||||||
|
{ FileIndex = 0; _str = 0; linenr = 0; next = 0; }
|
||||||
|
|
||||||
|
~TOKEN()
|
||||||
|
{ free(_str); }
|
||||||
|
|
||||||
|
void setstr( const char s[] )
|
||||||
|
{ free(_str); _str = strdup(s); str = _str ? _str : ""; }
|
||||||
|
|
||||||
|
const char *str;
|
||||||
|
|
||||||
unsigned int FileIndex;
|
unsigned int FileIndex;
|
||||||
char *str;
|
|
||||||
unsigned int linenr;
|
unsigned int linenr;
|
||||||
struct TOKEN *next;
|
TOKEN *next;
|
||||||
};
|
};
|
||||||
extern struct TOKEN *tokens, *tokens_back;
|
extern TOKEN *tokens, *tokens_back;
|
||||||
|
|
||||||
|
|
||||||
void Tokenize(std::istream &code, const char FileName[]);
|
void Tokenize(std::istream &code, const char FileName[]);
|
||||||
|
|
Loading…
Reference in New Issue