walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Tokenizer: removing more stuff from simplifyTokenList2

This commit is contained in:
Daniel Marjamäki 2022-06-19 10:41:15 +02:00
parent 3dcdd5f264
commit 6873f5237b
3 changed files with 1 additions and 1359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

393
lib/tokenize.cpp
View File

@ -4705,200 +4705,6 @@ void Tokenizer::sizeofAddParentheses()
}
}
bool Tokenizer::simplifySizeof()
{
// Locate variable declarations and calculate the size
std::map<int, int> sizeOfVar;
std::map<int, const Token *> declTokOfVar;
for (const Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->varId() != 0 && sizeOfVar.find(tok->varId()) == sizeOfVar.end()) {
const int varId = tok->varId();
if (Token::Match(tok->tokAt(-3), "[;{}(,] %type% * %name% [;,)]") ||
Token::Match(tok->tokAt(-4), "[;{}(,] const %type% * %name% [;),]") ||
Token::Match(tok->tokAt(-2), "[;{}(,] %type% %name% [;),]") ||
Token::Match(tok->tokAt(-3), "[;{}(,] const %type% %name% [;),]")) {
const int size = sizeOfType(tok->previous());
if (size == 0) {
continue;
}
sizeOfVar[varId] = size;
declTokOfVar[varId] = tok;
}
else if (Token::Match(tok->previous(), "%type% %name% [ %num% ] [[;=]") ||
Token::Match(tok->tokAt(-2), "%type% * %name% [ %num% ] [[;=]")) {
int size = sizeOfType(tok->previous());
if (size == 0)
continue;
const Token* tok2 = tok->next();
do {
const MathLib::bigint num = MathLib::toLongNumber(tok2->strAt(1));
if (num<0)
break;
size *= num;
tok2 = tok2->tokAt(3);
} while (Token::Match(tok2, "[ %num% ]"));
if (Token::Match(tok2, "[;=]")) {
sizeOfVar[varId] = size;
declTokOfVar[varId] = tok;
}
if (!tok2) {
syntaxError(tok);
}
tok = tok2;
}
else if (Token::Match(tok->previous(), "%type% %name% [ %num% ] [,)]") ||
Token::Match(tok->tokAt(-2), "%type% * %name% [ %num% ] [,)]")) {
Token tempTok;
tempTok.str("*");
sizeOfVar[varId] = sizeOfType(&tempTok);
declTokOfVar[varId] = tok;
}
}
}
bool ret = false;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (tok->str() != "sizeof")
continue;
if (Token::simpleMatch(tok->next(), "...")) {
//tok->deleteNext(3);
tok->deleteNext();
}
// sizeof('x')
if (Token::Match(tok->next(), "( %char% )")) {
tok->deleteNext();
tok->deleteThis();
tok->deleteNext();
std::ostringstream sz;
sz << ((isC()) ? mSettings->sizeof_int : 1);
tok->str(sz.str());
ret = true;
continue;
}
// sizeof ("text")
if (Token::Match(tok->next(), "( %str% )")) {
tok->deleteNext();
tok->deleteThis();
tok->deleteNext();
std::ostringstream ostr;
ostr << (Token::getStrLength(tok) + 1);
tok->str(ostr.str());
ret = true;
continue;
}
// sizeof(type *) => sizeof(*)
if (Token::Match(tok->next(), "( %type% * )")) {
tok->next()->deleteNext();
}
if (Token::simpleMatch(tok->next(), "( * )")) {
tok->str(MathLib::toString(sizeOfType(tok->tokAt(2))));
tok->deleteNext(3);
ret = true;
}
// sizeof( a )
else if (Token::Match(tok->next(), "( %var% )")) {
const std::map<int, int>::const_iterator sizeOfVarPos = sizeOfVar.find(tok->tokAt(2)->varId());
if (sizeOfVarPos != sizeOfVar.end()) {
tok->deleteNext();
tok->deleteThis();
tok->deleteNext();
tok->str(MathLib::toString(sizeOfVarPos->second));
ret = true;
} else {
// don't try to replace size of variable if variable has
// similar name with type (#329)
}
}
else if (Token::Match(tok->next(), "( %type% )")) {
const int size = sizeOfType(tok->tokAt(2));
if (size > 0) {
tok->str(MathLib::toString(size));
tok->deleteNext(3);
ret = true;
}
}
else if (Token::simpleMatch(tok->next(), "( *") || Token::Match(tok->next(), "( %name% [")) {
int derefs = 0;
const Token* nametok = tok->tokAt(2);
if (nametok->str() == "*") {
do {
nametok = nametok->next();
derefs++;
} while (nametok && nametok->str() == "*");
if (!Token::Match(nametok, "%name% )"))
continue;
} else {
const Token* tok2 = nametok->next();
do {
tok2 = tok2->link()->next();
derefs++;
} while (tok2 && tok2->str() == "[");
if (!tok2 || tok2->str() != ")")
continue;
}
// Some default value
MathLib::biguint size = 0;
const int varid = nametok->varId();
if (derefs != 0 && varid != 0 && declTokOfVar.find(varid) != declTokOfVar.end()) {
// Try to locate variable declaration..
const Token *decltok = declTokOfVar[varid];
if (Token::Match(decltok->previous(), "%type%|* %name% [")) {
size = sizeOfType(decltok->previous());
} else if (Token::Match(decltok->tokAt(-2), "%type% * %name%")) {
size = sizeOfType(decltok->tokAt(-2));
}
// Multi-dimensional array..
if (Token::Match(decltok, "%name% [") && Token::simpleMatch(decltok->linkAt(1), "] [")) {
const Token *tok2 = decltok;
for (int i = 0; i < derefs; i++)
tok2 = tok2->linkAt(1); // Skip all dimensions that are dereferenced before the sizeof call
while (Token::Match(tok2, "] [ %num% ]")) {
size *= MathLib::toULongNumber(tok2->strAt(2));
tok2 = tok2->linkAt(1);
}
if (Token::simpleMatch(tok2, "] ["))
continue;
}
} else if (nametok->strAt(1) == "[" && nametok->isStandardType()) {
size = sizeOfType(nametok);
if (size == 0)
continue;
const Token *tok2 = nametok->next();
while (Token::Match(tok2, "[ %num% ]")) {
size *= MathLib::toULongNumber(tok2->strAt(1));
tok2 = tok2->link()->next();
}
if (!tok2 || tok2->str() != ")")
continue;
}
if (size > 0) {
tok->str(MathLib::toString(size));
Token::eraseTokens(tok, tok->next()->link()->next());
ret = true;
}
}
}
return ret;
}
bool Tokenizer::simplifyTokenList1(const char FileName[])
{
if (Settings::terminated())
@ -5352,19 +5158,10 @@ bool Tokenizer::simplifyTokenList2()
if (Settings::terminated())
return false;
simplifySizeof();
simplifyUndefinedSizeArray();
simplifyCasts();
// Simplify simple calculations before replace constants, this allows the replacement of constants that are calculated
// e.g. const static int value = sizeof(X)/sizeof(Y);
simplifyCalculations();
simplifyNestedStrcat();
simplifyFuncInWhile();
bool modified = true;
while (modified) {
if (Settings::terminated())
@ -6287,125 +6084,6 @@ bool Tokenizer::simplifyConstTernaryOp()
return ret;
}
void Tokenizer::simplifyUndefinedSizeArray()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "%type%")) {
Token *tok2 = tok->next();
while (tok2 && tok2->str() == "*")
tok2 = tok2->next();
if (!Token::Match(tok2, "%name% [ ] ;|["))
continue;
tok = tok2->previous();
Token *end = tok2->next();
int count = 0;
do {
end = end->tokAt(2);
++count;
} while (Token::Match(end, "[ ] [;=[]"));
if (Token::Match(end, "[;=]")) {
do {
tok2->deleteNext(2);
tok->insertToken("*");
} while (--count);
tok = end;
} else
tok = tok->tokAt(3);
}
}
}
void Tokenizer::simplifyCasts()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
// Don't remove cast in such cases:
// *((char *)a + 1) = 0;
// Remove cast when casting a function pointer:
// (*(void (*)(char *))fp)(x);
if (!tok->isName() &&
Token::simpleMatch(tok->next(), "* (") &&
!Token::Match(tok->linkAt(2), ") %name%|&")) {
tok = tok->linkAt(2);
continue;
}
// #3935 : don't remove cast in such cases:
// ((char *)a)[1] = 0;
if (tok->str() == "(" && Token::simpleMatch(tok->link(), ") [")) {
tok = tok->link();
continue;
}
// #4164 : ((unsigned char)1) => (1)
if (Token::Match(tok->next(), "( %type% ) %num%") && tok->next()->link()->previous()->isStandardType()) {
const MathLib::bigint value = MathLib::toLongNumber(tok->next()->link()->next()->str());
int bits = mSettings->char_bit * mTypeSize[tok->next()->link()->previous()->str()];
if (!tok->tokAt(2)->isUnsigned() && bits > 0)
bits--;
if (bits < 31 && value >= 0 && value < (1LL << bits)) {
tok->linkAt(1)->next()->isCast(true);
Token::eraseTokens(tok, tok->next()->link()->next());
}
continue;
}
while ((Token::Match(tok->next(), "( %type% *| *| *|&| ) *|&| %name%") && (tok->str() != ")" || tok->tokAt(2)->isStandardType())) ||
Token::Match(tok->next(), "( const| %type% * *| *|&| ) *|&| %name%") ||
Token::Match(tok->next(), "( const| %type% %type% *| *| *|&| ) *|&| %name%") ||
(!tok->isName() && (Token::Match(tok->next(), "( %type% * *| *|&| ) (") ||
Token::Match(tok->next(), "( const| %type% %type% * *| *|&| ) (")))) {
if (tok->isName() && tok->str() != "return")
break;
if (isCPP() && tok->strAt(-1) == "operator")
break;
// Remove cast..
Token::eraseTokens(tok, tok->next()->link()->next());
// Set isCasted flag.
Token *tok2 = tok->next();
if (!Token::Match(tok2, "%name% [|."))
tok2->isCast(true);
else {
// TODO: handle more complex expressions
tok2->next()->isCast(true);
}
// Remove '* &'
if (Token::simpleMatch(tok, "* &")) {
tok->deleteNext();
tok->deleteThis();
}
if (tok->str() == ")" && tok->link()->previous()) {
// If there was another cast before this, go back
// there to check it also. e.g. "(int)(char)x"
tok = tok->link()->previous();
}
}
// Replace pointer casts of 0.. "(char *)0" => "0"
while (Token::Match(tok->next(), "( %type% %type%| * *| ) 0")) {
tok->linkAt(1)->next()->isCast(true);
Token::eraseTokens(tok, tok->next()->link()->next());
if (tok->str() == ")" && tok->link()->previous()) {
// If there was another cast before this, go back
// there to check it also. e.g. "(char*)(char*)0"
tok = tok->link()->previous();
}
}
if (Token::Match(tok->next(), "dynamic_cast|reinterpret_cast|const_cast|static_cast <")) {
Token *tok2 = tok->linkAt(2);
if (!Token::simpleMatch(tok2, "> ("))
break;
tok2->tokAt(2)->isCast(true);
Token::eraseTokens(tok, tok2->next());
}
}
}
void Tokenizer::simplifyFunctionParameters()
{
@ -8431,40 +8109,6 @@ bool Tokenizer::simplifyCalculations()
return mTemplateSimplifier->simplifyCalculations(nullptr, nullptr, false);
}
void Tokenizer::simplifyNestedStrcat()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (!Token::Match(tok, "[;{}] strcat ( strcat (")) {
continue;
}
// find inner strcat call
Token *tok2 = tok->tokAt(3);
while (Token::simpleMatch(tok2, "strcat ( strcat"))
tok2 = tok2->tokAt(2);
if (tok2->strAt(3) != ",")
continue;
// If we have this code:
// strcat(strcat(dst, foo), bar);
// We move this part of code before all strcat() calls: strcat(dst, foo)
// And place "dst" token where the code was.
Token *prevTok = tok2->previous();
// Move tokens to new place
Token::move(tok2, tok2->next()->link(), tok);
tok = tok2->next()->link();
// Insert the "dst" token
prevTok->insertToken(tok2->strAt(2));
prevTok->next()->varId(tok2->tokAt(2)->varId());
// Insert semicolon after the moved strcat()
tok->insertToken(";");
}
}
//---------------------------------------------------------------------------
// Helper functions for handling the tokens list
//---------------------------------------------------------------------------
@ -9643,43 +9287,6 @@ void Tokenizer::simplifyFunctionTryCatch()
}
void Tokenizer::simplifyFuncInWhile()
{
int count = 0;
for (Token *tok = list.front(); tok; tok = tok->next()) {
if (!Token::Match(tok, "while ( %name% ( %name% ) ) {"))
continue;
Token *func = tok->tokAt(2);
const Token * const var = tok->tokAt(4);
Token * const end = tok->next()->link()->next()->link();
const int varid = ++mVarId; // Create new variable
const std::string varname("cppcheck:r" + MathLib::toString(++count));
tok->str("int");
tok->next()->insertToken(varname);
tok->tokAt(2)->varId(varid);
tok->insertToken("while");
tok->insertToken(";");
tok->insertToken(")");
tok->insertToken(var->str());
tok->next()->varId(var->varId());
tok->insertToken("(");
tok->insertToken(func->str());
tok->insertToken("=");
tok->insertToken(varname);
tok->next()->varId(varid);
Token::createMutualLinks(tok->tokAt(4), tok->tokAt(6));
end->previous()->insertToken(varname);
end->previous()->varId(varid);
end->previous()->insertToken("=");
Token::move(func, func->tokAt(3), end->previous());
end->previous()->insertToken(";");
tok = end;
}
}
void Tokenizer::simplifyStructDecl()
{
const bool cpp = isCPP();

25
lib/tokenize.h
View File

@ -233,13 +233,6 @@ public:
/** Add parentheses for sizeof: sizeof x => sizeof(x) */
void sizeofAddParentheses();
/**
* Replace sizeof() to appropriate size.
* @return true if modifications to token-list are done.
* false if no modifications are done.
*/
bool simplifySizeof();
/**
* Simplify variable declarations (split up)
* \param only_k_r_fpar Only simplify K&R function parameters
@ -335,16 +328,6 @@ public:
*/
bool simplifyUsing();
/**
* Simplify casts
*/
void simplifyCasts();
/**
* Change (multiple) arrays to (multiple) pointers.
*/
void simplifyUndefinedSizeArray();
/**
* A simplify function that replaces a variable with its value in cases
* when the value is known. e.g. "x=10; if(x)" => "x=10;if(10)"
@ -374,9 +357,6 @@ public:
*/
void simplifyFlowControl();
/** Expand nested strcat() calls. */
void simplifyNestedStrcat();
/** Simplify "if else" */
void elseif();
@ -491,11 +471,6 @@ public:
private:
/**
* Simplify while(func(f))
*/
void simplifyFuncInWhile();
/** Simplify pointer to standard type (C only) */
void simplifyPointerToStandardType();

942
test/testsimplifytokens.cpp
View File

File diff suppressed because it is too large Load Diff