walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Refactoring: Extract several methods from tokenize()

This commit is contained in:
Reijo Tomperi 2011-12-24 23:23:08 +02:00
parent 8e6ebd4a22
commit 347982a347
2 changed files with 225 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

371
lib/tokenize.cpp
View File

@ -2146,143 +2146,14 @@ bool Tokenizer::tokenize(std::istream &code,
// ";a+=b;" => ";a=a+b;"
simplifyCompoundAssignment();
// check for more complicated syntax errors when using templates..
if (!preprocessorCondition) {
for (const Token *tok = _tokens; tok; tok = tok->next()) {
// skip executing scopes..
if (Token::Match(tok, ") const| {") || Token::Match(tok, "[,=] {")) {
while (tok->str() != "{")
tok = tok->next();
tok = tok->link();
}
// skip executing scopes (ticket #1984)..
if (Token::simpleMatch(tok, "; {"))
tok = tok->next()->link();
// skip executing scopes (ticket #3183)..
if (Token::simpleMatch(tok, "( {"))
tok = tok->next()->link();
// skip executing scopes (ticket #1985)..
if (Token::simpleMatch(tok, "try {")) {
tok = tok->next()->link();
while (Token::simpleMatch(tok, "} catch (")) {
tok = tok->linkAt(2);
if (Token::simpleMatch(tok, ") {"))
tok = tok->next()->link();
}
}
// not start of statement?
if (tok->previous() && !Token::Match(tok, "[;{}]"))
continue;
// skip starting tokens.. ;;; typedef typename foo::bar::..
while (Token::Match(tok, "[;{}]"))
tok = tok->next();
while (Token::Match(tok, "typedef|typename"))
tok = tok->next();
while (Token::Match(tok, "%type% ::"))
tok = tok->tokAt(2);
if (!tok)
break;
// template variable or type..
if (Token::Match(tok, "%type% <")) {
// these are used types..
std::set<std::string> usedtypes;
// parse this statement and see if the '<' and '>' are matching
unsigned int level = 0;
for (const Token *tok2 = tok; tok2 && !Token::Match(tok2, "[;{}]"); tok2 = tok2->next()) {
if (tok2->str() == "(")
tok2 = tok2->link();
else if (tok2->str() == "<") {
bool inclevel = false;
if (Token::simpleMatch(tok2->previous(), "operator <"))
;
else if (level == 0)
inclevel = true;
else if (tok2->next() && tok2->next()->isStandardType())
inclevel = true;
else if (Token::simpleMatch(tok2, "< typename"))
inclevel = true;
else if (Token::Match(tok2->tokAt(-2), "<|, %type% <") && usedtypes.find(tok2->previous()->str()) != usedtypes.end())
inclevel = true;
else if (Token::Match(tok2, "< %type%") && usedtypes.find(tok2->next()->str()) != usedtypes.end())
inclevel = true;
else if (Token::Match(tok2, "< %type%")) {
// is the next token a type and not a variable/constant?
// assume it's a type if there comes another "<"
const Token *tok3 = tok2->next();
while (Token::Match(tok3, "%type% ::"))
tok3 = tok3->tokAt(2);
if (Token::Match(tok3, "%type% <"))
inclevel = true;
}
if (inclevel) {
++level;
if (Token::Match(tok2->tokAt(-2), "<|, %type% <"))
usedtypes.insert(tok2->previous()->str());
}
} else if (tok2->str() == ">") {
if (level > 0)
--level;
} else if (tok2->str() == ">>") {
if (level > 0)
--level;
if (level > 0)
--level;
}
}
if (level > 0) {
syntaxError(tok);
deallocateTokens();
return false;
}
}
if (hasComplicatedSyntaxErrorsInTemplates()) {
deallocateTokens();
return false;
}
}
// Remove "= default|delete" inside class|struct definitions
// Todo: Remove it if it is used "externally" too.
for (Token *tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "struct|class %var% :|{")) {
unsigned int indentlevel = 0;
for (Token * tok2 = tok->tokAt(2); tok2; tok2 = tok2->next()) {
if (tok2->str() == "{")
++indentlevel;
else if (tok2->str() == "}") {
if (indentlevel <= 1)
break;
--indentlevel;
} else if (indentlevel == 1 && Token::Match(tok2, ") = delete|default ;")) {
Token * const end = tok2->tokAt(4);
tok2 = tok2->link()->previous();
// operator ==|>|<|..
if (Token::Match(tok2->previous(), "operator %any%"))
tok2 = tok2->previous();
else if (Token::simpleMatch(tok2->tokAt(-2), "operator [ ]"))
tok2 = tok2->tokAt(-2);
else if (Token::simpleMatch(tok2->tokAt(-2), "operator ( )"))
tok2 = tok2->tokAt(-2);
else if (Token::simpleMatch(tok2->tokAt(-3), "operator delete [ ]"))
tok2 = tok2->tokAt(-3);
while ((tok2->isName() && tok2->str().find(":") == std::string::npos) ||
Token::Match(tok2, "[&*~]"))
tok2 = tok2->previous();
if (Token::Match(tok2, "[;{}]") || tok2->isName())
Token::eraseTokens(tok2, end);
tok2 = end;
}
}
}
}
simplifyDefaultAndDeleteInsideClass();
// Remove __declspec()
simplifyDeclspec();
@ -2320,26 +2191,13 @@ bool Tokenizer::tokenize(std::istream &code,
// Remove __builtin_expect, likely and unlikely
simplifyBuiltinExpect();
// #2449: syntax error: enum with typedef in it
for (const Token *tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "enum %var% {")) {
for (const Token *tok2 = tok->tokAt(3); tok2; tok2 = tok2->next()) {
if (tok2->str() == "typedef") {
syntaxError(tok2);
deallocateTokens();
return false;
} else if (tok2->str() == "}") {
break;
}
}
}
if (hasEnumsWithTypedef()) {
// #2449: syntax error: enum with typedef in it
deallocateTokens();
return false;
}
// convert Microsoft DEBUG_NEW macro to new
for (Token *tok = _tokens; tok; tok = tok->next()) {
if (tok->str() == "DEBUG_NEW")
tok->str("new");
}
simplifyDebugNew();
// typedef..
simplifyTypedef();
@ -2539,6 +2397,193 @@ bool Tokenizer::tokenize(std::istream &code,
simplifyDoublePlusAndDoubleMinus();
simplifyArrayAccessSyntax();
_tokens->assignProgressValues();
removeRedundantSemicolons();
simplifyReservedWordNullptr();
simplifyParameterVoid();
simplifyRedundantConsecutiveBraces();
return validate();
}
//---------------------------------------------------------------------------
bool Tokenizer::hasComplicatedSyntaxErrorsInTemplates()
{
// check for more complicated syntax errors when using templates..
for (const Token *tok = _tokens; tok; tok = tok->next()) {
// skip executing scopes..
if (Token::Match(tok, ") const| {") || Token::Match(tok, "[,=] {")) {
while (tok->str() != "{")
tok = tok->next();
tok = tok->link();
}
// skip executing scopes (ticket #1984)..
if (Token::simpleMatch(tok, "; {"))
tok = tok->next()->link();
// skip executing scopes (ticket #3183)..
if (Token::simpleMatch(tok, "( {"))
tok = tok->next()->link();
// skip executing scopes (ticket #1985)..
if (Token::simpleMatch(tok, "try {")) {
tok = tok->next()->link();
while (Token::simpleMatch(tok, "} catch (")) {
tok = tok->linkAt(2);
if (Token::simpleMatch(tok, ") {"))
tok = tok->next()->link();
}
}
// not start of statement?
if (tok->previous() && !Token::Match(tok, "[;{}]"))
continue;
// skip starting tokens.. ;;; typedef typename foo::bar::..
while (Token::Match(tok, "[;{}]"))
tok = tok->next();
while (Token::Match(tok, "typedef|typename"))
tok = tok->next();
while (Token::Match(tok, "%type% ::"))
tok = tok->tokAt(2);
if (!tok)
break;
// template variable or type..
if (Token::Match(tok, "%type% <")) {
// these are used types..
std::set<std::string> usedtypes;
// parse this statement and see if the '<' and '>' are matching
unsigned int level = 0;
for (const Token *tok2 = tok; tok2 && !Token::Match(tok2, "[;{}]"); tok2 = tok2->next()) {
if (tok2->str() == "(")
tok2 = tok2->link();
else if (tok2->str() == "<") {
bool inclevel = false;
if (Token::simpleMatch(tok2->previous(), "operator <"))
;
else if (level == 0)
inclevel = true;
else if (tok2->next() && tok2->next()->isStandardType())
inclevel = true;
else if (Token::simpleMatch(tok2, "< typename"))
inclevel = true;
else if (Token::Match(tok2->tokAt(-2), "<|, %type% <") && usedtypes.find(tok2->previous()->str()) != usedtypes.end())
inclevel = true;
else if (Token::Match(tok2, "< %type%") && usedtypes.find(tok2->next()->str()) != usedtypes.end())
inclevel = true;
else if (Token::Match(tok2, "< %type%")) {
// is the next token a type and not a variable/constant?
// assume it's a type if there comes another "<"
const Token *tok3 = tok2->next();
while (Token::Match(tok3, "%type% ::"))
tok3 = tok3->tokAt(2);
if (Token::Match(tok3, "%type% <"))
inclevel = true;
}
if (inclevel) {
++level;
if (Token::Match(tok2->tokAt(-2), "<|, %type% <"))
usedtypes.insert(tok2->previous()->str());
}
} else if (tok2->str() == ">") {
if (level > 0)
--level;
} else if (tok2->str() == ">>") {
if (level > 0)
--level;
if (level > 0)
--level;
}
}
if (level > 0) {
syntaxError(tok);
return true;
}
}
}
return false;
}
void Tokenizer::simplifyDefaultAndDeleteInsideClass()
{
// Remove "= default|delete" inside class|struct definitions
// Todo: Remove it if it is used "externally" too.
for (Token *tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "struct|class %var% :|{")) {
unsigned int indentlevel = 0;
for (Token * tok2 = tok->tokAt(2); tok2; tok2 = tok2->next()) {
if (tok2->str() == "{")
++indentlevel;
else if (tok2->str() == "}") {
if (indentlevel <= 1)
break;
--indentlevel;
} else if (indentlevel == 1 && Token::Match(tok2, ") = delete|default ;")) {
Token * const end = tok2->tokAt(4);
tok2 = tok2->link()->previous();
// operator ==|>|<|..
if (Token::Match(tok2->previous(), "operator %any%"))
tok2 = tok2->previous();
else if (Token::simpleMatch(tok2->tokAt(-2), "operator [ ]"))
tok2 = tok2->tokAt(-2);
else if (Token::simpleMatch(tok2->tokAt(-2), "operator ( )"))
tok2 = tok2->tokAt(-2);
else if (Token::simpleMatch(tok2->tokAt(-3), "operator delete [ ]"))
tok2 = tok2->tokAt(-3);
while ((tok2->isName() && tok2->str().find(":") == std::string::npos) ||
Token::Match(tok2, "[&*~]"))
tok2 = tok2->previous();
if (Token::Match(tok2, "[;{}]") || tok2->isName())
Token::eraseTokens(tok2, end);
tok2 = end;
}
}
}
}
}
bool Tokenizer::hasEnumsWithTypedef()
{
for (const Token *tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "enum %var% {")) {
for (const Token *tok2 = tok->tokAt(3); tok2; tok2 = tok2->next()) {
if (tok2->str() == "typedef") {
syntaxError(tok2);
return true;
} else if (tok2->str() == "}") {
break;
}
}
}
}
return false;
}
void Tokenizer::simplifyDebugNew()
{
// convert Microsoft DEBUG_NEW macro to new
for (Token *tok = _tokens; tok; tok = tok->next()) {
if (tok->str() == "DEBUG_NEW")
tok->str("new");
}
}
void Tokenizer::simplifyArrayAccessSyntax()
{
// 0[a] -> a[0]
for (Token *tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "%num% [ %var% ]")) {
@ -2547,23 +2592,28 @@ bool Tokenizer::tokenize(std::istream &code,
tok->tokAt(2)->str(temp);
}
}
}
_tokens->assignProgressValues();
removeRedundantSemicolons();
void Tokenizer::simplifyParameterVoid()
{
for (Token* tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "%var% ( void )"))
tok->next()->deleteNext();
}
}
void Tokenizer::simplifyReservedWordNullptr()
{
if (_settings->standards.cpp11) {
for (Token *tok = _tokens; tok; tok = tok->next()) {
if (tok->str() == "nullptr")
tok->str("0");
}
}
}
for (Token* tok = _tokens; tok; tok = tok->next()) {
if (Token::Match(tok, "%var% ( void )"))
tok->next()->deleteNext();
}
void Tokenizer::simplifyRedundantConsecutiveBraces()
{
// Remove redundant consecutive braces, i.e. '.. { { .. } } ..' -> '.. { .. } ..'.
for (Token *tok = _tokens; tok;) {
if (Token::simpleMatch(tok, "{ {") && Token::simpleMatch(tok->next()->link(), "} }")) {
@ -2573,10 +2623,7 @@ bool Tokenizer::tokenize(std::istream &code,
} else
tok = tok->next();
}
return validate();
}
//---------------------------------------------------------------------------
void Tokenizer::simplifyDoublePlusAndDoubleMinus()
{

16
lib/tokenize.h
View File

@ -460,6 +460,22 @@ public:
void simplifyDoublePlusAndDoubleMinus();
void simplifyRedundantConsecutiveBraces();
void simplifyArrayAccessSyntax();
void simplifyParameterVoid();
void simplifyDebugNew();
bool hasEnumsWithTypedef();
void simplifyDefaultAndDeleteInsideClass();
bool hasComplicatedSyntaxErrorsInTemplates();
void simplifyReservedWordNullptr();
/**
* Get template declarations
* @return list of template declarations