walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Calculate token scopes in advance rather than as the tokenlist is iterated (#1882) 

* Added scopeinfo member to token class

Moved ScopeInfo2 declaration here as well because that's where it needs to be now.

* Added scopeinfo accessors and declaration to class

* Add new method for calculating scopes

This replaces the methods in the TemplateSimplifier which calculate the current scope as the token list is iterated.  The old method required checking if the scope had changed for every token multiple times (for multiple iterations), which was surprisingly costly.  Calculating scopes in advance like this decreases runtime on a worst-case file by around thirty percent.
ScopeInfo objects are disposed of when the TemplateSimplification is done as they are not used later.

* Add calculateScopes method to header

* Removed code that calculated current scope

This has been replaced by code that calculates the scopes up front and stores them with each token, which is much faster.

* Fixed compile errors from extra parentheses

* Added missing code to fix memory leak

* Added code to actually clean up ScopeInfo structs

* Tidy up a dodgy for loop

* Convert argument to const ref

* Calculate missing scopes

As the templatesimplificator expands templates and does multiple passes it needs to make sure all scopes are calculated.

* Remove copying the scope to the next token

This is now done properly when scopes are calculated.

* Remove call to calculateScopes

This is now done by the TemplateSimplifier.

* Recalculate scopes for every pass of simplifyTemplates

* Add code to calculate extra scopes as they are added

I thought that this might be useful for calculating scopes when Tokens are created, but as there are several ways of creating Tokens that don't guarantee that they are placed in a list it is easier to just calculate scopes when you know you have a list and when you know you're adding to a list.

* Fix several bugs and poorly designed code

Remove the global scopes collection, and clean them up instead by iterating through the tokenlist to find them.  This means scopes can be calculated by functions in the Token class as well as in the Tokenizer class without leaking the scope object.
Fix a couple of bugs in the calculateScopes method and make it more efficient.

* Remove unnecessary calls to calculateScopes

* Move brace to correct position

Calculating scopes during insertToken only needs to happen if we created a new Token.

* Handle 'using namespace' declarations separately

This fixes a bug caused by a statement matching 'struct B < 0 > ;'

* Fix argument name mismatch

* Actually use newScopeInfo when inserting Token

* Switch to using shared_ptr to hold scopeInfos

This means ScopeInfo2 objects get properly cleaned up when they are no longer needed.

* Change ScopeInfo member to be a shared_ptr

* Update code to use shared_ptr

* Add missing include for shared_ptr

* Remove unnecessary cleanup code

This has been replaced by shared_ptr for ScopeInfo2 objects
This commit is contained in:
rebnridgway 2019-06-29 13:34:22 +01:00 committed by Daniel Marjamäki
parent d1d622b74c
commit 0d7836f3a0
5 changed files with 220 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
lib/templatesimplifier.cpp
View File

@ -632,106 +632,10 @@ bool TemplateSimplifier::removeTemplate(Token *tok)
return false;
}
/// TODO: This is copy pasted from Tokenizer. We should reuse this code.
namespace {
struct ScopeInfo2 {
ScopeInfo2(const std::string &name_, const Token *bodyEnd_) : name(name_), bodyEnd(bodyEnd_) {}
const std::string name;
const Token * const bodyEnd;
std::set<std::string> usingNamespaces;
};
}
static std::string getScopeName(const std::list<ScopeInfo2> &scopeInfo)
{
std::string ret;
for (const ScopeInfo2 &i : scopeInfo) {
if (!i.name.empty())
ret += (ret.empty() ? "" : " :: ") + i.name;
}
return ret;
}
static void setScopeInfo(Token *tok, std::list<ScopeInfo2> *scopeInfo, bool all = false)
{
while (tok->str() == "}" && !scopeInfo->empty() && tok == scopeInfo->back().bodyEnd)
scopeInfo->pop_back();
if (!Token::Match(tok, "namespace|class|struct|union %name% {|:|::")) {
// check for using namespace
if (Token::Match(tok, "using namespace %name% ;|::")) {
const Token * tok1 = tok->tokAt(2);
std::string nameSpace;
while (tok1 && tok1->str() != ";") {
if (!nameSpace.empty())
nameSpace += " ";
nameSpace += tok1->str();
tok1 = tok1->next();
}
scopeInfo->back().usingNamespaces.insert(nameSpace);
}
// check for member function
else if (tok->str() == "{") {
bool added = false;
Token *tok1 = tok;
while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
tok1 = tok1->previous();
if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") {
tok1 = tok1->linkAt(-1);
if (Token::Match(tok1->previous(), "throw|noexcept")) {
tok1 = tok1->previous();
while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
tok1 = tok1->previous();
if (tok1->strAt(-1) != ")")
return;
} else if (Token::Match(tok->tokAt(-2), ":|, %name%")) {
tok1 = tok1->tokAt(-2);
if (tok1->strAt(-1) != ")")
return;
}
if (tok1->strAt(-1) == ">")
tok1 = tok1->previous()->findOpeningBracket();
if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) {
tok1 = tok1->tokAt(-2);
std::string scope = tok1->strAt(-1);
while (Token::Match(tok1->tokAt(-2), ":: %name%")) {
scope = tok1->strAt(-3) + " :: " + scope;
tok1 = tok1->tokAt(-2);
}
scopeInfo->emplace_back(scope, tok->link());
added = true;
}
}
if (all && !added)
scopeInfo->emplace_back("", tok->link());
}
return;
}
tok = tok->next();
std::string classname = tok->str();
while (Token::Match(tok, "%name% :: %name%")) {
tok = tok->tokAt(2);
classname += " :: " + tok->str();
}
tok = tok->next();
if (tok && tok->str() == ":") {
while (tok && !Token::Match(tok, ";|{"))
tok = tok->next();
}
if (tok && tok->str() == "{") {
scopeInfo->emplace_back(classname,tok->link());
}
}
bool TemplateSimplifier::getTemplateDeclarations()
{
bool codeWithTemplates = false;
std::list<ScopeInfo2> scopeInfo;
for (Token *tok = mTokenList.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "{|}|namespace|class|struct|union")) {
setScopeInfo(tok, &scopeInfo);
continue;
}
if (!Token::simpleMatch(tok, "template <"))
continue;
// ignore template template parameter
@ -756,7 +660,7 @@ bool TemplateSimplifier::getTemplateDeclarations()
else if (Token::Match(tok2, "{|=|;")) {
const int namepos = getTemplateNamePosition(parmEnd);
if (namepos > 0) {
TokenAndName decl(tok, getScopeName(scopeInfo), parmEnd->tokAt(namepos), parmEnd);
TokenAndName decl(tok, tok->scopeInfo()->name, parmEnd->tokAt(namepos), parmEnd);
if (decl.isForwardDeclaration()) {
// Declaration => add to mTemplateForwardDeclarations
mTemplateForwardDeclarations.emplace_back(decl);
@ -793,18 +697,9 @@ void TemplateSimplifier::getTemplateInstantiations()
functionNameMap.insert(std::make_pair(decl.name, &decl));
}
std::list<ScopeInfo2> scopeList;
const Token *skip = nullptr;
scopeList.emplace_back("", nullptr);
for (Token *tok = mTokenList.front(); tok; tok = tok->next()) {
if (Token::Match(tok, "{|}|namespace|class|struct|union") ||
Token::Match(tok, "using namespace %name% ;|::")) {
setScopeInfo(tok, &scopeList);
continue;
}
// template definition.. skip it
if (Token::simpleMatch(tok, "template <")) {
tok = tok->next()->findClosingBracket();
@ -855,7 +750,7 @@ void TemplateSimplifier::getTemplateInstantiations()
} else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|.|*|&|return|<|, %name% ::|<|(") ||
Token::Match(tok->previous(), "%type% %name% ::|<") ||
Token::Match(tok->tokAt(-2), "[,:] private|protected|public %name% ::|<")) {
std::string scopeName = getScopeName(scopeList);
std::string scopeName = tok->scopeInfo()->name;
std::string qualification;
Token * qualificationTok = tok;
while (Token::Match(tok, "%name% :: %name%")) {
@ -951,7 +846,7 @@ void TemplateSimplifier::getTemplateInstantiations()
for (; tok2 && tok2 != tok; tok2 = tok2->previous()) {
if (Token::Match(tok2, ",|< %name% <") &&
(tok2->strAt(3) == ">" || templateParameters(tok2->tokAt(2)))) {
addInstantiation(tok2->next(), getScopeName(scopeList));
addInstantiation(tok2->next(), tok->scopeInfo()->name);
} else if (Token::Match(tok2->next(), "class|struct"))
tok2->deleteNext();
}
@ -970,7 +865,7 @@ void TemplateSimplifier::getTemplateInstantiations()
} else {
// full name doesn't match so try with using namespaces if available
bool found = false;
for (const auto & nameSpace : scopeList.back().usingNamespaces) {
for (const auto & nameSpace : tok->scopeInfo()->usingNamespaces) {
std::string fullNameSpace = scopeName + (scopeName.empty()?"":" :: ") +
nameSpace + (qualification.empty()?"":" :: ") + qualification;
std::string newFullName = fullNameSpace + " :: " + tok->str();
@ -997,7 +892,7 @@ void TemplateSimplifier::getTemplateInstantiations()
if (!qualification.empty())
addInstantiation(tok, qualification);
else
addInstantiation(tok, getScopeName(scopeList));
addInstantiation(tok, tok->scopeInfo()->name);
break;
}
const std::string::size_type pos = scopeName.rfind(" :: ");
@ -1538,7 +1433,6 @@ void TemplateSimplifier::expandTemplate(
const std::string &newName,
bool copy)
{
std::list<ScopeInfo2> scopeInfo;
bool inTemplateDefinition = false;
const Token *startOfTemplateDeclaration = nullptr;
const Token *endOfTemplateDefinition = nullptr;
@ -1753,10 +1647,6 @@ void TemplateSimplifier::expandTemplate(
}
for (Token *tok3 = mTokenList.front(); tok3; tok3 = tok3 ? tok3->next() : nullptr) {
if (Token::Match(tok3, "{|}|namespace|class|struct|union")) {
setScopeInfo(tok3, &scopeInfo);
continue;
}
if (inTemplateDefinition) {
if (!endOfTemplateDefinition) {
if (isVariable) {
@ -2936,13 +2826,8 @@ void TemplateSimplifier::replaceTemplateUsage(
const std::list<std::string> &typeStringsUsedInTemplateInstantiation,
const std::string &newName)
{
std::list<ScopeInfo2> scopeInfo;
std::list< std::pair<Token *, Token *> > removeTokens;
for (Token *nameTok = mTokenList.front(); nameTok; nameTok = nameTok->next()) {
if (Token::Match(nameTok, "{|}|namespace|class|struct|union")) {
setScopeInfo(nameTok, &scopeInfo);
continue;
}
if (!Token::Match(nameTok, "%name% <") ||
Token::Match(nameTok, "template|const_cast|dynamic_cast|reinterpret_cast|static_cast"))
continue;
@ -3370,6 +3255,10 @@ void TemplateSimplifier::simplifyTemplates(
unsigned int passCount = 0;
const unsigned int passCountMax = 10;
for (; passCount < passCountMax; ++passCount) {
// Recalculate scopes from scratch every pass, in case a scope is missing or incorrect
for (auto tok = mTokenizer->list.front(); tok; tok = tok->next()) tok->scopeInfo(nullptr);
mTokenizer->calculateScopes();
if (passCount) {
// it may take more than one pass to simplify type aliases
bool usingChanged = false;

85
lib/token.cpp
View File

@ -990,6 +990,84 @@ void Token::insertToken(const std::string &tokenStr, const std::string &original
this->next(newToken);
newToken->previous(this);
}
// If the token we're inserting from has a scope, the new token needs one too
if (mImpl->mScopeInfo) {
// If it's a brace, we need to open a new scope
if (tokenStr == "{") {
std::string nextScopeNameAddition = "";
// This might be the opening of a member function
Token *tok1 = newToken;
while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
tok1 = tok1->previous();
if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") {
tok1 = tok1->linkAt(-1);
if (Token::Match(tok1->previous(), "throw|noexcept")) {
tok1 = tok1->previous();
while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
tok1 = tok1->previous();
if (tok1->strAt(-1) != ")")
return;
} else if (Token::Match(newToken->tokAt(-2), ":|, %name%")) {
tok1 = tok1->tokAt(-2);
if (tok1->strAt(-1) != ")")
return;
}
if (tok1->strAt(-1) == ">")
tok1 = tok1->previous()->findOpeningBracket();
if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) {
tok1 = tok1->tokAt(-2);
std::string scope = tok1->strAt(-1);
while (Token::Match(tok1->tokAt(-2), ":: %name%")) {
scope = tok1->strAt(-3) + " :: " + scope;
tok1 = tok1->tokAt(-2);
}
if (nextScopeNameAddition.length() > 0) nextScopeNameAddition += " :: ";
nextScopeNameAddition += scope;
}
}
// Or it might be a namespace/class/struct
if (Token::Match(newToken->previous(), "%name%|>")) {
Token* nameTok = newToken->previous();
while (nameTok && !Token::Match(nameTok, "namespace|class|struct|union %name% {|::|:|<")) {
nameTok = nameTok->previous();
}
if (nameTok) {
for (nameTok = nameTok->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) {
nextScopeNameAddition.append(nameTok->str());
nextScopeNameAddition.append(" ");
}
if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1);
}
}
// New scope is opening, record it here
std::shared_ptr<ScopeInfo2> newScopeInfo = std::make_shared<ScopeInfo2>(mImpl->mScopeInfo->name, nullptr, mImpl->mScopeInfo->usingNamespaces);
if (newScopeInfo->name != "") newScopeInfo->name.append(" :: ");
newScopeInfo->name.append(nextScopeNameAddition);
newToken->scopeInfo(newScopeInfo);
// If it's a closing brace, we need to find where the scope opened and take the scope before
} else if (tokenStr == "}") {
Token* matchingTok = newToken->previous();
int depth = 0;
while (matchingTok && (depth != 0 || !Token::simpleMatch(matchingTok, "{"))) {
if (Token::simpleMatch(matchingTok, "}")) depth++;
if (Token::simpleMatch(matchingTok, "{")) depth--;
matchingTok = matchingTok->previous();
}
if (matchingTok && matchingTok->previous()) {
newToken->mImpl->mScopeInfo = matchingTok->previous()->scopeInfo();
}
// Otherwise we can just take the previous scope
} else {
newToken->mImpl->mScopeInfo = mImpl->mScopeInfo;
}
}
}
}
@ -1846,6 +1924,13 @@ std::string Token::typeStr(const Token* tok)
return r.first->stringifyList(r.second, false);
}
void Token::scopeInfo(std::shared_ptr<ScopeInfo2> newScopeInfo) {
mImpl->mScopeInfo = newScopeInfo;
}
std::shared_ptr<ScopeInfo2> Token::scopeInfo() const {
return mImpl->mScopeInfo;
}
TokenImpl::~TokenImpl()
{
delete mOriginalName;

19
lib/token.h
View File

@ -30,6 +30,7 @@
#include <cstddef>
#include <functional>
#include <list>
#include <memory>
#include <ostream>
#include <string>
#include <vector>
@ -50,6 +51,17 @@ struct TokensFrontBack {
Token *back;
};
struct ScopeInfo2 {
ScopeInfo2(const std::string &name_, const Token *bodyEnd_, const std::set<std::string> &usingNamespaces_ = std::set<std::string>())
: name(name_),
bodyEnd(bodyEnd_),
usingNamespaces(usingNamespaces_)
{}
std::string name;
const Token * const bodyEnd;
std::set<std::string> usingNamespaces;
};
struct TokenImpl {
unsigned int mVarId;
unsigned int mFileIndex;
@ -97,6 +109,8 @@ struct TokenImpl {
// Pointer to a template in the template simplifier
std::set<TemplateSimplifier::TokenAndName*> mTemplateSimplifierPointers;
std::shared_ptr<ScopeInfo2> mScopeInfo;
TokenImpl()
: mVarId(0)
, mFileIndex(0)
@ -114,6 +128,7 @@ struct TokenImpl {
, mValues(nullptr)
, mBits(0)
, mTemplateSimplifierPointers()
, mScopeInfo(nullptr)
{}
~TokenImpl();
@ -1168,6 +1183,10 @@ public:
void printAst(bool verbose, bool xml, std::ostream &out) const;
void printValueFlow(bool xml, std::ostream &out) const;
void scopeInfo(std::shared_ptr<ScopeInfo2> newScopeInfo);
std::shared_ptr<ScopeInfo2> scopeInfo() const;
};
/// @}

108
lib/tokenize.cpp
View File

@ -39,6 +39,7 @@
#include <cstring>
#include <ctime>
#include <iostream>
#include <set>
#include <stack>
#include <unordered_map>
#include <utility>
@ -2810,7 +2811,108 @@ void Tokenizer::simplifyCaseRange()
}
}
void Tokenizer::calculateScopes()
{
// Build up the next scope name as we go and then reset it when we use it
std::string nextScopeNameAddition = "";
if (!list.front()->scopeInfo()) {
std::shared_ptr<ScopeInfo2> primaryScope = std::make_shared<ScopeInfo2>("", list.back());
list.front()->scopeInfo(primaryScope);
if (Token::Match(list.front(), "using namespace %name% ::|<|;")) {
std::string usingNamespaceName = "";
for (const Token* namespaceNameToken = list.front()->tokAt(2); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) {
usingNamespaceName += namespaceNameToken->str();
usingNamespaceName += " ";
}
if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1);
list.front()->scopeInfo()->usingNamespaces.insert(usingNamespaceName);
} else if (Token::Match(list.front(), "namespace|class|struct|union %name% {|::|:|<")) {
for (Token* nameTok = list.front()->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) {
nextScopeNameAddition.append(nameTok->str());
nextScopeNameAddition.append(" ");
}
if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1);
}
}
for (Token* tok = list.front(); tok; tok = tok->next()) {
if (!tok->scopeInfo()) {
tok->scopeInfo(tok->previous()->scopeInfo());
if (Token::Match(tok, "using namespace %name% ::|<|;")) {
std::string usingNamespaceName = "";
for (const Token* namespaceNameToken = tok->tokAt(2); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) {
usingNamespaceName += namespaceNameToken->str();
usingNamespaceName += " ";
}
if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1);
tok->scopeInfo()->usingNamespaces.insert(usingNamespaceName);
} else if (Token::Match(tok, "namespace|class|struct|union %name% {|::|:|<")) {
if (Token::simpleMatch(tok->previous(), "using namespace")) {
std::string usingNamespaceName = "";
for (const Token* namespaceNameToken = tok->next(); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) {
usingNamespaceName += namespaceNameToken->str();
usingNamespaceName += " ";
}
if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1);
tok->scopeInfo()->usingNamespaces.insert(usingNamespaceName);
} else {
for (Token* nameTok = tok->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) {
nextScopeNameAddition.append(nameTok->str());
nextScopeNameAddition.append(" ");
}
if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1);
}
}
if (Token::simpleMatch(tok, "{"))
{
// This might be the opening of a member function
Token *tok1 = tok;
while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
tok1 = tok1->previous();
if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") {
tok1 = tok1->linkAt(-1);
if (Token::Match(tok1->previous(), "throw|noexcept")) {
tok1 = tok1->previous();
while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
tok1 = tok1->previous();
if (tok1->strAt(-1) != ")")
return;
} else if (Token::Match(tok->tokAt(-2), ":|, %name%")) {
tok1 = tok1->tokAt(-2);
if (tok1->strAt(-1) != ")")
return;
}
if (tok1->strAt(-1) == ">")
tok1 = tok1->previous()->findOpeningBracket();
if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) {
tok1 = tok1->tokAt(-2);
std::string scope = tok1->strAt(-1);
while (Token::Match(tok1->tokAt(-2), ":: %name%")) {
scope = tok1->strAt(-3) + " :: " + scope;
tok1 = tok1->tokAt(-2);
}
if (nextScopeNameAddition.length() > 0) nextScopeNameAddition += " :: ";
nextScopeNameAddition += scope;
}
}
// New scope is opening, record it here
std::shared_ptr<ScopeInfo2> newScopeInfo = std::make_shared<ScopeInfo2>(tok->scopeInfo()->name, tok->link(), tok->scopeInfo()->usingNamespaces);
if (newScopeInfo->name != "") newScopeInfo->name.append(" :: ");
newScopeInfo->name.append(nextScopeNameAddition);
nextScopeNameAddition = "";
if (tok->link()) tok->link()->scopeInfo(tok->scopeInfo());
tok->scopeInfo(newScopeInfo);
}
}
}
}
void Tokenizer::simplifyTemplates()
{
@ -3427,12 +3529,6 @@ namespace {
std::list<std::string> scope;
Token *tok;
};
struct ScopeInfo2 {
ScopeInfo2(const std::string &name_, const Token *bodyEnd_) : name(name_), bodyEnd(bodyEnd_) {}
const std::string name;
const Token * const bodyEnd;
};
}
static std::string getScopeName(const std::list<ScopeInfo2> &scopeInfo)

5
lib/tokenize.h
View File

@ -517,6 +517,11 @@ public:
void findComplicatedSyntaxErrorsInTemplates();
/**
* Calculate every token's scope before simplifying templates
*/
void calculateScopes();
/**
* Simplify e.g. 'atol("0")' into '0'
*/