From d9a9743b8aef4e9112f086da4bb94ce02b31bd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Sun, 30 Jun 2019 09:12:18 +0200 Subject: [PATCH] Revert "Calculate token scopes in advance rather than as the tokenlist is iterated (#1882)" This reverts commit 0d7836f3a0e6923591a074c7c05529679516628b. --- lib/templatesimplifier.cpp | 129 ++++++++++++++++++++++++++++++++++--- lib/token.cpp | 85 ------------------------ lib/token.h | 19 ------ lib/tokenize.cpp | 108 ++----------------------------- lib/tokenize.h | 5 -- 5 files changed, 126 insertions(+), 220 deletions(-) diff --git a/lib/templatesimplifier.cpp b/lib/templatesimplifier.cpp index f8082b37f..4a121e642 100644 --- a/lib/templatesimplifier.cpp +++ b/lib/templatesimplifier.cpp @@ -632,10 +632,106 @@ bool TemplateSimplifier::removeTemplate(Token *tok) return false; } +/// TODO: This is copy pasted from Tokenizer. We should reuse this code. +namespace { + struct ScopeInfo2 { + ScopeInfo2(const std::string &name_, const Token *bodyEnd_) : name(name_), bodyEnd(bodyEnd_) {} + const std::string name; + const Token * const bodyEnd; + std::set usingNamespaces; + }; +} +static std::string getScopeName(const std::list &scopeInfo) +{ + std::string ret; + for (const ScopeInfo2 &i : scopeInfo) { + if (!i.name.empty()) + ret += (ret.empty() ? "" : " :: ") + i.name; + } + return ret; +} + +static void setScopeInfo(Token *tok, std::list *scopeInfo, bool all = false) +{ + while (tok->str() == "}" && !scopeInfo->empty() && tok == scopeInfo->back().bodyEnd) + scopeInfo->pop_back(); + if (!Token::Match(tok, "namespace|class|struct|union %name% {|:|::")) { + // check for using namespace + if (Token::Match(tok, "using namespace %name% ;|::")) { + const Token * tok1 = tok->tokAt(2); + std::string nameSpace; + while (tok1 && tok1->str() != ";") { + if (!nameSpace.empty()) + nameSpace += " "; + nameSpace += tok1->str(); + tok1 = tok1->next(); + } + scopeInfo->back().usingNamespaces.insert(nameSpace); + } + // check for member function + else if (tok->str() == "{") { + bool added = false; + Token *tok1 = tok; + while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept")) + tok1 = tok1->previous(); + if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") { + tok1 = tok1->linkAt(-1); + if (Token::Match(tok1->previous(), "throw|noexcept")) { + tok1 = tok1->previous(); + while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept")) + tok1 = tok1->previous(); + if (tok1->strAt(-1) != ")") + return; + } else if (Token::Match(tok->tokAt(-2), ":|, %name%")) { + tok1 = tok1->tokAt(-2); + if (tok1->strAt(-1) != ")") + return; + } + if (tok1->strAt(-1) == ">") + tok1 = tok1->previous()->findOpeningBracket(); + if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) { + tok1 = tok1->tokAt(-2); + std::string scope = tok1->strAt(-1); + while (Token::Match(tok1->tokAt(-2), ":: %name%")) { + scope = tok1->strAt(-3) + " :: " + scope; + tok1 = tok1->tokAt(-2); + } + scopeInfo->emplace_back(scope, tok->link()); + added = true; + } + } + + if (all && !added) + scopeInfo->emplace_back("", tok->link()); + } + return; + } + + tok = tok->next(); + std::string classname = tok->str(); + while (Token::Match(tok, "%name% :: %name%")) { + tok = tok->tokAt(2); + classname += " :: " + tok->str(); + } + tok = tok->next(); + if (tok && tok->str() == ":") { + while (tok && !Token::Match(tok, ";|{")) + tok = tok->next(); + } + if (tok && tok->str() == "{") { + scopeInfo->emplace_back(classname,tok->link()); + } +} + bool TemplateSimplifier::getTemplateDeclarations() { bool codeWithTemplates = false; + std::list scopeInfo; for (Token *tok = mTokenList.front(); tok; tok = tok->next()) { + if (Token::Match(tok, "{|}|namespace|class|struct|union")) { + setScopeInfo(tok, &scopeInfo); + continue; + } if (!Token::simpleMatch(tok, "template <")) continue; // ignore template template parameter @@ -660,7 +756,7 @@ bool TemplateSimplifier::getTemplateDeclarations() else if (Token::Match(tok2, "{|=|;")) { const int namepos = getTemplateNamePosition(parmEnd); if (namepos > 0) { - TokenAndName decl(tok, tok->scopeInfo()->name, parmEnd->tokAt(namepos), parmEnd); + TokenAndName decl(tok, getScopeName(scopeInfo), parmEnd->tokAt(namepos), parmEnd); if (decl.isForwardDeclaration()) { // Declaration => add to mTemplateForwardDeclarations mTemplateForwardDeclarations.emplace_back(decl); @@ -697,9 +793,18 @@ void TemplateSimplifier::getTemplateInstantiations() functionNameMap.insert(std::make_pair(decl.name, &decl)); } + std::list scopeList; const Token *skip = nullptr; + scopeList.emplace_back("", nullptr); + for (Token *tok = mTokenList.front(); tok; tok = tok->next()) { + if (Token::Match(tok, "{|}|namespace|class|struct|union") || + Token::Match(tok, "using namespace %name% ;|::")) { + setScopeInfo(tok, &scopeList); + continue; + } + // template definition.. skip it if (Token::simpleMatch(tok, "template <")) { tok = tok->next()->findClosingBracket(); @@ -750,7 +855,7 @@ void TemplateSimplifier::getTemplateInstantiations() } else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|.|*|&|return|<|, %name% ::|<|(") || Token::Match(tok->previous(), "%type% %name% ::|<") || Token::Match(tok->tokAt(-2), "[,:] private|protected|public %name% ::|<")) { - std::string scopeName = tok->scopeInfo()->name; + std::string scopeName = getScopeName(scopeList); std::string qualification; Token * qualificationTok = tok; while (Token::Match(tok, "%name% :: %name%")) { @@ -846,7 +951,7 @@ void TemplateSimplifier::getTemplateInstantiations() for (; tok2 && tok2 != tok; tok2 = tok2->previous()) { if (Token::Match(tok2, ",|< %name% <") && (tok2->strAt(3) == ">" || templateParameters(tok2->tokAt(2)))) { - addInstantiation(tok2->next(), tok->scopeInfo()->name); + addInstantiation(tok2->next(), getScopeName(scopeList)); } else if (Token::Match(tok2->next(), "class|struct")) tok2->deleteNext(); } @@ -865,7 +970,7 @@ void TemplateSimplifier::getTemplateInstantiations() } else { // full name doesn't match so try with using namespaces if available bool found = false; - for (const auto & nameSpace : tok->scopeInfo()->usingNamespaces) { + for (const auto & nameSpace : scopeList.back().usingNamespaces) { std::string fullNameSpace = scopeName + (scopeName.empty()?"":" :: ") + nameSpace + (qualification.empty()?"":" :: ") + qualification; std::string newFullName = fullNameSpace + " :: " + tok->str(); @@ -892,7 +997,7 @@ void TemplateSimplifier::getTemplateInstantiations() if (!qualification.empty()) addInstantiation(tok, qualification); else - addInstantiation(tok, tok->scopeInfo()->name); + addInstantiation(tok, getScopeName(scopeList)); break; } const std::string::size_type pos = scopeName.rfind(" :: "); @@ -1433,6 +1538,7 @@ void TemplateSimplifier::expandTemplate( const std::string &newName, bool copy) { + std::list scopeInfo; bool inTemplateDefinition = false; const Token *startOfTemplateDeclaration = nullptr; const Token *endOfTemplateDefinition = nullptr; @@ -1647,6 +1753,10 @@ void TemplateSimplifier::expandTemplate( } for (Token *tok3 = mTokenList.front(); tok3; tok3 = tok3 ? tok3->next() : nullptr) { + if (Token::Match(tok3, "{|}|namespace|class|struct|union")) { + setScopeInfo(tok3, &scopeInfo); + continue; + } if (inTemplateDefinition) { if (!endOfTemplateDefinition) { if (isVariable) { @@ -2826,8 +2936,13 @@ void TemplateSimplifier::replaceTemplateUsage( const std::list &typeStringsUsedInTemplateInstantiation, const std::string &newName) { + std::list scopeInfo; std::list< std::pair > removeTokens; for (Token *nameTok = mTokenList.front(); nameTok; nameTok = nameTok->next()) { + if (Token::Match(nameTok, "{|}|namespace|class|struct|union")) { + setScopeInfo(nameTok, &scopeInfo); + continue; + } if (!Token::Match(nameTok, "%name% <") || Token::Match(nameTok, "template|const_cast|dynamic_cast|reinterpret_cast|static_cast")) continue; @@ -3255,10 +3370,6 @@ void TemplateSimplifier::simplifyTemplates( unsigned int passCount = 0; const unsigned int passCountMax = 10; for (; passCount < passCountMax; ++passCount) { - // Recalculate scopes from scratch every pass, in case a scope is missing or incorrect - for (auto tok = mTokenizer->list.front(); tok; tok = tok->next()) tok->scopeInfo(nullptr); - mTokenizer->calculateScopes(); - if (passCount) { // it may take more than one pass to simplify type aliases bool usingChanged = false; diff --git a/lib/token.cpp b/lib/token.cpp index 162ccc5d2..0cb86a52c 100644 --- a/lib/token.cpp +++ b/lib/token.cpp @@ -990,84 +990,6 @@ void Token::insertToken(const std::string &tokenStr, const std::string &original this->next(newToken); newToken->previous(this); } - - // If the token we're inserting from has a scope, the new token needs one too - if (mImpl->mScopeInfo) { - // If it's a brace, we need to open a new scope - if (tokenStr == "{") { - std::string nextScopeNameAddition = ""; - // This might be the opening of a member function - Token *tok1 = newToken; - while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept")) - tok1 = tok1->previous(); - if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") { - tok1 = tok1->linkAt(-1); - if (Token::Match(tok1->previous(), "throw|noexcept")) { - tok1 = tok1->previous(); - while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept")) - tok1 = tok1->previous(); - if (tok1->strAt(-1) != ")") - return; - } else if (Token::Match(newToken->tokAt(-2), ":|, %name%")) { - tok1 = tok1->tokAt(-2); - if (tok1->strAt(-1) != ")") - return; - } - if (tok1->strAt(-1) == ">") - tok1 = tok1->previous()->findOpeningBracket(); - if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) { - tok1 = tok1->tokAt(-2); - std::string scope = tok1->strAt(-1); - while (Token::Match(tok1->tokAt(-2), ":: %name%")) { - scope = tok1->strAt(-3) + " :: " + scope; - tok1 = tok1->tokAt(-2); - } - - if (nextScopeNameAddition.length() > 0) nextScopeNameAddition += " :: "; - nextScopeNameAddition += scope; - } - } - - // Or it might be a namespace/class/struct - if (Token::Match(newToken->previous(), "%name%|>")) { - Token* nameTok = newToken->previous(); - while (nameTok && !Token::Match(nameTok, "namespace|class|struct|union %name% {|::|:|<")) { - nameTok = nameTok->previous(); - } - if (nameTok) { - for (nameTok = nameTok->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) { - nextScopeNameAddition.append(nameTok->str()); - nextScopeNameAddition.append(" "); - } - if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1); - } - } - - // New scope is opening, record it here - std::shared_ptr newScopeInfo = std::make_shared(mImpl->mScopeInfo->name, nullptr, mImpl->mScopeInfo->usingNamespaces); - - if (newScopeInfo->name != "") newScopeInfo->name.append(" :: "); - newScopeInfo->name.append(nextScopeNameAddition); - - newToken->scopeInfo(newScopeInfo); - - // If it's a closing brace, we need to find where the scope opened and take the scope before - } else if (tokenStr == "}") { - Token* matchingTok = newToken->previous(); - int depth = 0; - while (matchingTok && (depth != 0 || !Token::simpleMatch(matchingTok, "{"))) { - if (Token::simpleMatch(matchingTok, "}")) depth++; - if (Token::simpleMatch(matchingTok, "{")) depth--; - matchingTok = matchingTok->previous(); - } - if (matchingTok && matchingTok->previous()) { - newToken->mImpl->mScopeInfo = matchingTok->previous()->scopeInfo(); - } - // Otherwise we can just take the previous scope - } else { - newToken->mImpl->mScopeInfo = mImpl->mScopeInfo; - } - } } } @@ -1924,13 +1846,6 @@ std::string Token::typeStr(const Token* tok) return r.first->stringifyList(r.second, false); } -void Token::scopeInfo(std::shared_ptr newScopeInfo) { - mImpl->mScopeInfo = newScopeInfo; -} -std::shared_ptr Token::scopeInfo() const { - return mImpl->mScopeInfo; -} - TokenImpl::~TokenImpl() { delete mOriginalName; diff --git a/lib/token.h b/lib/token.h index 6e250e4fb..ef290dcb8 100644 --- a/lib/token.h +++ b/lib/token.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -51,17 +50,6 @@ struct TokensFrontBack { Token *back; }; -struct ScopeInfo2 { - ScopeInfo2(const std::string &name_, const Token *bodyEnd_, const std::set &usingNamespaces_ = std::set()) - : name(name_), - bodyEnd(bodyEnd_), - usingNamespaces(usingNamespaces_) - {} - std::string name; - const Token * const bodyEnd; - std::set usingNamespaces; -}; - struct TokenImpl { unsigned int mVarId; unsigned int mFileIndex; @@ -109,8 +97,6 @@ struct TokenImpl { // Pointer to a template in the template simplifier std::set mTemplateSimplifierPointers; - std::shared_ptr mScopeInfo; - TokenImpl() : mVarId(0) , mFileIndex(0) @@ -128,7 +114,6 @@ struct TokenImpl { , mValues(nullptr) , mBits(0) , mTemplateSimplifierPointers() - , mScopeInfo(nullptr) {} ~TokenImpl(); @@ -1183,10 +1168,6 @@ public: void printAst(bool verbose, bool xml, std::ostream &out) const; void printValueFlow(bool xml, std::ostream &out) const; - - void scopeInfo(std::shared_ptr newScopeInfo); - - std::shared_ptr scopeInfo() const; }; /// @} diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 61eed09f2..55ce26499 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -2811,108 +2810,7 @@ void Tokenizer::simplifyCaseRange() } } -void Tokenizer::calculateScopes() -{ - // Build up the next scope name as we go and then reset it when we use it - std::string nextScopeNameAddition = ""; - if (!list.front()->scopeInfo()) { - std::shared_ptr primaryScope = std::make_shared("", list.back()); - list.front()->scopeInfo(primaryScope); - if (Token::Match(list.front(), "using namespace %name% ::|<|;")) { - std::string usingNamespaceName = ""; - for (const Token* namespaceNameToken = list.front()->tokAt(2); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) { - usingNamespaceName += namespaceNameToken->str(); - usingNamespaceName += " "; - } - if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1); - list.front()->scopeInfo()->usingNamespaces.insert(usingNamespaceName); - } else if (Token::Match(list.front(), "namespace|class|struct|union %name% {|::|:|<")) { - for (Token* nameTok = list.front()->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) { - nextScopeNameAddition.append(nameTok->str()); - nextScopeNameAddition.append(" "); - } - if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1); - } - } - - for (Token* tok = list.front(); tok; tok = tok->next()) { - if (!tok->scopeInfo()) { - tok->scopeInfo(tok->previous()->scopeInfo()); - - if (Token::Match(tok, "using namespace %name% ::|<|;")) { - std::string usingNamespaceName = ""; - for (const Token* namespaceNameToken = tok->tokAt(2); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) { - usingNamespaceName += namespaceNameToken->str(); - usingNamespaceName += " "; - } - if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1); - tok->scopeInfo()->usingNamespaces.insert(usingNamespaceName); - } else if (Token::Match(tok, "namespace|class|struct|union %name% {|::|:|<")) { - if (Token::simpleMatch(tok->previous(), "using namespace")) { - std::string usingNamespaceName = ""; - for (const Token* namespaceNameToken = tok->next(); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) { - usingNamespaceName += namespaceNameToken->str(); - usingNamespaceName += " "; - } - if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1); - tok->scopeInfo()->usingNamespaces.insert(usingNamespaceName); - } else { - for (Token* nameTok = tok->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) { - nextScopeNameAddition.append(nameTok->str()); - nextScopeNameAddition.append(" "); - } - if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1); - } - } - - if (Token::simpleMatch(tok, "{")) - { - // This might be the opening of a member function - Token *tok1 = tok; - while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept")) - tok1 = tok1->previous(); - if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") { - tok1 = tok1->linkAt(-1); - if (Token::Match(tok1->previous(), "throw|noexcept")) { - tok1 = tok1->previous(); - while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept")) - tok1 = tok1->previous(); - if (tok1->strAt(-1) != ")") - return; - } else if (Token::Match(tok->tokAt(-2), ":|, %name%")) { - tok1 = tok1->tokAt(-2); - if (tok1->strAt(-1) != ")") - return; - } - if (tok1->strAt(-1) == ">") - tok1 = tok1->previous()->findOpeningBracket(); - if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) { - tok1 = tok1->tokAt(-2); - std::string scope = tok1->strAt(-1); - while (Token::Match(tok1->tokAt(-2), ":: %name%")) { - scope = tok1->strAt(-3) + " :: " + scope; - tok1 = tok1->tokAt(-2); - } - - if (nextScopeNameAddition.length() > 0) nextScopeNameAddition += " :: "; - nextScopeNameAddition += scope; - } - } - - // New scope is opening, record it here - std::shared_ptr newScopeInfo = std::make_shared(tok->scopeInfo()->name, tok->link(), tok->scopeInfo()->usingNamespaces); - - if (newScopeInfo->name != "") newScopeInfo->name.append(" :: "); - newScopeInfo->name.append(nextScopeNameAddition); - nextScopeNameAddition = ""; - - if (tok->link()) tok->link()->scopeInfo(tok->scopeInfo()); - tok->scopeInfo(newScopeInfo); - } - } - } -} void Tokenizer::simplifyTemplates() { @@ -3529,6 +3427,12 @@ namespace { std::list scope; Token *tok; }; + + struct ScopeInfo2 { + ScopeInfo2(const std::string &name_, const Token *bodyEnd_) : name(name_), bodyEnd(bodyEnd_) {} + const std::string name; + const Token * const bodyEnd; + }; } static std::string getScopeName(const std::list &scopeInfo) diff --git a/lib/tokenize.h b/lib/tokenize.h index c07c073e0..f7bf19a36 100644 --- a/lib/tokenize.h +++ b/lib/tokenize.h @@ -517,11 +517,6 @@ public: void findComplicatedSyntaxErrorsInTemplates(); - /** - * Calculate every token's scope before simplifying templates - */ - void calculateScopes(); - /** * Simplify e.g. 'atol("0")' into '0' */