Revert "Calculate token scopes in advance rather than as the tokenlist is iterated (#1882)"

This reverts commit 0d7836f3a0.

lib/templatesimplifier.cpp

@@ -632,10 +632,106 @@ bool TemplateSimplifier::removeTemplate(Token *tok)
     return false;
 }
 
+/// TODO: This is copy pasted from Tokenizer. We should reuse this code.
+namespace {
+    struct ScopeInfo2 {
+        ScopeInfo2(const std::string &name_, const Token *bodyEnd_) : name(name_), bodyEnd(bodyEnd_) {}
+        const std::string name;
+        const Token * const bodyEnd;
+        std::set<std::string> usingNamespaces;
+    };
+}
+static std::string getScopeName(const std::list<ScopeInfo2> &scopeInfo)
+{
+    std::string ret;
+    for (const ScopeInfo2 &i : scopeInfo) {
+        if (!i.name.empty())
+            ret += (ret.empty() ? "" : " :: ") + i.name;
+    }
+    return ret;
+}
+
+static void setScopeInfo(Token *tok, std::list<ScopeInfo2> *scopeInfo, bool all = false)
+{
+    while (tok->str() == "}" && !scopeInfo->empty() && tok == scopeInfo->back().bodyEnd)
+        scopeInfo->pop_back();
+    if (!Token::Match(tok, "namespace|class|struct|union %name% {|:|::")) {
+        // check for using namespace
+        if (Token::Match(tok, "using namespace %name% ;|::")) {
+            const Token * tok1 = tok->tokAt(2);
+            std::string nameSpace;
+            while (tok1 && tok1->str() != ";") {
+                if (!nameSpace.empty())
+                    nameSpace += " ";
+                nameSpace += tok1->str();
+                tok1 = tok1->next();
+            }
+            scopeInfo->back().usingNamespaces.insert(nameSpace);
+        }
+        // check for member function
+        else if (tok->str() == "{") {
+            bool added = false;
+            Token *tok1 = tok;
+            while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
+                tok1 = tok1->previous();
+            if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") {
+                tok1 = tok1->linkAt(-1);
+                if (Token::Match(tok1->previous(), "throw|noexcept")) {
+                    tok1 = tok1->previous();
+                    while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
+                        tok1 = tok1->previous();
+                    if (tok1->strAt(-1) != ")")
+                        return;
+                } else if (Token::Match(tok->tokAt(-2), ":|, %name%")) {
+                    tok1 = tok1->tokAt(-2);
+                    if (tok1->strAt(-1) != ")")
+                        return;
+                }
+                if (tok1->strAt(-1) == ">")
+                    tok1 = tok1->previous()->findOpeningBracket();
+                if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) {
+                    tok1 = tok1->tokAt(-2);
+                    std::string scope = tok1->strAt(-1);
+                    while (Token::Match(tok1->tokAt(-2), ":: %name%")) {
+                        scope = tok1->strAt(-3) + " :: " + scope;
+                        tok1 = tok1->tokAt(-2);
+                    }
+                    scopeInfo->emplace_back(scope, tok->link());
+                    added = true;
+                }
+            }
+
+            if (all && !added)
+                scopeInfo->emplace_back("", tok->link());
+        }
+        return;
+    }
+
+    tok = tok->next();
+    std::string classname = tok->str();
+    while (Token::Match(tok, "%name% :: %name%")) {
+        tok = tok->tokAt(2);
+        classname += " :: " + tok->str();
+    }
+    tok = tok->next();
+    if (tok && tok->str() == ":") {
+        while (tok && !Token::Match(tok, ";|{"))
+            tok = tok->next();
+    }
+    if (tok && tok->str() == "{") {
+        scopeInfo->emplace_back(classname,tok->link());
+    }
+}
+
 bool TemplateSimplifier::getTemplateDeclarations()
 {
     bool codeWithTemplates = false;
+    std::list<ScopeInfo2> scopeInfo;
     for (Token *tok = mTokenList.front(); tok; tok = tok->next()) {
+        if (Token::Match(tok, "{|}|namespace|class|struct|union")) {
+            setScopeInfo(tok, &scopeInfo);
+            continue;
+        }
         if (!Token::simpleMatch(tok, "template <"))
             continue;
         // ignore template template parameter
@@ -660,7 +756,7 @@ bool TemplateSimplifier::getTemplateDeclarations()
             else if (Token::Match(tok2, "{|=|;")) {
                 const int namepos = getTemplateNamePosition(parmEnd);
                 if (namepos > 0) {
-                    TokenAndName decl(tok, tok->scopeInfo()->name, parmEnd->tokAt(namepos), parmEnd);
+                    TokenAndName decl(tok, getScopeName(scopeInfo), parmEnd->tokAt(namepos), parmEnd);
                     if (decl.isForwardDeclaration()) {
                         // Declaration => add to mTemplateForwardDeclarations
                         mTemplateForwardDeclarations.emplace_back(decl);
@@ -697,9 +793,18 @@ void TemplateSimplifier::getTemplateInstantiations()
             functionNameMap.insert(std::make_pair(decl.name, &decl));
     }
 
+    std::list<ScopeInfo2> scopeList;
     const Token *skip = nullptr;
 
+    scopeList.emplace_back("", nullptr);
+
     for (Token *tok = mTokenList.front(); tok; tok = tok->next()) {
+        if (Token::Match(tok, "{|}|namespace|class|struct|union") ||
+            Token::Match(tok, "using namespace %name% ;|::")) {
+            setScopeInfo(tok, &scopeList);
+            continue;
+        }
+
         // template definition.. skip it
         if (Token::simpleMatch(tok, "template <")) {
             tok = tok->next()->findClosingBracket();
@@ -750,7 +855,7 @@ void TemplateSimplifier::getTemplateInstantiations()
         } else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|.|*|&|return|<|, %name% ::|<|(") ||
                    Token::Match(tok->previous(), "%type% %name% ::|<") ||
                    Token::Match(tok->tokAt(-2), "[,:] private|protected|public %name% ::|<")) {
-            std::string scopeName = tok->scopeInfo()->name;
+            std::string scopeName = getScopeName(scopeList);
             std::string qualification;
             Token * qualificationTok = tok;
             while (Token::Match(tok, "%name% :: %name%")) {
@@ -846,7 +951,7 @@ void TemplateSimplifier::getTemplateInstantiations()
             for (; tok2 && tok2 != tok; tok2 = tok2->previous()) {
                 if (Token::Match(tok2, ",|< %name% <") &&
                     (tok2->strAt(3) == ">" || templateParameters(tok2->tokAt(2)))) {
-                    addInstantiation(tok2->next(), tok->scopeInfo()->name);
+                    addInstantiation(tok2->next(), getScopeName(scopeList));
                 } else if (Token::Match(tok2->next(), "class|struct"))
                     tok2->deleteNext();
             }
@@ -865,7 +970,7 @@ void TemplateSimplifier::getTemplateInstantiations()
                     } else {
                         // full name doesn't match so try with using namespaces if available
                         bool found = false;
-                        for (const auto & nameSpace : tok->scopeInfo()->usingNamespaces) {
+                        for (const auto & nameSpace : scopeList.back().usingNamespaces) {
                             std::string fullNameSpace = scopeName + (scopeName.empty()?"":" :: ") +
                                                         nameSpace + (qualification.empty()?"":" :: ") + qualification;
                             std::string newFullName = fullNameSpace + " :: " + tok->str();
@@ -892,7 +997,7 @@ void TemplateSimplifier::getTemplateInstantiations()
                             if (!qualification.empty())
                                 addInstantiation(tok, qualification);
                             else
-                                addInstantiation(tok, tok->scopeInfo()->name);
+                                addInstantiation(tok, getScopeName(scopeList));
                             break;
                         }
                         const std::string::size_type pos = scopeName.rfind(" :: ");
@@ -1433,6 +1538,7 @@ void TemplateSimplifier::expandTemplate(
     const std::string &newName,
     bool copy)
 {
+    std::list<ScopeInfo2> scopeInfo;
     bool inTemplateDefinition = false;
     const Token *startOfTemplateDeclaration = nullptr;
     const Token *endOfTemplateDefinition = nullptr;
@@ -1647,6 +1753,10 @@ void TemplateSimplifier::expandTemplate(
     }
 
     for (Token *tok3 = mTokenList.front(); tok3; tok3 = tok3 ? tok3->next() : nullptr) {
+        if (Token::Match(tok3, "{|}|namespace|class|struct|union")) {
+            setScopeInfo(tok3, &scopeInfo);
+            continue;
+        }
         if (inTemplateDefinition) {
             if (!endOfTemplateDefinition) {
                 if (isVariable) {
@@ -2826,8 +2936,13 @@ void TemplateSimplifier::replaceTemplateUsage(
     const std::list<std::string> &typeStringsUsedInTemplateInstantiation,
     const std::string &newName)
 {
+    std::list<ScopeInfo2> scopeInfo;
     std::list< std::pair<Token *, Token *> > removeTokens;
     for (Token *nameTok = mTokenList.front(); nameTok; nameTok = nameTok->next()) {
+        if (Token::Match(nameTok, "{|}|namespace|class|struct|union")) {
+            setScopeInfo(nameTok, &scopeInfo);
+            continue;
+        }
         if (!Token::Match(nameTok, "%name% <") ||
             Token::Match(nameTok, "template|const_cast|dynamic_cast|reinterpret_cast|static_cast"))
             continue;
@@ -3255,10 +3370,6 @@ void TemplateSimplifier::simplifyTemplates(
     unsigned int passCount = 0;
     const unsigned int passCountMax = 10;
     for (; passCount < passCountMax; ++passCount) {
-        // Recalculate scopes from scratch every pass, in case a scope is missing or incorrect
-        for (auto tok = mTokenizer->list.front(); tok; tok = tok->next()) tok->scopeInfo(nullptr);
-        mTokenizer->calculateScopes();
-      
         if (passCount) {
             // it may take more than one pass to simplify type aliases
             bool usingChanged = false;

lib/token.cpp

@@ -990,84 +990,6 @@ void Token::insertToken(const std::string &tokenStr, const std::string &original
             this->next(newToken);
             newToken->previous(this);
         }
-    
-        // If the token we're inserting from has a scope, the new token needs one too
-        if (mImpl->mScopeInfo) {
-            // If it's a brace, we need to open a new scope
-            if (tokenStr == "{") {
-                std::string nextScopeNameAddition = "";
-                // This might be the opening of a member function
-                Token *tok1 = newToken;
-                while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
-                    tok1 = tok1->previous();
-                if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") {
-                    tok1 = tok1->linkAt(-1);
-                    if (Token::Match(tok1->previous(), "throw|noexcept")) {
-                        tok1 = tok1->previous();
-                        while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
-                            tok1 = tok1->previous();
-                        if (tok1->strAt(-1) != ")")
-                            return;
-                    } else if (Token::Match(newToken->tokAt(-2), ":|, %name%")) {
-                        tok1 = tok1->tokAt(-2);
-                        if (tok1->strAt(-1) != ")")
-                            return;
-                    }
-                    if (tok1->strAt(-1) == ">")
-                        tok1 = tok1->previous()->findOpeningBracket();
-                    if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) {
-                        tok1 = tok1->tokAt(-2);
-                        std::string scope = tok1->strAt(-1);
-                        while (Token::Match(tok1->tokAt(-2), ":: %name%")) {
-                            scope = tok1->strAt(-3) + " :: " + scope;
-                            tok1 = tok1->tokAt(-2);
-                        }
-    
-                        if (nextScopeNameAddition.length() > 0) nextScopeNameAddition += " :: ";
-                        nextScopeNameAddition += scope;
-                    }
-                }
-    
-                // Or it might be a namespace/class/struct
-                if (Token::Match(newToken->previous(), "%name%|>")) {
-                    Token* nameTok = newToken->previous();
-                    while (nameTok && !Token::Match(nameTok, "namespace|class|struct|union %name% {|::|:|<")) {
-                        nameTok = nameTok->previous();
-                    }
-                    if (nameTok) {
-                        for (nameTok = nameTok->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) {
-                            nextScopeNameAddition.append(nameTok->str());
-                            nextScopeNameAddition.append(" ");
-                        }
-                        if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1);
-                    }
-                }
-
-                // New scope is opening, record it here
-                std::shared_ptr<ScopeInfo2> newScopeInfo = std::make_shared<ScopeInfo2>(mImpl->mScopeInfo->name, nullptr, mImpl->mScopeInfo->usingNamespaces);
-
-                if (newScopeInfo->name != "") newScopeInfo->name.append(" :: ");
-                newScopeInfo->name.append(nextScopeNameAddition);
-                
-                newToken->scopeInfo(newScopeInfo);
-
-            // If it's a closing brace, we need to find where the scope opened and take the scope before
-            } else if (tokenStr == "}") {
-                Token* matchingTok = newToken->previous();
-                int depth = 0;
-                while (matchingTok && (depth != 0 || !Token::simpleMatch(matchingTok, "{"))) {
-                    if (Token::simpleMatch(matchingTok, "}")) depth++;
-                    if (Token::simpleMatch(matchingTok, "{")) depth--;
-                    matchingTok = matchingTok->previous();
-                }
-                if (matchingTok && matchingTok->previous()) {
-                    newToken->mImpl->mScopeInfo = matchingTok->previous()->scopeInfo();
-                }
-            // Otherwise we can just take the previous scope
-            } else {
-                newToken->mImpl->mScopeInfo = mImpl->mScopeInfo;
-            }
-        }
     }
 }
 
@@ -1924,13 +1846,6 @@ std::string Token::typeStr(const Token* tok)
     return r.first->stringifyList(r.second, false);
 }
 
-void Token::scopeInfo(std::shared_ptr<ScopeInfo2> newScopeInfo) {
-    mImpl->mScopeInfo = newScopeInfo;
-}
-std::shared_ptr<ScopeInfo2> Token::scopeInfo() const {
-    return mImpl->mScopeInfo;
-}
-
 TokenImpl::~TokenImpl()
 {
     delete mOriginalName;

lib/token.h

@@ -30,7 +30,6 @@
 #include <cstddef>
 #include <functional>
 #include <list>
-#include <memory>
 #include <ostream>
 #include <string>
 #include <vector>
@@ -51,17 +50,6 @@ struct TokensFrontBack {
     Token *back;
 };
 
-struct ScopeInfo2 {
-    ScopeInfo2(const std::string &name_, const Token *bodyEnd_, const std::set<std::string> &usingNamespaces_ = std::set<std::string>()) 
-        : name(name_), 
-        bodyEnd(bodyEnd_), 
-        usingNamespaces(usingNamespaces_) 
-        {}
-    std::string name;
-    const Token * const bodyEnd;
-    std::set<std::string> usingNamespaces;
-};
-
 struct TokenImpl {
     unsigned int mVarId;
     unsigned int mFileIndex;
@@ -109,8 +97,6 @@ struct TokenImpl {
     // Pointer to a template in the template simplifier
     std::set<TemplateSimplifier::TokenAndName*> mTemplateSimplifierPointers;
 
-    std::shared_ptr<ScopeInfo2> mScopeInfo;
-
     TokenImpl()
         : mVarId(0)
         , mFileIndex(0)
@@ -128,7 +114,6 @@ struct TokenImpl {
         , mValues(nullptr)
         , mBits(0)
         , mTemplateSimplifierPointers()
-        , mScopeInfo(nullptr)
     {}
 
     ~TokenImpl();
@@ -1183,10 +1168,6 @@ public:
     void printAst(bool verbose, bool xml, std::ostream &out) const;
 
     void printValueFlow(bool xml, std::ostream &out) const;
-    
-    void scopeInfo(std::shared_ptr<ScopeInfo2> newScopeInfo);
-    
-    std::shared_ptr<ScopeInfo2> scopeInfo() const;
 };
 
 /// @}

lib/tokenize.cpp

@@ -39,7 +39,6 @@
 #include <cstring>
 #include <ctime>
 #include <iostream>
-#include <set>
 #include <stack>
 #include <unordered_map>
 #include <utility>
@@ -2811,108 +2810,7 @@ void Tokenizer::simplifyCaseRange()
     }
 }
 
-void Tokenizer::calculateScopes()
-{
-    // Build up the next scope name as we go and then reset it when we use it
-    std::string nextScopeNameAddition = "";
-    if (!list.front()->scopeInfo()) {
-        std::shared_ptr<ScopeInfo2> primaryScope = std::make_shared<ScopeInfo2>("", list.back());
-        list.front()->scopeInfo(primaryScope);
 
-        if (Token::Match(list.front(), "using namespace %name% ::|<|;")) {
-            std::string usingNamespaceName = "";
-            for (const Token* namespaceNameToken = list.front()->tokAt(2); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) {
-                usingNamespaceName += namespaceNameToken->str();
-                usingNamespaceName += " ";
-            }
-            if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1);
-            list.front()->scopeInfo()->usingNamespaces.insert(usingNamespaceName);
-        } else if (Token::Match(list.front(), "namespace|class|struct|union %name% {|::|:|<")) {
-            for (Token* nameTok = list.front()->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) {
-                nextScopeNameAddition.append(nameTok->str());
-                nextScopeNameAddition.append(" ");
-            }
-            if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1);
-        }
-    }
-
-    for (Token* tok = list.front(); tok; tok = tok->next()) {
-        if (!tok->scopeInfo()) {
-            tok->scopeInfo(tok->previous()->scopeInfo());
-
-            if (Token::Match(tok, "using namespace %name% ::|<|;")) {
-                std::string usingNamespaceName = "";
-                for (const Token* namespaceNameToken = tok->tokAt(2); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) {
-                    usingNamespaceName += namespaceNameToken->str();
-                    usingNamespaceName += " ";
-                }
-                if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1);
-                tok->scopeInfo()->usingNamespaces.insert(usingNamespaceName);
-            } else if (Token::Match(tok, "namespace|class|struct|union %name% {|::|:|<")) {
-                if (Token::simpleMatch(tok->previous(), "using namespace")) {
-                    std::string usingNamespaceName = "";
-                    for (const Token* namespaceNameToken = tok->next(); !Token::simpleMatch(namespaceNameToken, ";"); namespaceNameToken = namespaceNameToken->next()) {
-                        usingNamespaceName += namespaceNameToken->str();
-                        usingNamespaceName += " ";
-                    }
-                    if (usingNamespaceName.length() > 0) usingNamespaceName = usingNamespaceName.substr(0, usingNamespaceName.length() - 1);
-                    tok->scopeInfo()->usingNamespaces.insert(usingNamespaceName);
-                } else {
-                    for (Token* nameTok = tok->next(); nameTok && !Token::Match(nameTok, "{|:|<"); nameTok = nameTok->next()) {
-                        nextScopeNameAddition.append(nameTok->str());
-                        nextScopeNameAddition.append(" ");
-                    }
-                    if (nextScopeNameAddition.length() > 0) nextScopeNameAddition = nextScopeNameAddition.substr(0, nextScopeNameAddition.length() - 1);
-                }
-            }
-
-            if (Token::simpleMatch(tok, "{"))
-            {
-                // This might be the opening of a member function
-                Token *tok1 = tok;
-                while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
-                    tok1 = tok1->previous();
-                if (tok1 && tok1->previous() && tok1->strAt(-1) == ")") {
-                    tok1 = tok1->linkAt(-1);
-                    if (Token::Match(tok1->previous(), "throw|noexcept")) {
-                        tok1 = tok1->previous();
-                        while (Token::Match(tok1->previous(), "const|volatile|final|override|&|&&|noexcept"))
-                            tok1 = tok1->previous();
-                        if (tok1->strAt(-1) != ")")
-                            return;
-                    } else if (Token::Match(tok->tokAt(-2), ":|, %name%")) {
-                        tok1 = tok1->tokAt(-2);
-                        if (tok1->strAt(-1) != ")")
-                            return;
-                    }
-                    if (tok1->strAt(-1) == ">")
-                        tok1 = tok1->previous()->findOpeningBracket();
-                    if (tok1 && Token::Match(tok1->tokAt(-3), "%name% :: %name%")) {
-                        tok1 = tok1->tokAt(-2);
-                        std::string scope = tok1->strAt(-1);
-                        while (Token::Match(tok1->tokAt(-2), ":: %name%")) {
-                            scope = tok1->strAt(-3) + " :: " + scope;
-                            tok1 = tok1->tokAt(-2);
-                        }
-
-                        if (nextScopeNameAddition.length() > 0) nextScopeNameAddition += " :: ";
-                        nextScopeNameAddition += scope;
-                    }
-                }
-
-                // New scope is opening, record it here
-                std::shared_ptr<ScopeInfo2> newScopeInfo = std::make_shared<ScopeInfo2>(tok->scopeInfo()->name, tok->link(), tok->scopeInfo()->usingNamespaces);
-
-                if (newScopeInfo->name != "") newScopeInfo->name.append(" :: ");
-                newScopeInfo->name.append(nextScopeNameAddition);
-                nextScopeNameAddition = "";
-
-                if (tok->link()) tok->link()->scopeInfo(tok->scopeInfo());
-                tok->scopeInfo(newScopeInfo);
-            }
-        }
-    }
-}
 
 void Tokenizer::simplifyTemplates()
 {
@@ -3529,6 +3427,12 @@ namespace {
         std::list<std::string> scope;
         Token *tok;
     };
+
+    struct ScopeInfo2 {
+        ScopeInfo2(const std::string &name_, const Token *bodyEnd_) : name(name_), bodyEnd(bodyEnd_) {}
+        const std::string name;
+        const Token * const bodyEnd;
+    };
 }
 
 static std::string getScopeName(const std::list<ScopeInfo2> &scopeInfo)

lib/tokenize.h

@@ -517,11 +517,6 @@ public:
 
     void findComplicatedSyntaxErrorsInTemplates();
 
-    /**
-     * Calculate every token's scope before simplifying templates
-     */
-    void calculateScopes();
-    
     /**
      * Simplify e.g. 'atol("0")' into '0'
      */