allow token iteration in range for

lib/cppcheck.vcxproj

@@ -160,6 +160,7 @@
     <ClInclude Include="token.h" />
     <ClInclude Include="tokenize.h" />
     <ClInclude Include="tokenlist.h" />
+    <ClInclude Include="tokenrange.h" />
     <ClInclude Include="utils.h" />
     <ClInclude Include="valueflow.h" />
     <ClInclude Include="version.h" />

lib/cppcheck.vcxproj.filters

@@ -361,6 +361,9 @@
     <ClInclude Include="bughuntingchecks.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="tokenrange.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="errortypes.h">
       <Filter>Header Files</Filter>
     </ClInclude>

lib/token.cpp

@@ -24,6 +24,7 @@
 #include "symboldatabase.h"
 #include "tokenlist.h"
 #include "utils.h"
+#include "tokenrange.h"
 #include "valueflow.h"
 
 #include <algorithm>
@@ -55,6 +56,16 @@ Token::~Token()
     delete mImpl;
 }
 
+/*
+* Get a TokenRange which starts at this token and contains every token following it in order up to but not including 't'
+* e.g. for the sequence of tokens A B C D E, C.until(E) would yield the Range C D
+* note t can be nullptr to iterate all the way to the end.
+*/
+ConstTokenRange Token::until(const Token* t) const
+{
+    return ConstTokenRange(this, t);
+}
+
 static const std::unordered_set<std::string> controlFlowKeywords = {
     "goto",
     "do",

lib/token.h

@@ -44,6 +44,8 @@ class ValueType;
 class Variable;
 class TokenList;
 
+class ConstTokenRange;
+
 /**
  * @brief This struct stores pointers to the front and back tokens of the list this token is in.
  */
@@ -187,6 +189,8 @@ public:
     explicit Token(TokensFrontBack *tokensFrontBack = nullptr);
     ~Token();
 
+    ConstTokenRange until(const Token * t) const;
+
     template<typename T>
     void str(T&& s) {
         mStr = s;

lib/tokenrange.h

@@ -0,0 +1,66 @@
+/*
+ * Cppcheck - A tool for static C/C++ code analysis
+ * Copyright (C) 2007-2020 Cppcheck team.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+ //---------------------------------------------------------------------------
+#ifndef tokenrangeH
+#define tokenrangeH
+//---------------------------------------------------------------------------
+
+#include "config.h"
+
+template<typename T, REQUIRES("T must be a Token class", std::is_convertible<T*, const Token*>)>
+class TokenRangeBase
+{
+    T* mFront;
+    T* mBack;
+
+public:
+    TokenRangeBase(T* front, T* back) : mFront(front), mBack(back) {}
+
+    struct TokenIterator
+    {
+        using iterator_category = std::forward_iterator_tag;
+        using value_type = T*;
+        using difference_type = std::ptrdiff_t;
+        using pointer = void;
+        using reference = T*;
+
+        T* mt;
+        TokenIterator() : mt(nullptr) {}
+        explicit TokenIterator(T* t) : mt(t) {}
+        TokenIterator& operator++() { mt = mt->next(); return *this; }
+        bool operator==(const TokenIterator& b) const { return mt == b.mt; }
+        bool operator!=(const TokenIterator& b) const { return mt != b.mt; }
+        T* operator*() const { return mt; }
+    };
+
+    TokenIterator begin() const { return TokenIterator(mFront); }
+    TokenIterator end() const { return TokenIterator(mBack); }
+};
+
+class TokenRange : public TokenRangeBase<Token> {
+public:
+    TokenRange(Token* front, Token* back) : TokenRangeBase<Token>(front, back) {}
+};
+
+class ConstTokenRange : public TokenRangeBase<const Token> {
+public:
+    ConstTokenRange(const Token* front, const Token* back) : TokenRangeBase<const Token>(front, back) {}
+};
+
+#endif // tokenrangeH

test/testrunner.vcxproj

@@ -88,6 +88,7 @@
     <ClCompile Include="testtoken.cpp" />
     <ClCompile Include="testtokenize.cpp" />
     <ClCompile Include="testtokenlist.cpp" />
+    <ClCompile Include="testtokenrange.cpp" />
     <ClCompile Include="testtype.cpp" />
     <ClCompile Include="testuninitvar.cpp" />
     <ClCompile Include="testunusedfunctions.cpp" />

test/testrunner.vcxproj.filters

@@ -223,6 +223,9 @@
     <ClCompile Include="testsummaries.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="testtokenrange.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="options.h">

test/testtokenrange.cpp

@@ -0,0 +1,133 @@
+/*
+ * Cppcheck - A tool for static C/C++ code analysis
+ * Copyright (C) 2007-2020 Cppcheck team.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "settings.h"
+#include "testsuite.h"
+#include "testutils.h"
+#include "token.h"
+#include "tokenize.h"
+#include "tokenlist.h"
+#include "tokenrange.h"
+#include "symboldatabase.h"
+
+#include <string>
+#include <vector>
+
+struct InternalError;
+
+
+class TestTokenRange : public TestFixture {
+public:
+    TestTokenRange() : TestFixture("TestTokenRange") {
+    }
+
+private:
+    void run() OVERRIDE {
+        TEST_CASE(enumerationToEnd);
+        TEST_CASE(untilHelperToEnd);
+        TEST_CASE(untilHelperPartWay);
+        TEST_CASE(partialEnumeration);
+        TEST_CASE(scopeExample);
+        TEST_CASE(exampleAlgorithms);
+    }
+
+    std::string testTokenRange(ConstTokenRange range, const Token* start, const Token* end) const
+    {
+        auto tokenToString = [](const Token* t) { return t ? t->str() : "<null>"; };
+        int index = 0;
+        const Token* expected = start;
+        for (const Token* t : range)
+        {
+            if (expected != t) {
+                std::ostringstream message;
+                message << "Failed to match token " << tokenToString(expected) << " at position " << index << ". Got " << tokenToString(t) << " instead";
+                return message.str();
+            }
+            index++;
+            expected = expected->next();
+        }
+        if (expected != end) {
+            std::ostringstream message;
+            message << "Failed to terminate on " << tokenToString(end) << ". Instead terminated on " << tokenToString(expected) << " at position " << index << ".";
+            return message.str();
+        }
+        return "";
+    }
+
+    void enumerationToEnd() const {
+        std::istringstream istr("void a(){} void main(){ if(true){a();} }");
+        TokenList tokenList(nullptr);
+        tokenList.createTokens(istr, "test.cpp");
+        ASSERT_EQUALS("", testTokenRange(ConstTokenRange{ tokenList.front(), nullptr }, tokenList.front(), nullptr));
+    }
+
+    void untilHelperToEnd() const {
+        std::istringstream istr("void a(){} void main(){ if(true){a();} }");
+        TokenList tokenList(nullptr);
+        tokenList.createTokens(istr, "test.cpp");
+        ASSERT_EQUALS("", testTokenRange(tokenList.front()->until(nullptr), tokenList.front(), nullptr));
+    }
+
+    void untilHelperPartWay() const {
+        std::istringstream istr("void a(){} void main(){ if(true){a();} }");
+        TokenList tokenList(nullptr);
+        tokenList.createTokens(istr, "test.cpp");
+        const Token* start = tokenList.front()->tokAt(4);
+        const Token* end = start->tokAt(8);
+        ASSERT_EQUALS("", testTokenRange(start->until(end), start, end));
+    }
+
+    void partialEnumeration() const {
+        std::istringstream istr("void a(){} void main(){ if(true){a();} }");
+        TokenList tokenList(nullptr);
+        tokenList.createTokens(istr, "test.cpp");
+        const Token* start = tokenList.front()->tokAt(4);
+        const Token* end = tokenList.front()->tokAt(10);
+        ASSERT_EQUALS("", testTokenRange(ConstTokenRange{ start, end }, start, end));
+    }
+
+    void scopeExample() const {
+        Settings settings;
+        Tokenizer tokenizer{ &settings, nullptr };
+        std::istringstream sample("void a(){} void main(){ if(true){a();} }");
+        tokenizer.tokenize(sample, "test.cpp");
+
+        const SymbolDatabase* sd = tokenizer.getSymbolDatabase();
+        const Scope& scope = *std::next(sd->scopeList.begin(), 3); //The scope of the if block
+
+        std::ostringstream contents;
+        for (const Token* t : ConstTokenRange{ scope.bodyStart->next(), scope.bodyEnd })
+        {
+            contents << t->str();
+        }
+        ASSERT_EQUALS("a();", contents.str());
+    }
+
+    void exampleAlgorithms() const {
+        std::istringstream istr("void a(){} void main(){ if(true){a();} }");
+        TokenList tokenList(nullptr);
+        tokenList.createTokens(istr, "test.cpp");
+        ConstTokenRange range{ tokenList.front(), nullptr };
+        ASSERT_EQUALS(true, std::all_of(range.begin(), range.end(), [](const Token*) {return true;}));
+        ASSERT_EQUALS(true, std::any_of(range.begin(), range.end(), [](const Token* t) {return t->str() == "true";}));
+        ASSERT_EQUALS("true", (*std::find_if(range.begin(), range.end(), [](const Token* t) {return t->str() == "true";}))->str());
+        ASSERT_EQUALS(3, std::count_if(range.begin(), range.end(), [](const Token* t) {return t->str() == "{";}));
+    }
+};
+
+REGISTER_TEST(TestTokenRange)