From a5664c3e4900f46df5ad774bdc2181ebdb4dc56c Mon Sep 17 00:00:00 2001 From: shane Date: Mon, 14 Jun 2021 07:39:01 +0200 Subject: [PATCH] allow token iteration in range for --- lib/cppcheck.vcxproj | 1 + lib/cppcheck.vcxproj.filters | 3 + lib/token.cpp | 11 +++ lib/token.h | 4 + lib/tokenrange.h | 66 ++++++++++++++++ test/testrunner.vcxproj | 1 + test/testrunner.vcxproj.filters | 3 + test/testtokenrange.cpp | 133 ++++++++++++++++++++++++++++++++ 8 files changed, 222 insertions(+) create mode 100644 lib/tokenrange.h create mode 100644 test/testtokenrange.cpp diff --git a/lib/cppcheck.vcxproj b/lib/cppcheck.vcxproj index 2c40cfe10..8d44b55e9 100644 --- a/lib/cppcheck.vcxproj +++ b/lib/cppcheck.vcxproj @@ -160,6 +160,7 @@ + diff --git a/lib/cppcheck.vcxproj.filters b/lib/cppcheck.vcxproj.filters index 0fe164bf6..018a50ca6 100644 --- a/lib/cppcheck.vcxproj.filters +++ b/lib/cppcheck.vcxproj.filters @@ -361,6 +361,9 @@ Header Files + + Header Files + Header Files diff --git a/lib/token.cpp b/lib/token.cpp index 081c591a2..0f9ee5400 100644 --- a/lib/token.cpp +++ b/lib/token.cpp @@ -24,6 +24,7 @@ #include "symboldatabase.h" #include "tokenlist.h" #include "utils.h" +#include "tokenrange.h" #include "valueflow.h" #include @@ -55,6 +56,16 @@ Token::~Token() delete mImpl; } +/* +* Get a TokenRange which starts at this token and contains every token following it in order up to but not including 't' +* e.g. for the sequence of tokens A B C D E, C.until(E) would yield the Range C D +* note t can be nullptr to iterate all the way to the end. +*/ +ConstTokenRange Token::until(const Token* t) const +{ + return ConstTokenRange(this, t); +} + static const std::unordered_set controlFlowKeywords = { "goto", "do", diff --git a/lib/token.h b/lib/token.h index 0cc3e1f32..bbdad70f9 100644 --- a/lib/token.h +++ b/lib/token.h @@ -44,6 +44,8 @@ class ValueType; class Variable; class TokenList; +class ConstTokenRange; + /** * @brief This struct stores pointers to the front and back tokens of the list this token is in. */ @@ -187,6 +189,8 @@ public: explicit Token(TokensFrontBack *tokensFrontBack = nullptr); ~Token(); + ConstTokenRange until(const Token * t) const; + template void str(T&& s) { mStr = s; diff --git a/lib/tokenrange.h b/lib/tokenrange.h new file mode 100644 index 000000000..1c25cbaf1 --- /dev/null +++ b/lib/tokenrange.h @@ -0,0 +1,66 @@ +/* + * Cppcheck - A tool for static C/C++ code analysis + * Copyright (C) 2007-2020 Cppcheck team. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + //--------------------------------------------------------------------------- +#ifndef tokenrangeH +#define tokenrangeH +//--------------------------------------------------------------------------- + +#include "config.h" + +template)> +class TokenRangeBase +{ + T* mFront; + T* mBack; + +public: + TokenRangeBase(T* front, T* back) : mFront(front), mBack(back) {} + + struct TokenIterator + { + using iterator_category = std::forward_iterator_tag; + using value_type = T*; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = T*; + + T* mt; + TokenIterator() : mt(nullptr) {} + explicit TokenIterator(T* t) : mt(t) {} + TokenIterator& operator++() { mt = mt->next(); return *this; } + bool operator==(const TokenIterator& b) const { return mt == b.mt; } + bool operator!=(const TokenIterator& b) const { return mt != b.mt; } + T* operator*() const { return mt; } + }; + + TokenIterator begin() const { return TokenIterator(mFront); } + TokenIterator end() const { return TokenIterator(mBack); } +}; + +class TokenRange : public TokenRangeBase { +public: + TokenRange(Token* front, Token* back) : TokenRangeBase(front, back) {} +}; + +class ConstTokenRange : public TokenRangeBase { +public: + ConstTokenRange(const Token* front, const Token* back) : TokenRangeBase(front, back) {} +}; + +#endif // tokenrangeH diff --git a/test/testrunner.vcxproj b/test/testrunner.vcxproj index fb7be2815..ac68fca33 100755 --- a/test/testrunner.vcxproj +++ b/test/testrunner.vcxproj @@ -88,6 +88,7 @@ + diff --git a/test/testrunner.vcxproj.filters b/test/testrunner.vcxproj.filters index 5e074f7f2..5d7a236d4 100644 --- a/test/testrunner.vcxproj.filters +++ b/test/testrunner.vcxproj.filters @@ -223,6 +223,9 @@ Source Files + + Source Files + diff --git a/test/testtokenrange.cpp b/test/testtokenrange.cpp new file mode 100644 index 000000000..abe2dcbb1 --- /dev/null +++ b/test/testtokenrange.cpp @@ -0,0 +1,133 @@ +/* + * Cppcheck - A tool for static C/C++ code analysis + * Copyright (C) 2007-2020 Cppcheck team. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "settings.h" +#include "testsuite.h" +#include "testutils.h" +#include "token.h" +#include "tokenize.h" +#include "tokenlist.h" +#include "tokenrange.h" +#include "symboldatabase.h" + +#include +#include + +struct InternalError; + + +class TestTokenRange : public TestFixture { +public: + TestTokenRange() : TestFixture("TestTokenRange") { + } + +private: + void run() OVERRIDE { + TEST_CASE(enumerationToEnd); + TEST_CASE(untilHelperToEnd); + TEST_CASE(untilHelperPartWay); + TEST_CASE(partialEnumeration); + TEST_CASE(scopeExample); + TEST_CASE(exampleAlgorithms); + } + + std::string testTokenRange(ConstTokenRange range, const Token* start, const Token* end) const + { + auto tokenToString = [](const Token* t) { return t ? t->str() : ""; }; + int index = 0; + const Token* expected = start; + for (const Token* t : range) + { + if (expected != t) { + std::ostringstream message; + message << "Failed to match token " << tokenToString(expected) << " at position " << index << ". Got " << tokenToString(t) << " instead"; + return message.str(); + } + index++; + expected = expected->next(); + } + if (expected != end) { + std::ostringstream message; + message << "Failed to terminate on " << tokenToString(end) << ". Instead terminated on " << tokenToString(expected) << " at position " << index << "."; + return message.str(); + } + return ""; + } + + void enumerationToEnd() const { + std::istringstream istr("void a(){} void main(){ if(true){a();} }"); + TokenList tokenList(nullptr); + tokenList.createTokens(istr, "test.cpp"); + ASSERT_EQUALS("", testTokenRange(ConstTokenRange{ tokenList.front(), nullptr }, tokenList.front(), nullptr)); + } + + void untilHelperToEnd() const { + std::istringstream istr("void a(){} void main(){ if(true){a();} }"); + TokenList tokenList(nullptr); + tokenList.createTokens(istr, "test.cpp"); + ASSERT_EQUALS("", testTokenRange(tokenList.front()->until(nullptr), tokenList.front(), nullptr)); + } + + void untilHelperPartWay() const { + std::istringstream istr("void a(){} void main(){ if(true){a();} }"); + TokenList tokenList(nullptr); + tokenList.createTokens(istr, "test.cpp"); + const Token* start = tokenList.front()->tokAt(4); + const Token* end = start->tokAt(8); + ASSERT_EQUALS("", testTokenRange(start->until(end), start, end)); + } + + void partialEnumeration() const { + std::istringstream istr("void a(){} void main(){ if(true){a();} }"); + TokenList tokenList(nullptr); + tokenList.createTokens(istr, "test.cpp"); + const Token* start = tokenList.front()->tokAt(4); + const Token* end = tokenList.front()->tokAt(10); + ASSERT_EQUALS("", testTokenRange(ConstTokenRange{ start, end }, start, end)); + } + + void scopeExample() const { + Settings settings; + Tokenizer tokenizer{ &settings, nullptr }; + std::istringstream sample("void a(){} void main(){ if(true){a();} }"); + tokenizer.tokenize(sample, "test.cpp"); + + const SymbolDatabase* sd = tokenizer.getSymbolDatabase(); + const Scope& scope = *std::next(sd->scopeList.begin(), 3); //The scope of the if block + + std::ostringstream contents; + for (const Token* t : ConstTokenRange{ scope.bodyStart->next(), scope.bodyEnd }) + { + contents << t->str(); + } + ASSERT_EQUALS("a();", contents.str()); + } + + void exampleAlgorithms() const { + std::istringstream istr("void a(){} void main(){ if(true){a();} }"); + TokenList tokenList(nullptr); + tokenList.createTokens(istr, "test.cpp"); + ConstTokenRange range{ tokenList.front(), nullptr }; + ASSERT_EQUALS(true, std::all_of(range.begin(), range.end(), [](const Token*) {return true;})); + ASSERT_EQUALS(true, std::any_of(range.begin(), range.end(), [](const Token* t) {return t->str() == "true";})); + ASSERT_EQUALS("true", (*std::find_if(range.begin(), range.end(), [](const Token* t) {return t->str() == "true";}))->str()); + ASSERT_EQUALS(3, std::count_if(range.begin(), range.end(), [](const Token* t) {return t->str() == "{";})); + } +}; + +REGISTER_TEST(TestTokenRange)