From 453800211cfc38d3af0d1f2e7ea26c25b4014944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Wed, 6 Dec 2023 22:01:00 +0100 Subject: [PATCH] testrunner: make sure that more redirects/outputs are actually being consumed (#5721) --- Makefile | 2 +- test/fixture.cpp | 17 ++++++++++++++++- test/fixture.h | 10 ++++++++-- test/testimportproject.cpp | 16 ++++++++++++++++ test/testprocessexecutor.cpp | 3 +-- test/testsingleexecutor.cpp | 5 ++--- test/testsuppressions.cpp | 8 +++++--- test/testthreadexecutor.cpp | 3 +-- 8 files changed, 50 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 8c698501d..d4e8ad837 100644 --- a/Makefile +++ b/Makefile @@ -758,7 +758,7 @@ test/testfunctions.o: test/testfunctions.cpp lib/addoninfo.h lib/check.h lib/che test/testgarbage.o: test/testgarbage.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h $(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testgarbage.cpp -test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h +test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h test/redirect.h $(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testimportproject.cpp test/testincompletestatement.o: test/testincompletestatement.cpp lib/addoninfo.h lib/check.h lib/checkother.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h test/fixture.h test/helpers.h diff --git a/test/fixture.cpp b/test/fixture.cpp index 32863bbc6..67d022bc3 100644 --- a/test/fixture.cpp +++ b/test/fixture.cpp @@ -114,6 +114,20 @@ bool TestFixture::prepareTest(const char testname[]) void TestFixture::teardownTest() { teardownTestInternal(); + + // TODO: enable + /* + { + const std::string s = errout.str(); + if (!s.empty()) + throw std::runtime_error("unconsumed ErrorLogger err: " + s); + } + */ + { + const std::string s = output_str(); + if (!s.empty()) + throw std::runtime_error("unconsumed ErrorLogger out: " + s); + } } std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const @@ -381,7 +395,7 @@ std::size_t TestFixture::runTests(const options& args) void TestFixture::reportOut(const std::string & outmsg, Color /*c*/) { - output << outmsg << std::endl; + mOutput << outmsg << std::endl; } void TestFixture::reportErr(const ErrorMessage &msg) @@ -389,6 +403,7 @@ void TestFixture::reportErr(const ErrorMessage &msg) if (msg.severity == Severity::none && msg.id == "logChecker") return; const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation)); + // TODO: remove the unique error handling? if (errout.str().find(errormessage) == std::string::npos) errout << errormessage << std::endl; } diff --git a/test/fixture.h b/test/fixture.h index dc1c62dcf..f5d94cd02 100644 --- a/test/fixture.h +++ b/test/fixture.h @@ -236,11 +236,17 @@ protected: return SettingsBuilder(*this, std::move(settings)); } - // TODO: make sure the output has been consumed in the test + std::string output_str() { + std::string s = mOutput.str(); + mOutput.str(""); + return s; + } + std::ostringstream errout; - std::ostringstream output; private: + std::ostringstream mOutput; + void reportOut(const std::string &outmsg, Color c = Color::Reset) override; void reportErr(const ErrorMessage &msg) override; void run(const std::string &str); diff --git a/test/testimportproject.cpp b/test/testimportproject.cpp index fb6a66707..df88f8ce1 100644 --- a/test/testimportproject.cpp +++ b/test/testimportproject.cpp @@ -20,6 +20,7 @@ #include "settings.h" #include "filesettings.h" #include "fixture.h" +#include "redirect.h" #include #include @@ -112,6 +113,7 @@ private: } void importCompileCommands1() const { + REDIRECT; constexpr char json[] = R"([{ "directory": "/tmp", "command": "gcc -DTEST1 -DTEST2=2 -o /tmp/src.o -c /tmp/src.c", @@ -125,6 +127,7 @@ private: } void importCompileCommands2() const { + REDIRECT; // Absolute file path #ifdef _WIN32 const char json[] = R"([{ @@ -152,6 +155,7 @@ private: } void importCompileCommands3() const { + REDIRECT; const char json[] = R"([{ "directory": "/tmp/", "command": "gcc -c src.c", @@ -165,6 +169,7 @@ private: } void importCompileCommands4() const { + REDIRECT; constexpr char json[] = R"([{ "directory": "/tmp/", "command": "gcc -c src.mm", @@ -177,6 +182,7 @@ private: } void importCompileCommands5() const { + REDIRECT; constexpr char json[] = R"([{ "directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug", @@ -196,6 +202,7 @@ private: } void importCompileCommands6() const { + REDIRECT; constexpr char json[] = R"([{ "directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug", @@ -217,6 +224,7 @@ private: void importCompileCommands7() const { + REDIRECT; // cmake -DFILESDIR="/some/path" .. constexpr char json[] = R"([{ @@ -237,6 +245,7 @@ private: } void importCompileCommands8() const { + REDIRECT; // cmake -DFILESDIR="C:\Program Files\Cppcheck" -G"NMake Makefiles" .. constexpr char json[] = R"([{ @@ -250,6 +259,7 @@ private: } void importCompileCommands9() const { + REDIRECT; // IAR output (https://sourceforge.net/p/cppcheck/discussion/general/thread/608af51e0a/) constexpr char json[] = R"([{ @@ -266,6 +276,7 @@ private: } void importCompileCommands10() const { // #10887 + REDIRECT; constexpr char json[] = R"([{ "file": "/home/danielm/cppcheck/1/test folder/1.c" , @@ -285,6 +296,7 @@ private: } void importCompileCommands11() const { // include path order + REDIRECT; constexpr char json[] = R"([{ "file": "1.c" , @@ -307,6 +319,7 @@ private: } void importCompileCommandsArgumentsSection() const { + REDIRECT; constexpr char json[] = "[ { \"directory\": \"/tmp/\"," "\"arguments\": [\"gcc\", \"-c\", \"src.c\"]," "\"file\": \"src.c\" } ]"; @@ -318,15 +331,18 @@ private: } void importCompileCommandsNoCommandSection() const { + REDIRECT; constexpr char json[] = "[ { \"directory\": \"/tmp/\"," "\"file\": \"src.mm\" } ]"; std::istringstream istr(json); TestImporter importer; ASSERT_EQUALS(false, importer.importCompileCommands(istr)); ASSERT_EQUALS(0, importer.fileSettings.size()); + ASSERT_EQUALS("cppcheck: error: no 'arguments' or 'command' field found in compilation database entry\n", GET_REDIRECT_OUTPUT); } void importCppcheckGuiProject() const { + REDIRECT; constexpr char xml[] = "\n" "\n" " \n" diff --git a/test/testprocessexecutor.cpp b/test/testprocessexecutor.cpp index 4e624c02e..59c5577e8 100644 --- a/test/testprocessexecutor.cpp +++ b/test/testprocessexecutor.cpp @@ -67,7 +67,6 @@ private: */ void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) { errout.str(""); - output.str(""); std::list fileSettings; @@ -265,7 +264,7 @@ private: $.executeCommandCalled = true, $.exe = exe, $.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}*/)); - ASSERT_EQUALS("Checking " + file + " ...\n", output.str()); + ASSERT_EQUALS("Checking " + file + " ...\n", output_str()); } // TODO: provide data which actually shows values above 0 diff --git a/test/testsingleexecutor.cpp b/test/testsingleexecutor.cpp index 53f9be99e..6ee535838 100644 --- a/test/testsingleexecutor.cpp +++ b/test/testsingleexecutor.cpp @@ -72,7 +72,6 @@ private: void check(int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) { errout.str(""); - output.str(""); std::list fileSettings; @@ -169,7 +168,7 @@ private: expected += "Checking " + fprefix() + "_" + zpad3(i) + ".cpp ...\n"; expected += std::to_string(i) + "/100 files checked " + std::to_string(i) + "% done\n"; } - ASSERT_EQUALS(expected, output.str()); + ASSERT_EQUALS(expected, output_str()); } void many_files_showtime() { @@ -259,7 +258,7 @@ private: $.executeCommandCalled = true, $.exe = exe, $.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"})); - ASSERT_EQUALS("Checking " + file + " ...\n", output.str()); + ASSERT_EQUALS("Checking " + file + " ...\n", output_str()); } // TODO: provide data which actually shows values above 0 diff --git a/test/testsuppressions.cpp b/test/testsuppressions.cpp index 2c8eccf06..2f91cbdde 100644 --- a/test/testsuppressions.cpp +++ b/test/testsuppressions.cpp @@ -200,7 +200,6 @@ private: unsigned int checkSuppression(std::map &f, const std::string &suppression = emptyString) { // Clear the error log errout.str(""); - output.str(""); std::list> files; for (std::map::const_iterator i = f.cbegin(); i != f.cend(); ++i) { @@ -210,6 +209,7 @@ private: CppCheck cppCheck(*this, true, nullptr); Settings& settings = cppCheck.settings(); settings.jobs = 1; + settings.quiet = true; settings.inlineSuppressions = true; settings.severity.enable(Severity::information); if (suppression == "unusedFunction") @@ -234,13 +234,13 @@ private: unsigned int checkSuppressionThreads(const char code[], const std::string &suppression = emptyString) { errout.str(""); - output.str(""); std::list> files; files.emplace_back("test.cpp", strlen(code)); Settings settings; settings.jobs = 2; + settings.quiet = true; settings.inlineSuppressions = true; settings.severity.enable(Severity::information); if (!suppression.empty()) { @@ -264,13 +264,13 @@ private: #if !defined(WIN32) && !defined(__MINGW32__) && !defined(__CYGWIN__) unsigned int checkSuppressionProcesses(const char code[], const std::string &suppression = emptyString) { errout.str(""); - output.str(""); std::list> files; files.emplace_back("test.cpp", strlen(code)); Settings settings; settings.jobs = 2; + settings.quiet = true; settings.inlineSuppressions = true; settings.severity.enable(Severity::information); if (!suppression.empty()) { @@ -1089,6 +1089,7 @@ private: CppCheck cppCheck(*this, false, nullptr); // <- do not "use global suppressions". pretend this is a thread that just checks a file. Settings& settings = cppCheck.settings(); + settings.quiet = true; settings.nomsg.addSuppressionLine("uninitvar"); settings.exitCode = 1; @@ -1123,6 +1124,7 @@ private: CppCheck cppCheck(*this, true, nullptr); Settings& settings = cppCheck.settings(); + settings.quiet = true; settings.severity.enable(Severity::style); settings.inlineSuppressions = true; settings.relativePaths = true; diff --git a/test/testthreadexecutor.cpp b/test/testthreadexecutor.cpp index f41def335..ea4e3a9e1 100644 --- a/test/testthreadexecutor.cpp +++ b/test/testthreadexecutor.cpp @@ -67,7 +67,6 @@ private: */ void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) { errout.str(""); - output.str(""); std::list fileSettings; @@ -262,7 +261,7 @@ private: $.executeCommandCalled = true, $.exe = exe, $.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"})); - ASSERT_EQUALS("Checking " + file + " ...\n", output.str()); + ASSERT_EQUALS("Checking " + file + " ...\n", output_str()); } // TODO: provide data which actually shows values above 0