testrunner: make sure that more redirects/outputs are actually being consumed (#5721)

Makefile

@@ -758,7 +758,7 @@ test/testfunctions.o: test/testfunctions.cpp lib/addoninfo.h lib/check.h lib/che
 test/testgarbage.o: test/testgarbage.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testgarbage.cpp
 
-test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h
+test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h test/redirect.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testimportproject.cpp
 
 test/testincompletestatement.o: test/testincompletestatement.cpp lib/addoninfo.h lib/check.h lib/checkother.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h test/fixture.h test/helpers.h

test/fixture.cpp

@@ -114,6 +114,20 @@ bool TestFixture::prepareTest(const char testname[])
 void TestFixture::teardownTest()
 {
     teardownTestInternal();
+
+    // TODO: enable
+    /*
+        {
+        const std::string s = errout.str();
+        if (!s.empty())
+            throw std::runtime_error("unconsumed ErrorLogger err: " + s);
+        }
+     */
+    {
+        const std::string s = output_str();
+        if (!s.empty())
+            throw std::runtime_error("unconsumed ErrorLogger out: " + s);
+    }
 }
 
 std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
@@ -381,7 +395,7 @@ std::size_t TestFixture::runTests(const options& args)
 
 void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
 {
-    output << outmsg << std::endl;
+    mOutput << outmsg << std::endl;
 }
 
 void TestFixture::reportErr(const ErrorMessage &msg)
@@ -389,6 +403,7 @@ void TestFixture::reportErr(const ErrorMessage &msg)
     if (msg.severity == Severity::none && msg.id == "logChecker")
         return;
     const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
+    // TODO: remove the unique error handling?
     if (errout.str().find(errormessage) == std::string::npos)
         errout << errormessage << std::endl;
 }

test/fixture.h

@@ -236,11 +236,17 @@ protected:
         return SettingsBuilder(*this, std::move(settings));
     }
 
-    // TODO: make sure the output has been consumed in the test
+    std::string output_str() {
+        std::string s = mOutput.str();
+        mOutput.str("");
+        return s;
+    }
+
     std::ostringstream errout;
-    std::ostringstream output;
 
 private:
+    std::ostringstream mOutput;
+
     void reportOut(const std::string &outmsg, Color c = Color::Reset) override;
     void reportErr(const ErrorMessage &msg) override;
     void run(const std::string &str);

test/testimportproject.cpp

@@ -20,6 +20,7 @@
 #include "settings.h"
 #include "filesettings.h"
 #include "fixture.h"
+#include "redirect.h"
 
 #include <list>
 #include <map>
@@ -112,6 +113,7 @@ private:
     }
 
     void importCompileCommands1() const {
+        REDIRECT;
         constexpr char json[] = R"([{
                                    "directory": "/tmp",
                                    "command": "gcc -DTEST1 -DTEST2=2 -o /tmp/src.o -c /tmp/src.c",
@@ -125,6 +127,7 @@ private:
     }
 
     void importCompileCommands2() const {
+        REDIRECT;
         // Absolute file path
 #ifdef _WIN32
         const char json[] = R"([{
@@ -152,6 +155,7 @@ private:
     }
 
     void importCompileCommands3() const {
+        REDIRECT;
         const char json[] = R"([{
                                     "directory": "/tmp/",
                                     "command": "gcc -c src.c",
@@ -165,6 +169,7 @@ private:
     }
 
     void importCompileCommands4() const {
+        REDIRECT;
         constexpr char json[] = R"([{
                                     "directory": "/tmp/",
                                     "command": "gcc -c src.mm",
@@ -177,6 +182,7 @@ private:
     }
 
     void importCompileCommands5() const {
+        REDIRECT;
         constexpr char json[] =
             R"([{
                 "directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug",
@@ -196,6 +202,7 @@ private:
     }
 
     void importCompileCommands6() const {
+        REDIRECT;
         constexpr char json[] =
             R"([{
                 "directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug",
@@ -217,6 +224,7 @@ private:
 
 
     void importCompileCommands7() const {
+        REDIRECT;
         // cmake -DFILESDIR="/some/path" ..
         constexpr char json[] =
             R"([{
@@ -237,6 +245,7 @@ private:
     }
 
     void importCompileCommands8() const {
+        REDIRECT;
         // cmake -DFILESDIR="C:\Program Files\Cppcheck" -G"NMake Makefiles" ..
         constexpr char json[] =
             R"([{
@@ -250,6 +259,7 @@ private:
     }
 
     void importCompileCommands9() const {
+        REDIRECT;
         // IAR output (https://sourceforge.net/p/cppcheck/discussion/general/thread/608af51e0a/)
         constexpr char json[] =
             R"([{
@@ -266,6 +276,7 @@ private:
     }
 
     void importCompileCommands10() const { // #10887
+        REDIRECT;
         constexpr char json[] =
             R"([{
                "file": "/home/danielm/cppcheck/1/test folder/1.c" ,
@@ -285,6 +296,7 @@ private:
     }
 
     void importCompileCommands11() const { // include path order
+        REDIRECT;
         constexpr char json[] =
             R"([{
                "file": "1.c" ,
@@ -307,6 +319,7 @@ private:
     }
 
     void importCompileCommandsArgumentsSection() const {
+        REDIRECT;
         constexpr char json[] = "[ { \"directory\": \"/tmp/\","
                                 "\"arguments\": [\"gcc\", \"-c\", \"src.c\"],"
                                 "\"file\": \"src.c\" } ]";
@@ -318,15 +331,18 @@ private:
     }
 
     void importCompileCommandsNoCommandSection() const {
+        REDIRECT;
         constexpr char json[] = "[ { \"directory\": \"/tmp/\","
                                 "\"file\": \"src.mm\" } ]";
         std::istringstream istr(json);
         TestImporter importer;
         ASSERT_EQUALS(false, importer.importCompileCommands(istr));
         ASSERT_EQUALS(0, importer.fileSettings.size());
+        ASSERT_EQUALS("cppcheck: error: no 'arguments' or 'command' field found in compilation database entry\n", GET_REDIRECT_OUTPUT);
     }
 
     void importCppcheckGuiProject() const {
+        REDIRECT;
         constexpr char xml[] = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                                "<project version=\"1\">\n"
                                "    <root name=\".\"/>\n"

test/testprocessexecutor.cpp

@@ -67,7 +67,6 @@ private:
      */
     void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
         errout.str("");
-        output.str("");
 
         std::list<FileSettings> fileSettings;
 
@@ -265,7 +264,7 @@ private:
                                               $.executeCommandCalled = true,
                                               $.exe = exe,
                                               $.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}*/));
-        ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
+        ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
     }
 
     // TODO: provide data which actually shows values above 0

test/testsingleexecutor.cpp

@@ -72,7 +72,6 @@ private:
 
     void check(int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
         errout.str("");
-        output.str("");
 
         std::list<FileSettings> fileSettings;
 
@@ -169,7 +168,7 @@ private:
             expected += "Checking " + fprefix() + "_" + zpad3(i) + ".cpp ...\n";
             expected += std::to_string(i) + "/100 files checked " + std::to_string(i) + "% done\n";
         }
-        ASSERT_EQUALS(expected, output.str());
+        ASSERT_EQUALS(expected, output_str());
     }
 
     void many_files_showtime() {
@@ -259,7 +258,7 @@ private:
                         $.executeCommandCalled = true,
                         $.exe = exe,
                         $.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}));
-        ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
+        ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
     }
 
 // TODO: provide data which actually shows values above 0

test/testsuppressions.cpp

@@ -200,7 +200,6 @@ private:
     unsigned int checkSuppression(std::map<std::string, std::string> &f, const std::string &suppression = emptyString) {
         // Clear the error log
         errout.str("");
-        output.str("");
 
         std::list<std::pair<std::string, std::size_t>> files;
         for (std::map<std::string, std::string>::const_iterator i = f.cbegin(); i != f.cend(); ++i) {
@@ -210,6 +209,7 @@ private:
         CppCheck cppCheck(*this, true, nullptr);
         Settings& settings = cppCheck.settings();
         settings.jobs = 1;
+        settings.quiet = true;
         settings.inlineSuppressions = true;
         settings.severity.enable(Severity::information);
         if (suppression == "unusedFunction")
@@ -234,13 +234,13 @@ private:
 
     unsigned int checkSuppressionThreads(const char code[], const std::string &suppression = emptyString) {
         errout.str("");
-        output.str("");
 
         std::list<std::pair<std::string, std::size_t>> files;
         files.emplace_back("test.cpp", strlen(code));
 
         Settings settings;
         settings.jobs = 2;
+        settings.quiet = true;
         settings.inlineSuppressions = true;
         settings.severity.enable(Severity::information);
         if (!suppression.empty()) {
@@ -264,13 +264,13 @@ private:
 #if !defined(WIN32) && !defined(__MINGW32__) && !defined(__CYGWIN__)
     unsigned int checkSuppressionProcesses(const char code[], const std::string &suppression = emptyString) {
         errout.str("");
-        output.str("");
 
         std::list<std::pair<std::string, std::size_t>> files;
         files.emplace_back("test.cpp", strlen(code));
 
         Settings settings;
         settings.jobs = 2;
+        settings.quiet = true;
         settings.inlineSuppressions = true;
         settings.severity.enable(Severity::information);
         if (!suppression.empty()) {
@@ -1089,6 +1089,7 @@ private:
 
         CppCheck cppCheck(*this, false, nullptr); // <- do not "use global suppressions". pretend this is a thread that just checks a file.
         Settings& settings = cppCheck.settings();
+        settings.quiet = true;
         settings.nomsg.addSuppressionLine("uninitvar");
         settings.exitCode = 1;
 
@@ -1123,6 +1124,7 @@ private:
 
         CppCheck cppCheck(*this, true, nullptr);
         Settings& settings = cppCheck.settings();
+        settings.quiet = true;
         settings.severity.enable(Severity::style);
         settings.inlineSuppressions = true;
         settings.relativePaths = true;

test/testthreadexecutor.cpp

@@ -67,7 +67,6 @@ private:
      */
     void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
         errout.str("");
-        output.str("");
 
         std::list<FileSettings> fileSettings;
 
@@ -262,7 +261,7 @@ private:
                         $.executeCommandCalled = true,
                         $.exe = exe,
                         $.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}));
-        ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
+        ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
     }
 
     // TODO: provide data which actually shows values above 0