testrunner: make sure all redirected output is being consumed / some cleanups (#5714)

test/fixture.cpp

@@ -34,9 +34,6 @@
 
 #include "xml.h"
 
-std::ostringstream errout;
-std::ostringstream output;
-
 /**
  * TestRegistry
  **/

test/fixture.h

@@ -236,10 +236,16 @@ protected:
         return SettingsBuilder(*this, std::move(settings));
     }
 
-public:
+    // TODO: make sure the output has been consumed in the test
+    std::ostringstream errout;
+    std::ostringstream output;
+
+private:
     void reportOut(const std::string &outmsg, Color c = Color::Reset) override;
     void reportErr(const ErrorMessage &msg) override;
     void run(const std::string &str);
+
+public:
     static void printHelp();
     const std::string classname;
 
@@ -248,12 +254,6 @@ public:
     static std::size_t runTests(const options& args);
 };
 
-// TODO: fix these
-// NOLINTNEXTLINE(readability-redundant-declaration)
-extern std::ostringstream errout;
-// NOLINTNEXTLINE(readability-redundant-declaration)
-extern std::ostringstream output;
-
 // TODO: most asserts do not actually assert i.e. do not return
 #define TEST_CASE( NAME )  do { if (prepareTest(#NAME)) { setVerbose(false); NAME(); teardownTest(); } } while (false)
 #define ASSERT( CONDITION )  if (!assert_(__FILE__, __LINE__, (CONDITION))) return

test/redirect.h

@@ -19,12 +19,9 @@
 
 #include <iostream>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 
-// NOLINTNEXTLINE(readability-redundant-declaration) - TODO: fix this
-extern std::ostringstream errout;
-// NOLINTNEXTLINE(readability-redundant-declaration) - TODO: fix this
-extern std::ostringstream output;
 /**
  * @brief Utility class for capturing cout and cerr to ostringstream buffers
  * for later use. Uses RAII to stop redirection when the object goes out of
@@ -47,34 +44,35 @@ public:
     }
 
     /** Revert cout and cerr behaviour */
-    ~RedirectOutputError() {
+    ~RedirectOutputError() noexcept(false) {
         std::cout.rdbuf(_oldCout); // restore cout's original streambuf
         std::cerr.rdbuf(_oldCerr); // restore cerrs's original streambuf
 
-        errout << _err.str();
+        {
-        output << _out.str();
+            const std::string s = _out.str();
+            if (!s.empty())
+                throw std::runtime_error("unconsumed stdout: " + s); // cppcheck-suppress exceptThrowInDestructor - FP #11031
+        }
+
+        {
+            const std::string s = _err.str();
+            if (!s.empty())
+                throw std::runtime_error("consumed stderr: " + s);
+        }
     }
 
-    /** Return what would be printed to cout. See also clearOutput() */
+    /** Return what would be printed to cout. */
-    std::string getOutput() const {
+    std::string getOutput() {
-        return _out.str();
+        std::string s = _out.str();
-    }
-
-    /** Normally called after getOutput() to prevent same text to be returned
-       twice. */
-    void clearOutput() {
         _out.str("");
+        return s;
     }
 
-    /** Return what would be printed to cerr. See also clearErrout() */
+    /** Return what would be printed to cerr. */
-    std::string getErrout() const {
+    std::string getErrout() {
-        return _err.str();
+        std::string s = _err.str();
-    }
-
-    /** Normally called after getErrout() to prevent same text to be returned
-       twice. */
-    void clearErrout() {
         _err.str("");
+        return s;
     }
 
 private:
@@ -112,9 +110,7 @@ private:
 
 #define REDIRECT RedirectOutputError redir
 #define GET_REDIRECT_OUTPUT redir.getOutput()
-#define CLEAR_REDIRECT_OUTPUT redir.clearOutput()
 #define GET_REDIRECT_ERROUT redir.getErrout()
-#define CLEAR_REDIRECT_ERROUT redir.clearErrout()
 
 #define SUPPRESS SuppressOutput supprout
 

test/testcmdlineparser.cpp

@@ -102,7 +102,6 @@ private:
 
     void teardownTestInternal() override {
         logger->destroy();
-        // TODO: verify that the redirect output is empty
     }
 
     // add overload so the enums can be compared without a cast
@@ -373,7 +372,6 @@ private:
         const char * const argv[] = {"cppcheck"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(1, argv));
         ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis"));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void helpshort() {
@@ -381,7 +379,6 @@ private:
         const char * const argv[] = {"cppcheck", "-h"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv));
         ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis"));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void helpshortExclusive() {
@@ -389,7 +386,6 @@ private:
         const char * const argv[] = {"cppcheck", "--library=missing", "-h"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv));
         ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis"));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void helplong() {
@@ -397,7 +393,6 @@ private:
         const char * const argv[] = {"cppcheck", "--help"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv));
         ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis"));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void helplongExclusive() {
@@ -405,7 +400,6 @@ private:
         const char * const argv[] = {"cppcheck", "--library=missing", "--help"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv));
         ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis"));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void version() {
@@ -413,7 +407,6 @@ private:
         const char * const argv[] = {"cppcheck", "--version"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS("Cppcheck 2.13 dev\n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void versionWithCfg() {
@@ -426,7 +419,6 @@ private:
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv));
         // TODO: somehow the config is not loaded on some systems
         (void)logger->str(); //ASSERT_EQUALS("The Product\n", logger->str()); // TODO: include version?
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     // TODO: test --version with extraVersion
@@ -436,7 +428,6 @@ private:
         const char * const argv[] = {"cppcheck", "--library=missing", "--version"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS("Cppcheck 2.13 dev\n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void versionWithInvalidCfg() {
@@ -446,7 +437,6 @@ private:
         const char * const argv[] = {"cppcheck", "--version"};
         ASSERT_EQUALS(CmdLineParser::Result::Fail, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS("cppcheck: error: could not load cppcheck.cfg - not a valid JSON - syntax error at line 1 near: \n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void onefile() {
@@ -455,7 +445,6 @@ private:
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS(1, (int)parser->getPathNames().size());
         ASSERT_EQUALS("file.cpp", parser->getPathNames().at(0));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void onepath() {
@@ -464,7 +453,6 @@ private:
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS(1, (int)parser->getPathNames().size());
         ASSERT_EQUALS("src", parser->getPathNames().at(0));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void optionwithoutfile() {
@@ -473,7 +461,6 @@ private:
         ASSERT_EQUALS(CmdLineParser::Result::Fail, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS(0, (int)parser->getPathNames().size());
         ASSERT_EQUALS("cppcheck: error: no C or C++ source files found.\n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void verboseshort() {
@@ -482,7 +469,6 @@ private:
         settings->verbose = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->verbose);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void verboselong() {
@@ -491,7 +477,6 @@ private:
         settings->verbose = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->verbose);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void debugSimplified() {
@@ -500,7 +485,6 @@ private:
         settings->debugSimplified = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->debugSimplified);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void debugwarnings() {
@@ -509,7 +493,6 @@ private:
         settings->debugwarnings = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->debugwarnings);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void forceshort() {
@@ -518,7 +501,6 @@ private:
         settings->force = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->force);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void forcelong() {
@@ -527,7 +509,6 @@ private:
         settings->force = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->force);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void relativePaths1() {
@@ -536,7 +517,6 @@ private:
         const char * const argv[] = {"cppcheck", "-rp", "file.cpp"};
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->relativePaths);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void relativePaths2() {
@@ -545,7 +525,6 @@ private:
         const char * const argv[] = {"cppcheck", "--relative-paths", "file.cpp"};
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->relativePaths);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void relativePaths3() {
@@ -558,7 +537,6 @@ private:
         ASSERT_EQUALS(2, settings->basePaths.size());
         ASSERT_EQUALS("C:/foo", settings->basePaths[0]);
         ASSERT_EQUALS("C:/bar", settings->basePaths[1]);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void relativePaths4() {
@@ -572,7 +550,6 @@ private:
         ASSERT_EQUALS(2, settings->basePaths.size());
         ASSERT_EQUALS("C:/foo", settings->basePaths[0]);
         ASSERT_EQUALS("C:/bar", settings->basePaths[1]);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void quietshort() {
@@ -581,7 +558,6 @@ private:
         settings->quiet = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->quiet);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void quietlong() {
@@ -590,7 +566,6 @@ private:
         settings->quiet = false;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(true, settings->quiet);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void defines_noarg() {
@@ -623,7 +598,6 @@ private:
         settings->userDefines.clear();
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS("_WIN32=1", settings->userDefines);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void defines2() {
@@ -632,7 +606,6 @@ private:
         settings->userDefines.clear();
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv));
         ASSERT_EQUALS("_WIN32=1;NODEBUG=1", settings->userDefines);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void defines3() {
@@ -641,7 +614,6 @@ private:
         settings->userDefines.clear();
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv));
         ASSERT_EQUALS("DEBUG=1", settings->userDefines);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void defines4() {
@@ -650,7 +622,6 @@ private:
         settings->userDefines.clear();
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS("DEBUG=", settings->userDefines);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void enforceLanguage1() {
@@ -1619,7 +1590,6 @@ private:
         const char * const argv[] = {"cppcheck", "--doc"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv));
         ASSERT(startsWith(logger->str(), "## "));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void docExclusive() {
@@ -1627,7 +1597,6 @@ private:
         const char * const argv[] = {"cppcheck", "--library=missing", "--doc"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv));
         ASSERT(startsWith(logger->str(), "## "));
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void showtimeSummary() {
@@ -1706,8 +1675,9 @@ private:
         const char * const argv[] = {"cppcheck", "--errorlist"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS("", logger->str()); // empty since it is logged via ErrorLogger
-        ASSERT(startsWith(GET_REDIRECT_OUTPUT, ErrorMessage::getXMLHeader("")));
+        const std::string errout_s = GET_REDIRECT_OUTPUT;
-        ASSERT(endsWith(GET_REDIRECT_OUTPUT, "</results>\n"));
+        ASSERT(startsWith(errout_s, ErrorMessage::getXMLHeader("")));
+        ASSERT(endsWith(errout_s, "</results>\n"));
     }
 
     void errorlistWithCfg() {
@@ -1725,8 +1695,9 @@ private:
         const char * const argv[] = {"cppcheck", "--library=missing", "--errorlist"};
         ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS("", logger->str()); // empty since it is logged via ErrorLogger
-        ASSERT(startsWith(GET_REDIRECT_OUTPUT, "<?xml"));
+        const std::string errout_s = GET_REDIRECT_OUTPUT;
-        ASSERT(endsWith(GET_REDIRECT_OUTPUT, "</results>\n"));
+        ASSERT(startsWith(errout_s, ErrorMessage::getXMLHeader("")));
+        ASSERT(endsWith(errout_s, "</results>\n"));
     }
 
     void errorlistWithInvalidCfg() {
@@ -1736,7 +1707,6 @@ private:
         const char * const argv[] = {"cppcheck", "--errorlist"};
         ASSERT_EQUALS(CmdLineParser::Result::Fail, parser->parseFromArgs(2, argv));
         ASSERT_EQUALS("cppcheck: error: could not load cppcheck.cfg - not a valid JSON - syntax error at line 1 near: \n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void ignorepathsnopath() {
@@ -1998,7 +1968,6 @@ private:
         settings->typedefMaxTime = 0;
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(12, settings->typedefMaxTime);
-        ASSERT_EQUALS("", logger->str());
     }
 
     void typedefMaxTimeInvalid() {
@@ -2074,7 +2043,6 @@ private:
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(1, settings->addons.size());
         ASSERT_EQUALS("misra", *settings->addons.cbegin());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void addonMissing() {
@@ -2085,7 +2053,6 @@ private:
         ASSERT_EQUALS(1, settings->addons.size());
         ASSERT_EQUALS("misra2", *settings->addons.cbegin());
         ASSERT_EQUALS("Did not find addon misra2.py\n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void signedChar() {
@@ -2094,7 +2061,6 @@ private:
         settings->platform.defaultSign = '\0';
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS('s', settings->platform.defaultSign);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void signedChar2() {
@@ -2103,7 +2069,6 @@ private:
         settings->platform.defaultSign = '\0';
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv));
         ASSERT_EQUALS('s', settings->platform.defaultSign);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void unsignedChar() {
@@ -2112,7 +2077,6 @@ private:
         settings->platform.defaultSign = '\0';
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS('u', settings->platform.defaultSign);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void unsignedChar2() {
@@ -2121,7 +2085,6 @@ private:
         settings->platform.defaultSign = '\0';
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv));
         ASSERT_EQUALS('u', settings->platform.defaultSign);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void signedCharUnsignedChar() {
@@ -2130,7 +2093,6 @@ private:
         settings->platform.defaultSign = '\0';
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv));
         ASSERT_EQUALS('u', settings->platform.defaultSign);
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
 #ifdef HAVE_RULES
@@ -2204,7 +2166,6 @@ private:
         ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv));
         ASSERT_EQUALS(1, settings->libraries.size());
         ASSERT_EQUALS("posix", *settings->libraries.cbegin());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void libraryMissing() {
@@ -2215,7 +2176,6 @@ private:
         ASSERT_EQUALS(1, settings->libraries.size());
         ASSERT_EQUALS("posix2", *settings->libraries.cbegin());
         ASSERT_EQUALS("cppcheck: Failed to load library configuration file 'posix2'. File not found\n", logger->str());
-        ASSERT_EQUALS("", GET_REDIRECT_OUTPUT);
     }
 
     void ignorepaths1() {

test/testsymboldatabase.cpp

@@ -76,7 +76,7 @@ private:
         typetok = nullptr;
     }
 
-    const static SymbolDatabase* getSymbolDB_inner(Tokenizer& tokenizer, const char* code, const char* filename) {
+    const SymbolDatabase* getSymbolDB_inner(Tokenizer& tokenizer, const char* code, const char* filename) {
         errout.str("");
         std::istringstream istr(code);
         return tokenizer.tokenize(istr, filename) ? tokenizer.getSymbolDatabase() : nullptr;

test/testtokenlist.cpp

@@ -57,7 +57,7 @@ private:
         ASSERT_EQUALS("0xF0000000", tokenlist.front()->str());
     }
 
-    void inc() const {
+    void inc() {
         const char code[] = "a++1;1++b;";
 
         errout.str("");