/* * Cppcheck - A tool for static C/C++ code analysis * Copyright (C) 2007-2023 Cppcheck team. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "fixture.h" #include "cppcheck.h" #include "errortypes.h" #include "options.h" #include "redirect.h" #include #include #include #include #include #include #include #include #include "xml.h" /** * TestRegistry **/ namespace { struct CompareFixtures { bool operator()(const TestFixture* lhs, const TestFixture* rhs) const { return lhs->classname < rhs->classname; } }; } using TestSet = std::set; namespace { class TestRegistry { TestSet _tests; public: static TestRegistry &theInstance() { static TestRegistry testreg; return testreg; } void addTest(TestFixture *t) { _tests.insert(t); } const TestSet &tests() const { return _tests; } }; } /** * TestFixture **/ std::ostringstream TestFixture::errmsg; unsigned int TestFixture::countTests; std::size_t TestFixture::fails_counter = 0; std::size_t TestFixture::todos_counter = 0; std::size_t TestFixture::succeeded_todos_counter = 0; TestFixture::TestFixture(const char * const _name) : classname(_name) { TestRegistry::theInstance().addTest(this); } bool TestFixture::prepareTest(const char testname[]) { mVerbose = false; mTemplateFormat.clear(); mTemplateLocation.clear(); CppCheck::resetTimerResults(); prepareTestInternal(); // Check if tests should be executed if (testToRun.empty() || testToRun == testname) { // Tests will be executed - prepare them mTestname = testname; ++countTests; if (quiet_tests) { std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr std::fflush(stdout); } else { std::cout << classname << "::" << mTestname << std::endl; } return true; } return false; } void TestFixture::teardownTest() { teardownTestInternal(); // TODO: enable /* { const std::string s = errout.str(); if (!s.empty()) throw std::runtime_error("unconsumed ErrorLogger err: " + s); } */ { const std::string s = output_str(); if (!s.empty()) throw std::runtime_error("unconsumed ErrorLogger out: " + s); } } std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const { return std::string(filename) + ':' + std::to_string(linenr) + '(' + classname + "::" + mTestname + ')'; } static std::string writestr(const std::string &str, bool gccStyle = false) { std::ostringstream ostr; if (gccStyle) ostr << '\"'; for (std::string::const_iterator i = str.cbegin(); i != str.cend(); ++i) { if (*i == '\n') { ostr << "\\n"; if ((i+1) != str.end() && !gccStyle) ostr << std::endl; } else if (*i == '\t') ostr << "\\t"; else if (*i == '\"') ostr << "\\\""; else if (std::isprint(static_cast(*i))) ostr << *i; else ostr << "\\x" << std::hex << short{*i}; } if (!str.empty() && !gccStyle) ostr << std::endl; else if (gccStyle) ostr << '\"'; return ostr.str(); } bool TestFixture::assert_(const char * const filename, const unsigned int linenr, const bool condition) const { if (!condition) { ++fails_counter; errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl; } return condition; } void TestFixture::assertEqualsFailed(const char* const filename, const unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const { ++fails_counter; errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << std::endl << "Expected: " << std::endl << writestr(expected) << std::endl << "Actual: " << std::endl << writestr(actual) << std::endl; if (!msg.empty()) errmsg << "Hint:" << std::endl << msg << std::endl; errmsg << "_____" << std::endl; } bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const { if (expected != actual) { assertEqualsFailed(filename, linenr, expected, actual, msg); } return expected == actual; } std::string TestFixture::deleteLineNumber(const std::string &message) { std::string result(message); // delete line number in "...:NUMBER:..." std::string::size_type pos = 0; while ((pos = result.find(':', pos)) != std::string::npos) { // get number if (pos + 1 == result.find_first_of("0123456789", pos + 1)) { const std::string::size_type after = result.find_first_not_of("0123456789", pos + 1); if (after != std::string::npos && result.at(after) == ':') { // erase NUMBER result.erase(pos + 1, after - pos - 1); pos = after; } else { ++pos; } } else { ++pos; } } return result; } void TestFixture::assertEqualsWithoutLineNumbers(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const { assertEquals(filename, linenr, deleteLineNumber(expected), deleteLineNumber(actual), msg); } bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const std::string& actual, const std::string &msg) const { return assertEquals(filename, linenr, std::string(expected), actual, msg); } bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const char actual[], const std::string &msg) const { return assertEquals(filename, linenr, std::string(expected), std::string(actual), msg); } bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string& expected, const char actual[], const std::string &msg) const { return assertEquals(filename, linenr, expected, std::string(actual), msg); } bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const long long expected, const long long actual, const std::string &msg) const { if (expected != actual) { assertEquals(filename, linenr, std::to_string(expected), std::to_string(actual), msg); } return expected == actual; } void TestFixture::assertEqualsDouble(const char * const filename, const unsigned int linenr, const double expected, const double actual, const double tolerance, const std::string &msg) const { if (expected < (actual - tolerance) || expected > (actual + tolerance)) { std::ostringstream ostr1; ostr1 << expected; std::ostringstream ostr2; ostr2 << actual; assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg); } } void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const std::string &wanted, const std::string ¤t, const std::string &actual) const { if (wanted == actual) { errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded unexpectedly. " << "Result: " << writestr(wanted, true) << std::endl << "_____" << std::endl; ++succeeded_todos_counter; } else { assertEquals(filename, linenr, current, actual); ++todos_counter; } } void TestFixture::todoAssertEquals(const char* const filename, const unsigned int linenr, const char wanted[], const char current[], const std::string& actual) const { todoAssertEquals(filename, linenr, std::string(wanted), std::string(current), actual); } void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const long long wanted, const long long current, const long long actual) const { todoAssertEquals(filename, linenr, std::to_string(wanted), std::to_string(current), std::to_string(actual)); } void TestFixture::assertThrow(const char * const filename, const unsigned int linenr) const { ++fails_counter; errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. " << "The expected exception was thrown" << std::endl << "_____" << std::endl; } void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const { ++fails_counter; errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << "The expected exception was not thrown" << std::endl << "_____" << std::endl; } void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const { ++fails_counter; errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << "Unexpected exception was thrown" << std::endl << "_____" << std::endl; } void TestFixture::printHelp() { std::cout << "Testrunner - run Cppcheck tests\n" "\n" "Syntax:\n" " testrunner [OPTIONS] [TestClass::TestCase...]\n" " run all test cases:\n" " testrunner\n" " run all test cases in TestClass:\n" " testrunner TestClass\n" " run TestClass::TestCase:\n" " testrunner TestClass::TestCase\n" " run all test cases in TestClass1 and TestClass2::TestCase:\n" " testrunner TestClass1 TestClass2::TestCase\n" "\n" "Options:\n" " -q Do not print the test cases that have run.\n" " -h, --help Print this help.\n"; } void TestFixture::run(const std::string &str) { testToRun = str; try { if (quiet_tests) { std::cout << '\n' << classname << ':'; SUPPRESS; run(); } else run(); } catch (const InternalError& e) { ++fails_counter; errmsg << classname << "::" << mTestname << " - InternalError: " << e.errorMessage << std::endl; } catch (const std::exception& error) { ++fails_counter; errmsg << classname << "::" << mTestname << " - Exception: " << error.what() << std::endl; } catch (...) { ++fails_counter; errmsg << classname << "::" << mTestname << " - Unknown exception" << std::endl; } } void TestFixture::processOptions(const options& args) { quiet_tests = args.quiet(); exename = args.exe(); } std::size_t TestFixture::runTests(const options& args) { countTests = 0; errmsg.str(""); for (std::string classname : args.which_test()) { std::string testname; if (classname.find("::") != std::string::npos) { testname = classname.substr(classname.find("::") + 2); classname.erase(classname.find("::")); } for (TestFixture * test : TestRegistry::theInstance().tests()) { if (classname.empty() || test->classname == classname) { test->processOptions(args); test->run(testname); } } } std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl; std::cout << "Number of todos: " << todos_counter; if (succeeded_todos_counter > 0) std::cout << " (" << succeeded_todos_counter << " succeeded)"; std::cout << std::endl; // calling flush here, to do all output before the error messages (in case the output is buffered) std::cout.flush(); std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl; std::cerr << errmsg.str(); std::cerr.flush(); return fails_counter + succeeded_todos_counter; } void TestFixture::reportOut(const std::string & outmsg, Color /*c*/) { mOutput << outmsg << std::endl; } void TestFixture::reportErr(const ErrorMessage &msg) { if (msg.severity == Severity::internal) return; const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation)); // TODO: remove the unique error handling? if (errout.str().find(errormessage) == std::string::npos) errout << errormessage << std::endl; } void TestFixture::setTemplateFormat(const std::string &templateFormat) { if (templateFormat == "multiline") { mTemplateFormat = "{file}:{line}:{severity}:{message}"; mTemplateLocation = "{file}:{line}:note:{info}"; } else if (templateFormat == "simple") { mTemplateFormat = "{file}:{line}:{column}: {severity}:{inconclusive:inconclusive:} {message} [{id}]"; mTemplateLocation = ""; } else { mTemplateFormat = templateFormat; mTemplateLocation = ""; } } TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::library(const char lib[]) { if (REDUNDANT_CHECK && std::find(settings.libraries.cbegin(), settings.libraries.cend(), lib) != settings.libraries.cend()) throw std::runtime_error("redundant setting: libraries (" + std::string(lib) + ")"); // TODO: exename is not yet set LOAD_LIB_2_EXE(settings.library, lib, fixture.exename.c_str()); // strip extension std::string lib_s(lib); const std::string ext(".cfg"); const auto pos = lib_s.find(ext); if (pos != std::string::npos) lib_s.erase(pos, ext.size()); settings.libraries.emplace_back(lib_s); return *this; } TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::platform(Platform::Type type) { const std::string platformStr = Platform::toString(type); if (REDUNDANT_CHECK && settings.platform.type == type) throw std::runtime_error("redundant setting: platform (" + platformStr + ")"); std::string errstr; // TODO: exename is not yet set if (!settings.platform.set(platformStr, errstr, {fixture.exename})) throw std::runtime_error("platform '" + platformStr + "' not found"); return *this; } TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::libraryxml(const char xmldata[], std::size_t len) { tinyxml2::XMLDocument doc; if (tinyxml2::XML_SUCCESS != doc.Parse(xmldata, len)) throw std::runtime_error("loading XML data failed"); if (settings.library.load(doc).errorcode != Library::ErrorCode::OK) throw std::runtime_error("loading library XML failed"); return *this; }