/*
* Cppcheck - A tool for static C/C++ code analysis
* Copyright (C) 2007-2023 Cppcheck team.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*/
#include "fixture.h"
#include "cppcheck.h"
#include "errortypes.h"
#include "options.h"
#include "redirect.h"
#include
#include
#include
#include
#include
#include
#include
#include
#include "xml.h"
/**
* TestRegistry
**/
namespace {
struct CompareFixtures {
bool operator()(const TestFixture* lhs, const TestFixture* rhs) const {
return lhs->classname < rhs->classname;
}
};
}
using TestSet = std::set;
namespace {
class TestRegistry {
TestSet _tests;
public:
static TestRegistry &theInstance() {
static TestRegistry testreg;
return testreg;
}
void addTest(TestFixture *t) {
_tests.insert(t);
}
const TestSet &tests() const {
return _tests;
}
};
}
/**
* TestFixture
**/
std::ostringstream TestFixture::errmsg;
unsigned int TestFixture::countTests;
std::size_t TestFixture::fails_counter = 0;
std::size_t TestFixture::todos_counter = 0;
std::size_t TestFixture::succeeded_todos_counter = 0;
TestFixture::TestFixture(const char * const _name)
: classname(_name)
{
TestRegistry::theInstance().addTest(this);
}
bool TestFixture::prepareTest(const char testname[])
{
mVerbose = false;
mTemplateFormat.clear();
mTemplateLocation.clear();
CppCheck::resetTimerResults();
prepareTestInternal();
// Check if tests should be executed
if (testToRun.empty() || testToRun == testname) {
// Tests will be executed - prepare them
mTestname = testname;
++countTests;
if (quiet_tests) {
std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
std::fflush(stdout);
} else {
std::cout << classname << "::" << mTestname << std::endl;
}
return true;
}
return false;
}
void TestFixture::teardownTest()
{
teardownTestInternal();
// TODO: enable
/*
{
const std::string s = errout.str();
if (!s.empty())
throw std::runtime_error("unconsumed ErrorLogger err: " + s);
}
*/
{
const std::string s = output_str();
if (!s.empty())
throw std::runtime_error("unconsumed ErrorLogger out: " + s);
}
}
std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
{
return std::string(filename) + ':' + std::to_string(linenr) + '(' + classname + "::" + mTestname + ')';
}
static std::string writestr(const std::string &str, bool gccStyle = false)
{
std::ostringstream ostr;
if (gccStyle)
ostr << '\"';
for (std::string::const_iterator i = str.cbegin(); i != str.cend(); ++i) {
if (*i == '\n') {
ostr << "\\n";
if ((i+1) != str.end() && !gccStyle)
ostr << std::endl;
} else if (*i == '\t')
ostr << "\\t";
else if (*i == '\"')
ostr << "\\\"";
else if (std::isprint(static_cast(*i)))
ostr << *i;
else
ostr << "\\x" << std::hex << short{*i};
}
if (!str.empty() && !gccStyle)
ostr << std::endl;
else if (gccStyle)
ostr << '\"';
return ostr.str();
}
bool TestFixture::assert_(const char * const filename, const unsigned int linenr, const bool condition) const
{
if (!condition) {
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;
}
return condition;
}
void TestFixture::assertEqualsFailed(const char* const filename, const unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const
{
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << std::endl
<< "Expected: " << std::endl
<< writestr(expected) << std::endl
<< "Actual: " << std::endl
<< writestr(actual) << std::endl;
if (!msg.empty())
errmsg << "Hint:" << std::endl << msg << std::endl;
errmsg << "_____" << std::endl;
}
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
{
if (expected != actual) {
assertEqualsFailed(filename, linenr, expected, actual, msg);
}
return expected == actual;
}
std::string TestFixture::deleteLineNumber(const std::string &message)
{
std::string result(message);
// delete line number in "...:NUMBER:..."
std::string::size_type pos = 0;
while ((pos = result.find(':', pos)) != std::string::npos) {
// get number
if (pos + 1 == result.find_first_of("0123456789", pos + 1)) {
const std::string::size_type after = result.find_first_not_of("0123456789", pos + 1);
if (after != std::string::npos
&& result.at(after) == ':') {
// erase NUMBER
result.erase(pos + 1, after - pos - 1);
pos = after;
} else {
++pos;
}
} else {
++pos;
}
}
return result;
}
void TestFixture::assertEqualsWithoutLineNumbers(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
{
assertEquals(filename, linenr, deleteLineNumber(expected), deleteLineNumber(actual), msg);
}
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const std::string& actual, const std::string &msg) const
{
return assertEquals(filename, linenr, std::string(expected), actual, msg);
}
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const char actual[], const std::string &msg) const
{
return assertEquals(filename, linenr, std::string(expected), std::string(actual), msg);
}
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string& expected, const char actual[], const std::string &msg) const
{
return assertEquals(filename, linenr, expected, std::string(actual), msg);
}
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const long long expected, const long long actual, const std::string &msg) const
{
if (expected != actual) {
assertEquals(filename, linenr, std::to_string(expected), std::to_string(actual), msg);
}
return expected == actual;
}
void TestFixture::assertEqualsDouble(const char * const filename, const unsigned int linenr, const double expected, const double actual, const double tolerance, const std::string &msg) const
{
if (expected < (actual - tolerance) || expected > (actual + tolerance)) {
std::ostringstream ostr1;
ostr1 << expected;
std::ostringstream ostr2;
ostr2 << actual;
assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg);
}
}
void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr,
const std::string &wanted,
const std::string ¤t,
const std::string &actual) const
{
if (wanted == actual) {
errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded unexpectedly. "
<< "Result: " << writestr(wanted, true) << std::endl << "_____" << std::endl;
++succeeded_todos_counter;
} else {
assertEquals(filename, linenr, current, actual);
++todos_counter;
}
}
void TestFixture::todoAssertEquals(const char* const filename, const unsigned int linenr,
const char wanted[],
const char current[],
const std::string& actual) const
{
todoAssertEquals(filename, linenr, std::string(wanted), std::string(current), actual);
}
void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const long long wanted, const long long current, const long long actual) const
{
todoAssertEquals(filename, linenr, std::to_string(wanted), std::to_string(current), std::to_string(actual));
}
void TestFixture::assertThrow(const char * const filename, const unsigned int linenr) const
{
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "
<< "The expected exception was thrown" << std::endl << "_____" << std::endl;
}
void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
{
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
<< "The expected exception was not thrown" << std::endl << "_____" << std::endl;
}
void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
{
++fails_counter;
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
<< "Unexpected exception was thrown" << std::endl << "_____" << std::endl;
}
void TestFixture::printHelp()
{
std::cout << "Testrunner - run Cppcheck tests\n"
"\n"
"Syntax:\n"
" testrunner [OPTIONS] [TestClass::TestCase...]\n"
" run all test cases:\n"
" testrunner\n"
" run all test cases in TestClass:\n"
" testrunner TestClass\n"
" run TestClass::TestCase:\n"
" testrunner TestClass::TestCase\n"
" run all test cases in TestClass1 and TestClass2::TestCase:\n"
" testrunner TestClass1 TestClass2::TestCase\n"
"\n"
"Options:\n"
" -q Do not print the test cases that have run.\n"
" -h, --help Print this help.\n";
}
void TestFixture::run(const std::string &str)
{
testToRun = str;
try {
if (quiet_tests) {
std::cout << '\n' << classname << ':';
SUPPRESS;
run();
}
else
run();
}
catch (const InternalError& e) {
++fails_counter;
errmsg << classname << "::" << mTestname << " - InternalError: " << e.errorMessage << std::endl;
}
catch (const std::exception& error) {
++fails_counter;
errmsg << classname << "::" << mTestname << " - Exception: " << error.what() << std::endl;
}
catch (...) {
++fails_counter;
errmsg << classname << "::" << mTestname << " - Unknown exception" << std::endl;
}
}
void TestFixture::processOptions(const options& args)
{
quiet_tests = args.quiet();
exename = args.exe();
}
std::size_t TestFixture::runTests(const options& args)
{
countTests = 0;
errmsg.str("");
for (std::string classname : args.which_test()) {
std::string testname;
if (classname.find("::") != std::string::npos) {
testname = classname.substr(classname.find("::") + 2);
classname.erase(classname.find("::"));
}
for (TestFixture * test : TestRegistry::theInstance().tests()) {
if (classname.empty() || test->classname == classname) {
test->processOptions(args);
test->run(testname);
}
}
}
std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
std::cout << "Number of todos: " << todos_counter;
if (succeeded_todos_counter > 0)
std::cout << " (" << succeeded_todos_counter << " succeeded)";
std::cout << std::endl;
// calling flush here, to do all output before the error messages (in case the output is buffered)
std::cout.flush();
std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
std::cerr << errmsg.str();
std::cerr.flush();
return fails_counter + succeeded_todos_counter;
}
void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
{
mOutput << outmsg << std::endl;
}
void TestFixture::reportErr(const ErrorMessage &msg)
{
if (msg.severity == Severity::internal)
return;
const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
errout << errormessage << std::endl;
}
void TestFixture::setTemplateFormat(const std::string &templateFormat)
{
if (templateFormat == "multiline") {
mTemplateFormat = "{file}:{line}:{severity}:{message}";
mTemplateLocation = "{file}:{line}:note:{info}";
}
else if (templateFormat == "simple") {
mTemplateFormat = "{file}:{line}:{column}: {severity}:{inconclusive:inconclusive:} {message} [{id}]";
mTemplateLocation = "";
}
else {
mTemplateFormat = templateFormat;
mTemplateLocation = "";
}
}
TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::library(const char lib[]) {
if (REDUNDANT_CHECK && std::find(settings.libraries.cbegin(), settings.libraries.cend(), lib) != settings.libraries.cend())
throw std::runtime_error("redundant setting: libraries (" + std::string(lib) + ")");
// TODO: exename is not yet set
LOAD_LIB_2_EXE(settings.library, lib, fixture.exename.c_str());
// strip extension
std::string lib_s(lib);
const std::string ext(".cfg");
const auto pos = lib_s.find(ext);
if (pos != std::string::npos)
lib_s.erase(pos, ext.size());
settings.libraries.emplace_back(lib_s);
return *this;
}
TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::platform(Platform::Type type)
{
const std::string platformStr = Platform::toString(type);
if (REDUNDANT_CHECK && settings.platform.type == type)
throw std::runtime_error("redundant setting: platform (" + platformStr + ")");
std::string errstr;
// TODO: exename is not yet set
if (!settings.platform.set(platformStr, errstr, {fixture.exename}))
throw std::runtime_error("platform '" + platformStr + "' not found");
return *this;
}
TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::libraryxml(const char xmldata[], std::size_t len)
{
tinyxml2::XMLDocument doc;
if (tinyxml2::XML_SUCCESS != doc.Parse(xmldata, len))
throw std::runtime_error("loading XML data failed");
if (settings.library.load(doc).errorcode != Library::ErrorCode::OK)
throw std::runtime_error("loading library XML failed");
return *this;
}