465 lines
15 KiB
C++
465 lines
15 KiB
C++
/*
|
|
* Cppcheck - A tool for static C/C++ code analysis
|
|
* Copyright (C) 2007-2023 Cppcheck team.
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include "fixture.h"
|
|
|
|
#include "cppcheck.h"
|
|
#include "errortypes.h"
|
|
#include "options.h"
|
|
#include "redirect.h"
|
|
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <cctype>
|
|
#include <exception>
|
|
#include <iostream>
|
|
#include <set>
|
|
#include <sstream>
|
|
#include <string>
|
|
|
|
#include "xml.h"
|
|
|
|
/**
|
|
* TestRegistry
|
|
**/
|
|
namespace {
|
|
struct CompareFixtures {
|
|
bool operator()(const TestFixture* lhs, const TestFixture* rhs) const {
|
|
return lhs->classname < rhs->classname;
|
|
}
|
|
};
|
|
}
|
|
using TestSet = std::set<TestFixture*, CompareFixtures>;
|
|
namespace {
|
|
class TestRegistry {
|
|
TestSet _tests;
|
|
public:
|
|
|
|
static TestRegistry &theInstance() {
|
|
static TestRegistry testreg;
|
|
return testreg;
|
|
}
|
|
|
|
void addTest(TestFixture *t) {
|
|
_tests.insert(t);
|
|
}
|
|
|
|
const TestSet &tests() const {
|
|
return _tests;
|
|
}
|
|
};
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
* TestFixture
|
|
**/
|
|
|
|
std::ostringstream TestFixture::errmsg;
|
|
unsigned int TestFixture::countTests;
|
|
|
|
std::size_t TestFixture::fails_counter = 0;
|
|
std::size_t TestFixture::todos_counter = 0;
|
|
std::size_t TestFixture::succeeded_todos_counter = 0;
|
|
|
|
TestFixture::TestFixture(const char * const _name)
|
|
: classname(_name)
|
|
{
|
|
TestRegistry::theInstance().addTest(this);
|
|
}
|
|
|
|
|
|
bool TestFixture::prepareTest(const char testname[])
|
|
{
|
|
mVerbose = false;
|
|
mTemplateFormat.clear();
|
|
mTemplateLocation.clear();
|
|
CppCheck::resetTimerResults();
|
|
|
|
prepareTestInternal();
|
|
|
|
// Check if tests should be executed
|
|
if (testToRun.empty() || testToRun == testname) {
|
|
// Tests will be executed - prepare them
|
|
mTestname = testname;
|
|
++countTests;
|
|
if (quiet_tests) {
|
|
std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
|
|
std::fflush(stdout);
|
|
} else {
|
|
std::cout << classname << "::" << mTestname << std::endl;
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void TestFixture::teardownTest()
|
|
{
|
|
teardownTestInternal();
|
|
|
|
// TODO: enable
|
|
/*
|
|
{
|
|
const std::string s = errout.str();
|
|
if (!s.empty())
|
|
throw std::runtime_error("unconsumed ErrorLogger err: " + s);
|
|
}
|
|
*/
|
|
{
|
|
const std::string s = output_str();
|
|
if (!s.empty())
|
|
throw std::runtime_error("unconsumed ErrorLogger out: " + s);
|
|
}
|
|
}
|
|
|
|
std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
|
|
{
|
|
return std::string(filename) + ':' + std::to_string(linenr) + '(' + classname + "::" + mTestname + ')';
|
|
}
|
|
|
|
static std::string writestr(const std::string &str, bool gccStyle = false)
|
|
{
|
|
std::ostringstream ostr;
|
|
if (gccStyle)
|
|
ostr << '\"';
|
|
for (std::string::const_iterator i = str.cbegin(); i != str.cend(); ++i) {
|
|
if (*i == '\n') {
|
|
ostr << "\\n";
|
|
if ((i+1) != str.end() && !gccStyle)
|
|
ostr << std::endl;
|
|
} else if (*i == '\t')
|
|
ostr << "\\t";
|
|
else if (*i == '\"')
|
|
ostr << "\\\"";
|
|
else if (std::isprint(static_cast<unsigned char>(*i)))
|
|
ostr << *i;
|
|
else
|
|
ostr << "\\x" << std::hex << short{*i};
|
|
}
|
|
if (!str.empty() && !gccStyle)
|
|
ostr << std::endl;
|
|
else if (gccStyle)
|
|
ostr << '\"';
|
|
return ostr.str();
|
|
}
|
|
|
|
bool TestFixture::assert_(const char * const filename, const unsigned int linenr, const bool condition) const
|
|
{
|
|
if (!condition) {
|
|
++fails_counter;
|
|
errmsg << getLocationStr(filename, linenr) << ": Assertion failed." << std::endl << "_____" << std::endl;
|
|
}
|
|
return condition;
|
|
}
|
|
|
|
void TestFixture::assertEqualsFailed(const char* const filename, const unsigned int linenr, const std::string& expected, const std::string& actual, const std::string& msg) const
|
|
{
|
|
++fails_counter;
|
|
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. " << std::endl
|
|
<< "Expected: " << std::endl
|
|
<< writestr(expected) << std::endl
|
|
<< "Actual: " << std::endl
|
|
<< writestr(actual) << std::endl;
|
|
if (!msg.empty())
|
|
errmsg << "Hint:" << std::endl << msg << std::endl;
|
|
errmsg << "_____" << std::endl;
|
|
}
|
|
|
|
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
|
|
{
|
|
if (expected != actual) {
|
|
assertEqualsFailed(filename, linenr, expected, actual, msg);
|
|
}
|
|
return expected == actual;
|
|
}
|
|
|
|
std::string TestFixture::deleteLineNumber(const std::string &message)
|
|
{
|
|
std::string result(message);
|
|
// delete line number in "...:NUMBER:..."
|
|
std::string::size_type pos = 0;
|
|
while ((pos = result.find(':', pos)) != std::string::npos) {
|
|
// get number
|
|
if (pos + 1 == result.find_first_of("0123456789", pos + 1)) {
|
|
const std::string::size_type after = result.find_first_not_of("0123456789", pos + 1);
|
|
if (after != std::string::npos
|
|
&& result.at(after) == ':') {
|
|
// erase NUMBER
|
|
result.erase(pos + 1, after - pos - 1);
|
|
pos = after;
|
|
} else {
|
|
++pos;
|
|
}
|
|
} else {
|
|
++pos;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void TestFixture::assertEqualsWithoutLineNumbers(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
|
|
{
|
|
assertEquals(filename, linenr, deleteLineNumber(expected), deleteLineNumber(actual), msg);
|
|
}
|
|
|
|
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const std::string& actual, const std::string &msg) const
|
|
{
|
|
return assertEquals(filename, linenr, std::string(expected), actual, msg);
|
|
}
|
|
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const char actual[], const std::string &msg) const
|
|
{
|
|
return assertEquals(filename, linenr, std::string(expected), std::string(actual), msg);
|
|
}
|
|
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string& expected, const char actual[], const std::string &msg) const
|
|
{
|
|
return assertEquals(filename, linenr, expected, std::string(actual), msg);
|
|
}
|
|
|
|
bool TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const long long expected, const long long actual, const std::string &msg) const
|
|
{
|
|
if (expected != actual) {
|
|
assertEquals(filename, linenr, std::to_string(expected), std::to_string(actual), msg);
|
|
}
|
|
return expected == actual;
|
|
}
|
|
|
|
void TestFixture::assertEqualsDouble(const char * const filename, const unsigned int linenr, const double expected, const double actual, const double tolerance, const std::string &msg) const
|
|
{
|
|
if (expected < (actual - tolerance) || expected > (actual + tolerance)) {
|
|
std::ostringstream ostr1;
|
|
ostr1 << expected;
|
|
std::ostringstream ostr2;
|
|
ostr2 << actual;
|
|
assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg);
|
|
}
|
|
}
|
|
|
|
void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr,
|
|
const std::string &wanted,
|
|
const std::string ¤t,
|
|
const std::string &actual) const
|
|
{
|
|
if (wanted == actual) {
|
|
errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded unexpectedly. "
|
|
<< "Result: " << writestr(wanted, true) << std::endl << "_____" << std::endl;
|
|
|
|
++succeeded_todos_counter;
|
|
} else {
|
|
assertEquals(filename, linenr, current, actual);
|
|
++todos_counter;
|
|
}
|
|
}
|
|
|
|
void TestFixture::todoAssertEquals(const char* const filename, const unsigned int linenr,
|
|
const char wanted[],
|
|
const char current[],
|
|
const std::string& actual) const
|
|
{
|
|
todoAssertEquals(filename, linenr, std::string(wanted), std::string(current), actual);
|
|
}
|
|
|
|
|
|
void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const long long wanted, const long long current, const long long actual) const
|
|
{
|
|
todoAssertEquals(filename, linenr, std::to_string(wanted), std::to_string(current), std::to_string(actual));
|
|
}
|
|
|
|
void TestFixture::assertThrow(const char * const filename, const unsigned int linenr) const
|
|
{
|
|
++fails_counter;
|
|
errmsg << getLocationStr(filename, linenr) << ": Assertion succeeded. "
|
|
<< "The expected exception was thrown" << std::endl << "_____" << std::endl;
|
|
|
|
}
|
|
|
|
void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
|
|
{
|
|
++fails_counter;
|
|
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
|
|
<< "The expected exception was not thrown" << std::endl << "_____" << std::endl;
|
|
|
|
}
|
|
|
|
void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
|
|
{
|
|
++fails_counter;
|
|
errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
|
|
<< "Unexpected exception was thrown" << std::endl << "_____" << std::endl;
|
|
|
|
}
|
|
|
|
void TestFixture::printHelp()
|
|
{
|
|
std::cout << "Testrunner - run Cppcheck tests\n"
|
|
"\n"
|
|
"Syntax:\n"
|
|
" testrunner [OPTIONS] [TestClass::TestCase...]\n"
|
|
" run all test cases:\n"
|
|
" testrunner\n"
|
|
" run all test cases in TestClass:\n"
|
|
" testrunner TestClass\n"
|
|
" run TestClass::TestCase:\n"
|
|
" testrunner TestClass::TestCase\n"
|
|
" run all test cases in TestClass1 and TestClass2::TestCase:\n"
|
|
" testrunner TestClass1 TestClass2::TestCase\n"
|
|
"\n"
|
|
"Options:\n"
|
|
" -q Do not print the test cases that have run.\n"
|
|
" -h, --help Print this help.\n";
|
|
}
|
|
|
|
void TestFixture::run(const std::string &str)
|
|
{
|
|
testToRun = str;
|
|
try {
|
|
if (quiet_tests) {
|
|
std::cout << '\n' << classname << ':';
|
|
SUPPRESS;
|
|
run();
|
|
}
|
|
else
|
|
run();
|
|
}
|
|
catch (const InternalError& e) {
|
|
++fails_counter;
|
|
errmsg << classname << "::" << mTestname << " - InternalError: " << e.errorMessage << std::endl;
|
|
}
|
|
catch (const std::exception& error) {
|
|
++fails_counter;
|
|
errmsg << classname << "::" << mTestname << " - Exception: " << error.what() << std::endl;
|
|
}
|
|
catch (...) {
|
|
++fails_counter;
|
|
errmsg << classname << "::" << mTestname << " - Unknown exception" << std::endl;
|
|
}
|
|
}
|
|
|
|
void TestFixture::processOptions(const options& args)
|
|
{
|
|
quiet_tests = args.quiet();
|
|
exename = args.exe();
|
|
}
|
|
|
|
std::size_t TestFixture::runTests(const options& args)
|
|
{
|
|
countTests = 0;
|
|
errmsg.str("");
|
|
|
|
for (std::string classname : args.which_test()) {
|
|
std::string testname;
|
|
if (classname.find("::") != std::string::npos) {
|
|
testname = classname.substr(classname.find("::") + 2);
|
|
classname.erase(classname.find("::"));
|
|
}
|
|
|
|
for (TestFixture * test : TestRegistry::theInstance().tests()) {
|
|
if (classname.empty() || test->classname == classname) {
|
|
test->processOptions(args);
|
|
test->run(testname);
|
|
}
|
|
}
|
|
}
|
|
|
|
std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
|
|
std::cout << "Number of todos: " << todos_counter;
|
|
if (succeeded_todos_counter > 0)
|
|
std::cout << " (" << succeeded_todos_counter << " succeeded)";
|
|
std::cout << std::endl;
|
|
// calling flush here, to do all output before the error messages (in case the output is buffered)
|
|
std::cout.flush();
|
|
|
|
std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
|
|
std::cerr << errmsg.str();
|
|
|
|
std::cerr.flush();
|
|
return fails_counter + succeeded_todos_counter;
|
|
}
|
|
|
|
void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
|
|
{
|
|
mOutput << outmsg << std::endl;
|
|
}
|
|
|
|
void TestFixture::reportErr(const ErrorMessage &msg)
|
|
{
|
|
if (msg.severity == Severity::internal)
|
|
return;
|
|
const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
|
|
// TODO: remove the unique error handling?
|
|
if (errout.str().find(errormessage) == std::string::npos)
|
|
errout << errormessage << std::endl;
|
|
}
|
|
|
|
void TestFixture::setTemplateFormat(const std::string &templateFormat)
|
|
{
|
|
if (templateFormat == "multiline") {
|
|
mTemplateFormat = "{file}:{line}:{severity}:{message}";
|
|
mTemplateLocation = "{file}:{line}:note:{info}";
|
|
}
|
|
else if (templateFormat == "simple") {
|
|
mTemplateFormat = "{file}:{line}:{column}: {severity}:{inconclusive:inconclusive:} {message} [{id}]";
|
|
mTemplateLocation = "";
|
|
}
|
|
else {
|
|
mTemplateFormat = templateFormat;
|
|
mTemplateLocation = "";
|
|
}
|
|
}
|
|
|
|
TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::library(const char lib[]) {
|
|
if (REDUNDANT_CHECK && std::find(settings.libraries.cbegin(), settings.libraries.cend(), lib) != settings.libraries.cend())
|
|
throw std::runtime_error("redundant setting: libraries (" + std::string(lib) + ")");
|
|
// TODO: exename is not yet set
|
|
LOAD_LIB_2_EXE(settings.library, lib, fixture.exename.c_str());
|
|
// strip extension
|
|
std::string lib_s(lib);
|
|
const std::string ext(".cfg");
|
|
const auto pos = lib_s.find(ext);
|
|
if (pos != std::string::npos)
|
|
lib_s.erase(pos, ext.size());
|
|
settings.libraries.emplace_back(lib_s);
|
|
return *this;
|
|
}
|
|
|
|
TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::platform(Platform::Type type)
|
|
{
|
|
const std::string platformStr = Platform::toString(type);
|
|
|
|
if (REDUNDANT_CHECK && settings.platform.type == type)
|
|
throw std::runtime_error("redundant setting: platform (" + platformStr + ")");
|
|
|
|
std::string errstr;
|
|
// TODO: exename is not yet set
|
|
if (!settings.platform.set(platformStr, errstr, {fixture.exename}))
|
|
throw std::runtime_error("platform '" + platformStr + "' not found");
|
|
return *this;
|
|
}
|
|
|
|
TestFixture::SettingsBuilder& TestFixture::SettingsBuilder::libraryxml(const char xmldata[], std::size_t len)
|
|
{
|
|
tinyxml2::XMLDocument doc;
|
|
if (tinyxml2::XML_SUCCESS != doc.Parse(xmldata, len))
|
|
throw std::runtime_error("loading XML data failed");
|
|
if (settings.library.load(doc).errorcode != Library::ErrorCode::OK)
|
|
throw std::runtime_error("loading library XML failed");
|
|
return *this;
|
|
}
|