TestTrac: Add separate file for black box / integration tests in testrunner

Makefile

@@ -272,6 +272,7 @@ TESTOBJ =     test/options.o \
               test/testtoken.o \
               test/testtokenize.o \
               test/testtokenlist.o \
+              test/testtrac.o \
               test/testtype.o \
               test/testuninitvar.o \
               test/testunusedfunctions.o \
@@ -495,7 +496,7 @@ $(libcppdir)/ctu.o: lib/ctu.cpp externals/tinyxml/tinyxml2.h lib/astutils.h lib/
 $(libcppdir)/errorlogger.o: lib/errorlogger.cpp externals/tinyxml/tinyxml2.h lib/analyzerinfo.h lib/check.h lib/config.h lib/cppcheck.h lib/errorlogger.h lib/importproject.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CPPFILESDIR) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o $(libcppdir)/errorlogger.o $(libcppdir)/errorlogger.cpp
 
-$(libcppdir)/exprengine.o: lib/exprengine.cpp lib/astutils.h lib/config.h lib/errorlogger.h lib/exprengine.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h
+$(libcppdir)/exprengine.o: lib/exprengine.cpp externals/z3_version.h lib/astutils.h lib/config.h lib/errorlogger.h lib/exprengine.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h
 	$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CPPFILESDIR) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o $(libcppdir)/exprengine.o $(libcppdir)/exprengine.cpp
 
 $(libcppdir)/forwardanalyzer.o: lib/forwardanalyzer.cpp lib/astutils.h lib/config.h lib/errorlogger.h lib/forwardanalyzer.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/utils.h lib/valueflow.h lib/valueptr.h
@@ -732,6 +733,9 @@ test/testtokenize.o: test/testtokenize.cpp externals/simplecpp/simplecpp.h lib/c
 test/testtokenlist.o: test/testtokenlist.cpp lib/config.h lib/errorlogger.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h test/testsuite.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CPPFILESDIR) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o test/testtokenlist.o test/testtokenlist.cpp
 
+test/testtrac.o: test/testtrac.cpp lib/check.h lib/checkbool.h lib/checknullpointer.h lib/checkother.h lib/config.h lib/ctu.h lib/errorlogger.h lib/mathlib.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h test/testsuite.h
+	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CPPFILESDIR) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o test/testtrac.o test/testtrac.cpp
+
 test/testtype.o: test/testtype.cpp lib/check.h lib/checktype.h lib/config.h lib/errorlogger.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h test/testsuite.h
 	$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CPPFILESDIR) $(CXXFLAGS) $(UNDEF_STRICT_ANSI) -c -o test/testtype.o test/testtype.cpp
 

test/testastutils.cpp

@@ -188,14 +188,6 @@ private:
         ASSERT_EQUALS(true,  isSameExpression("void f() {double y = 1e1; (x + y) < (x + 10.0); } ", "+", "+"));
         ASSERT_EQUALS(true,  isSameExpression("void f() {double y = 1e1; (x + 10.0) < (y + x); } ", "+", "+"));
         ASSERT_EQUALS(true,  isSameExpression("void f() {double y = 1e1; double z = 10.0; (x + y) < (x + z); } ", "+", "+"));
-        ASSERT_EQUALS(true,  isSameExpression("A + A", "A", "A"));
-
-        //https://trac.cppcheck.net/ticket/9700
-        ASSERT_EQUALS(true, isSameExpression("A::B + A::B;", "::", "::"));
-        ASSERT_EQUALS(false, isSameExpression("A::B + A::C;", "::", "::"));
-        ASSERT_EQUALS(true, isSameExpression("A::B* get() { if(x) return new A::B(true); else return new A::B(true); }", "new", "new"));
-        ASSERT_EQUALS(false, isSameExpression("A::B* get() { if(x) return new A::B(true); else return new A::C(true); }", "new", "new"));
-        ASSERT_EQUALS(true, true);
     }
 
     bool isVariableChanged(const char code[], const char startPattern[], const char endPattern[]) {

test/testbool.cpp

@@ -62,8 +62,6 @@ private:
         TEST_CASE(checkComparisonOfFuncReturningBool4);
         TEST_CASE(checkComparisonOfFuncReturningBool5);
         TEST_CASE(checkComparisonOfFuncReturningBool6);
-        // Integration tests..
-        TEST_CASE(checkComparisonOfFuncReturningBoolIntegrationTest1); // #7798 overloaded functions
 
         TEST_CASE(checkComparisonOfBoolWithBool);
 
@@ -748,18 +746,6 @@ private:
         ASSERT_EQUALS("", errout.str());
     }
 
-    void checkComparisonOfFuncReturningBoolIntegrationTest1() { // #7798
-        check("bool eval(double *) { return false; }\n"
-              "double eval(char *) { return 1.0; }\n"
-              "int main(int argc, char *argv[])\n"
-              "{\n"
-              "  if ( eval(argv[1]) > eval(argv[2]) )\n"
-              "    return 1;\n"
-              "  return 0;\n"
-              "}");
-        ASSERT_EQUALS("", errout.str());
-    }
-
     void checkComparisonOfBoolWithBool() {
         const char code[] = "void f(){\n"
                             "    int temp = 4;\n"

test/testnullpointer.cpp

@@ -94,7 +94,6 @@ private:
         TEST_CASE(nullpointer51);
         TEST_CASE(nullpointer52);
         TEST_CASE(nullpointer53); // #8005
-        TEST_CASE(nullpointer54); // #9573
         TEST_CASE(nullpointer_addressOf); // address of
         TEST_CASE(nullpointerSwitch); // #2626
         TEST_CASE(nullpointer_cast); // #4692
@@ -1763,22 +1762,6 @@ private:
         ASSERT_EQUALS("[test.cpp:3]: (warning) Possible null pointer dereference: params\n", errout.str());
     }
 
-    void nullpointer54() {
-        check("int foo (int **array, size_t n_array) {\n"
-              "    size_t i;\n"
-              "    for (i = 0; i < n_array; ++i) {\n"
-              "        if (*array[i] == 1)\n"
-              "            return 1;\n"
-              "    }\n"
-              "    return 0;\n"
-              "} \n"
-              "int bar() {\n"
-              "    int **array = NULL; \n"
-              "    foo (array, 0);\n"
-              "}\n");
-        ASSERT_EQUALS("", errout.str());
-    }
-
     void nullpointer_addressOf() { // address of
         check("void f() {\n"
               "  struct X *x = 0;\n"

test/testother.cpp

@@ -130,7 +130,6 @@ private:
         TEST_CASE(duplicateBranch1); // tests extracted by http://www.viva64.com/en/b/0149/ ( Comparison between PVS-Studio and cppcheck ): Errors detected in Quake 3: Arena by PVS-Studio: Fragment 2
         TEST_CASE(duplicateBranch2); // empty macro
         TEST_CASE(duplicateBranch3);
-        TEST_CASE(duplicateBranch4);
         TEST_CASE(duplicateExpression1);
         TEST_CASE(duplicateExpression2); // ticket #2730
         TEST_CASE(duplicateExpression3); // ticket #3317
@@ -4102,17 +4101,6 @@ private:
         ASSERT_EQUALS("", errout.str());
     }
 
-    void duplicateBranch4() {
-        check("void* f(bool b) {\n"
-              "    if (b) {\n"
-              "        return new A::Y(true);\n"
-              "    } else {\n"
-              "        return new A::Z(true);\n"
-              "    }\n"
-              "}\n");
-        ASSERT_EQUALS("", errout.str());
-    }
-
     void duplicateExpression1() {
         check("void foo(int a) {\n"
               "    if (a == a) { }\n"

test/testtrac.cpp

@@ -0,0 +1,115 @@
+/*
+ * Cppcheck - A tool for static C/C++ code analysis
+ * Copyright (C) 2007-2020 Cppcheck team.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+// This file is for integration tests for trac tickets.
+//
+// The intention is mostly to put tests here for false positives that
+// are fixed by fixing AST/ValueFlow/SymbolDatabase/etc. The check itself
+// worked as it should but there was problem with the input.
+//
+// These tests are typically "black box" tests and don't consider how the
+// checker works internally.
+
+#include "checkbool.h"
+#include "checknullpointer.h"
+#include "checkother.h"
+#include "testsuite.h"
+#include "settings.h"
+
+class TestTrac : public TestFixture {
+public:
+    TestTrac() : TestFixture("TestTrac") {
+    }
+
+private:
+
+    void run() OVERRIDE {
+        TEST_CASE(ticket_7798);
+        TEST_CASE(ticket_9573);
+        TEST_CASE(ticket_9700);
+    }
+
+    template<class C>
+    void check(const char code[], const char *filename = "test.cpp") {
+        // Clear the error buffer..
+        errout.str("");
+
+        Settings settings;
+        settings.addEnabled("style");
+        settings.addEnabled("warning");
+        settings.addEnabled("portability");
+        settings.addEnabled("performance");
+        settings.standards.c = Standards::CLatest;
+        settings.standards.cpp = Standards::CPPLatest;
+        settings.inconclusive = true;
+
+        // Tokenize..
+        Tokenizer tokenizer(&settings, this);
+        std::istringstream istr(code);
+        tokenizer.tokenize(istr, filename);
+
+        // Check..
+        C c(&tokenizer, &settings, this);
+        c.runChecks(&tokenizer, &settings, this);
+    }
+
+    void ticket_7798() {
+        // checkComparisonOfFuncReturningBool (overloaded functions)
+        check<CheckBool>("bool eval(double *) { return false; }\n"
+                         "double eval(char *) { return 1.0; }\n"
+                         "int main(int argc, char *argv[])\n"
+                         "{\n"
+                         "  if ( eval(argv[1]) > eval(argv[2]) )\n"
+                         "    return 1;\n"
+                         "  return 0;\n"
+                         "}");
+        ASSERT_EQUALS("", errout.str());
+    }
+
+    void ticket_9573() {
+        // nullpointer (valueflow)
+        check<CheckNullPointer>("int foo (int **array, size_t n_array) {\n"
+                                "    size_t i;\n"
+                                "    for (i = 0; i < n_array; ++i) {\n"
+                                "        if (*array[i] == 1)\n"
+                                "            return 1;\n"
+                                "    }\n"
+                                "    return 0;\n"
+                                "} \n"
+                                "int bar() {\n"
+                                "    int **array = NULL; \n"
+                                "    foo (array, 0);\n"
+                                "}\n");
+        ASSERT_EQUALS("", errout.str());
+    }
+
+    void ticket_9700() {
+        // FP: duplicateBranch
+        check<CheckOther>("void* f(bool b) {\n"
+                          "    if (b) {\n"
+                          "        return new A::Y(true);\n"
+                          "    } else {\n"
+                          "        return new A::Z(true);\n"
+                          "    }\n"
+                          "}\n");
+        ASSERT_EQUALS("", errout.str());
+    }
+};
+
+REGISTER_TEST(TestTrac)