From 1135520aa8bcf751eccf90a1e2d673bce085cc98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sat, 16 Dec 2023 17:03:03 +0100 Subject: [PATCH] run more tests in sanitizer workflows (#5744) --- .github/workflows/asan.yml | 24 +++++++++++++++++++++++- .github/workflows/tsan.yml | 24 +++++++++++++++++++++++- .github/workflows/ubsan.yml | 24 +++++++++++++++++++++++- test/cli/testutils.py | 23 ++++++++++++++++------- 4 files changed, 85 insertions(+), 10 deletions(-) diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml index d0a1c29c3..cc2f4a405 100644 --- a/.github/workflows/asan.yml +++ b/.github/workflows/asan.yml @@ -42,7 +42,7 @@ jobs: - name: Install missing software on ubuntu run: | sudo apt-get update - sudo apt-get install -y cmake make libpcre3-dev libboost-container-dev + sudo apt-get install -y cmake make libpcre3-dev libboost-container-dev libxml2-utils - name: Install clang run: | @@ -59,6 +59,12 @@ jobs: modules: 'qtcharts' cache: true + - name: Install missing Python packages + run: | + python3 -m pip install pip --upgrade + python3 -m pip install pytest + python3 -m pip install pytest-timeout + # TODO: disable all warnings - name: CMake run: | @@ -78,6 +84,22 @@ jobs: - name: Run tests run: ./cmake.output/bin/testrunner + - name: Run cfg tests + run: | + cmake --build cmake.output --target checkcfg -- -j $(nproc) + + # TODO: we should use CTest instead to parallelize tests but the start-up overhead will slow things down + - name: Run CTest + if: false + run: | + ctest --test-dir cmake.output --output-on-failure -j$(nproc) + + - name: Run test/cli + run: | + pwd=$(pwd) + cd test/cli + TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test-*.py + - name: Generate dependencies if: false run: | diff --git a/.github/workflows/tsan.yml b/.github/workflows/tsan.yml index fdb17dae7..8c76c507d 100644 --- a/.github/workflows/tsan.yml +++ b/.github/workflows/tsan.yml @@ -42,7 +42,7 @@ jobs: - name: Install missing software on ubuntu run: | sudo apt-get update - sudo apt-get install -y cmake make libpcre3-dev libboost-container-dev + sudo apt-get install -y cmake make libpcre3-dev libboost-container-dev libxml2-utils - name: Install clang run: | @@ -59,6 +59,12 @@ jobs: modules: 'qtcharts' cache: true + - name: Install missing Python packages + run: | + python3 -m pip install pip --upgrade + python3 -m pip install pytest + python3 -m pip install pytest-timeout + - name: CMake run: | cmake -S . -B cmake.output -DCMAKE_BUILD_TYPE=RelWithDebInfo -DHAVE_RULES=On -DBUILD_TESTS=On -DBUILD_GUI=Off -DWITH_QCHART=Off -DUSE_MATCHCOMPILER=Verify -DANALYZE_THREAD=On -DUSE_THREADS=On -DENABLE_CHECK_INTERNAL=On -DUSE_BOOST=On -DCPPCHK_GLIBCXX_DEBUG=Off -DCMAKE_DISABLE_PRECOMPILE_HEADERS=On -DCMAKE_GLOBAL_AUTOGEN_TARGET=Off -DDISABLE_DMAKE=On -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -77,6 +83,22 @@ jobs: - name: Run tests run: ./cmake.output/bin/testrunner + - name: Run cfg tests + run: | + cmake --build cmake.output --target checkcfg -- -j $(nproc) + + # TODO: we should use CTest instead to parallelize tests but the start-up overhead will slow things down + - name: Run CTest + if: false + run: | + ctest --test-dir cmake.output --output-on-failure -j$(nproc) + + - name: Run test/cli + run: | + pwd=$(pwd) + cd test/cli + TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test-*.py + - name: Generate dependencies if: false run: | diff --git a/.github/workflows/ubsan.yml b/.github/workflows/ubsan.yml index 654448987..a48fcb4a5 100644 --- a/.github/workflows/ubsan.yml +++ b/.github/workflows/ubsan.yml @@ -42,7 +42,7 @@ jobs: - name: Install missing software on ubuntu run: | sudo apt-get update - sudo apt-get install -y cmake make libpcre3-dev libboost-container-dev + sudo apt-get install -y cmake make libpcre3-dev libboost-container-dev libxml2-utils - name: Install clang run: | @@ -58,6 +58,12 @@ jobs: modules: 'qtcharts' cache: true + - name: Install missing Python packages + run: | + python3 -m pip install pip --upgrade + python3 -m pip install pytest + python3 -m pip install pytest-timeout + # TODO: disable warnings - name: CMake run: | @@ -77,6 +83,22 @@ jobs: - name: Run tests run: ./cmake.output/bin/testrunner + - name: Run cfg tests + run: | + cmake --build cmake.output --target checkcfg -- -j $(nproc) + + # TODO: we should use CTest instead to parallelize tests but the start-up overhead will slow things down + - name: Run CTest + if: false + run: | + ctest --test-dir cmake.output --output-on-failure -j$(nproc) + + - name: Run test/cli + run: | + pwd=$(pwd) + cd test/cli + TEST_CPPCHECK_EXE_LOOKUP_PATH="$pwd/cmake.output" python3 -m pytest -Werror --strict-markers -vv test-*.py + - name: Generate dependencies run: | # make sure auto-generated GUI files exist diff --git a/test/cli/testutils.py b/test/cli/testutils.py index 589a823cb..1dd932a9e 100644 --- a/test/cli/testutils.py +++ b/test/cli/testutils.py @@ -51,14 +51,23 @@ def __lookup_cppcheck_exe(): if sys.platform == "win32": exe_name += ".exe" - for base in (script_path + '/../../', './'): - for path in ('', 'bin/', 'bin/debug/'): - exe_path = base + path + exe_name - if os.path.isfile(exe_path): - print("using '{}'".format(exe_path)) - return exe_path + exe_path = None - return None + if 'TEST_CPPCHECK_EXE_LOOKUP_PATH' in os.environ: + lookup_paths = [os.environ['TEST_CPPCHECK_EXE_LOOKUP_PATH']] + else: + lookup_paths = [os.path.join(script_path, '..', '..'), '.'] + + for base in lookup_paths: + for path in ('', 'bin', os.path.join('bin', 'debug')): + tmp_exe_path = os.path.join(base, path, exe_name) + if os.path.isfile(tmp_exe_path): + exe_path = tmp_exe_path + break + + if exe_path: + print("using '{}'".format(exe_path)) + return exe_path # Run Cppcheck with args