diff --git a/.travis.yml b/.travis.yml index 526b1e2ae..20d474a6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,6 +57,23 @@ matrix: compiler: gcc script: - CXXFLAGS="-fsanitize=address -Og -g3" make cppcheck check checkcfg -j 2 -s + +# bug hunting + - name: "bug hunting" + compiler: gcc + script: + - make clean + - make USE_Z3=yes -j2 + - python3 test/bug-hunting/cve.py + - git clone https://github.com/regehr/itc-benchmarks.git ~/itc + - python3 test/bug-hunting/itc.py + - mkdir ~/juliet + - curl https://samate.nist.gov/SARD/testsuites/juliet/Juliet_Test_Suite_v1.3_for_C_Cpp.zip -o ~/juliet/juliet.zip + - cd ~/juliet + - unzip -qq ~/juliet/juliet.zip + - cd - + - python3 test/bug-hunting/juliet.py + # check a lot of stuff that only needs to be checked in a single configuration - name: "misc" compiler: clang @@ -156,22 +173,16 @@ matrix: - mkdir cmake.output - cd cmake.output - cmake -G "Unix Makefiles" -DBUILD_TESTS=On .. - - make -j2 check + - make -s -j2 check - cd .. # -funsigned-char - make clean - - make -j2 CXXFLAGS=-funsigned-char testrunner + - make -s -j2 CXXFLAGS=-funsigned-char testrunner - ./testrunner TestSymbolDatabase # check .json files - find . -name '*.json' -not -path '*/\.*' | xargs jsonlint -s # build fuzz client - - make -j2 CXXFLAGS="-fsanitize=address" -C oss-fuzz fuzz-client -# bug hunting - - make clean - - make -s USE_Z3=yes -j2 - - python3 test/bug-hunting/cve.py - - git clone https://github.com/regehr/itc-benchmarks.git ~/itc - - python3 test/bug-hunting/itc.py + - make -s -j2 CXXFLAGS="-fsanitize=address" -C oss-fuzz fuzz-client # check if dmake needs to be rerun (this job may fail) - name: "rerun dmake?" diff --git a/test/bug-hunting/juliet.py b/test/bug-hunting/juliet.py index 3de6c47fc..67e371524 100644 --- a/test/bug-hunting/juliet.py +++ b/test/bug-hunting/juliet.py @@ -60,19 +60,18 @@ def check(tc:str, warning_id:str): if warning_id in stderr: num_ok += 1 else: - print(f'fail: ' + ' '.join(cmd)) + print('fail: ' + ' '.join(cmd)) num_failed += 1 cwepos = tc.find('CWE') cwe = tc[cwepos:cwepos+6] - return f'{cwe} ok:{num_ok}, fail:{num_failed}\n' + print('%s ok:%i, fail:%i' % (cwe, num_ok, num_failed)) + if num_failed != 0: + sys.exit(1) -final_report = '' -final_report += check('C/testcases/CWE369_Divide_by_Zero/s*/*.c', 'bughuntingDivByZero') -#final_report += check('C/testcases/CWE457_Use_of_Uninitialized_Variable/s*/*.c', 'verificationUninit') - -print(final_report) +check('C/testcases/CWE369_Divide_by_Zero/s*/*.c', 'bughuntingDivByZero') +#check('C/testcases/CWE457_Use_of_Uninitialized_Variable/s*/*.c', 'verificationUninit')