Travis: Improve bug hunting testing

This commit is contained in:
Daniel Marjamäki 2020-05-07 18:55:22 +02:00
parent 61926ed7c3
commit c04a73f6d8
2 changed files with 26 additions and 16 deletions

View File

@ -57,6 +57,23 @@ matrix:
compiler: gcc compiler: gcc
script: script:
- CXXFLAGS="-fsanitize=address -Og -g3" make cppcheck check checkcfg -j 2 -s - CXXFLAGS="-fsanitize=address -Og -g3" make cppcheck check checkcfg -j 2 -s
# bug hunting
- name: "bug hunting"
compiler: gcc
script:
- make clean
- make USE_Z3=yes -j2
- python3 test/bug-hunting/cve.py
- git clone https://github.com/regehr/itc-benchmarks.git ~/itc
- python3 test/bug-hunting/itc.py
- mkdir ~/juliet
- curl https://samate.nist.gov/SARD/testsuites/juliet/Juliet_Test_Suite_v1.3_for_C_Cpp.zip -o ~/juliet/juliet.zip
- cd ~/juliet
- unzip -qq ~/juliet/juliet.zip
- cd -
- python3 test/bug-hunting/juliet.py
# check a lot of stuff that only needs to be checked in a single configuration # check a lot of stuff that only needs to be checked in a single configuration
- name: "misc" - name: "misc"
compiler: clang compiler: clang
@ -156,22 +173,16 @@ matrix:
- mkdir cmake.output - mkdir cmake.output
- cd cmake.output - cd cmake.output
- cmake -G "Unix Makefiles" -DBUILD_TESTS=On .. - cmake -G "Unix Makefiles" -DBUILD_TESTS=On ..
- make -j2 check - make -s -j2 check
- cd .. - cd ..
# -funsigned-char # -funsigned-char
- make clean - make clean
- make -j2 CXXFLAGS=-funsigned-char testrunner - make -s -j2 CXXFLAGS=-funsigned-char testrunner
- ./testrunner TestSymbolDatabase - ./testrunner TestSymbolDatabase
# check .json files # check .json files
- find . -name '*.json' -not -path '*/\.*' | xargs jsonlint -s - find . -name '*.json' -not -path '*/\.*' | xargs jsonlint -s
# build fuzz client # build fuzz client
- make -j2 CXXFLAGS="-fsanitize=address" -C oss-fuzz fuzz-client - make -s -j2 CXXFLAGS="-fsanitize=address" -C oss-fuzz fuzz-client
# bug hunting
- make clean
- make -s USE_Z3=yes -j2
- python3 test/bug-hunting/cve.py
- git clone https://github.com/regehr/itc-benchmarks.git ~/itc
- python3 test/bug-hunting/itc.py
# check if dmake needs to be rerun (this job may fail) # check if dmake needs to be rerun (this job may fail)
- name: "rerun dmake?" - name: "rerun dmake?"

View File

@ -60,19 +60,18 @@ def check(tc:str, warning_id:str):
if warning_id in stderr: if warning_id in stderr:
num_ok += 1 num_ok += 1
else: else:
print(f'fail: ' + ' '.join(cmd)) print('fail: ' + ' '.join(cmd))
num_failed += 1 num_failed += 1
cwepos = tc.find('CWE') cwepos = tc.find('CWE')
cwe = tc[cwepos:cwepos+6] cwe = tc[cwepos:cwepos+6]
return f'{cwe} ok:{num_ok}, fail:{num_failed}\n' print('%s ok:%i, fail:%i' % (cwe, num_ok, num_failed))
if num_failed != 0:
sys.exit(1)
final_report = '' check('C/testcases/CWE369_Divide_by_Zero/s*/*.c', 'bughuntingDivByZero')
final_report += check('C/testcases/CWE369_Divide_by_Zero/s*/*.c', 'bughuntingDivByZero') #check('C/testcases/CWE457_Use_of_Uninitialized_Variable/s*/*.c', 'verificationUninit')
#final_report += check('C/testcases/CWE457_Use_of_Uninitialized_Variable/s*/*.c', 'verificationUninit')
print(final_report)