From 025cee45dbf458b4f427ed6717de8de75c1bc9c5 Mon Sep 17 00:00:00 2001 From: Julien Hachenberger Date: Mon, 23 Sep 2019 11:19:22 +0200 Subject: [PATCH 1/3] Move all test related files into a subdirectory In order to keep all test-related files within one folder, this commit creates a new folder 'test/', moves the test-related files into it and intrdocues a second makefile within this folder dedicated to run the tests. The original makefile still contains a 'test' target, but simply delegates this task to the second makefile. (fixes david-a-wheeler/flawfinder#11) Signed-off-by: Julien Hachenberger --- makefile | 95 ++----------------- .../correct-results-004.txt | 0 .../correct-results-005.txt | 0 .../correct-results-006.txt | 0 .../correct-results-008.txt | 0 .../correct-results.csv | 0 .../correct-results.html | 0 .../correct-results.txt | 0 test/makefile | 94 ++++++++++++++++++ .../no-ending-newline.c | 0 .../test-diff-005.patch | 0 test-patched.c => test/test-patched.c | 0 test.c => test/test.c | 0 test2.c => test/test2.c | 0 14 files changed, 101 insertions(+), 88 deletions(-) rename correct-results-004.txt => test/correct-results-004.txt (100%) mode change 100755 => 100644 rename correct-results-005.txt => test/correct-results-005.txt (100%) mode change 100755 => 100644 rename correct-results-006.txt => test/correct-results-006.txt (100%) mode change 100755 => 100644 rename correct-results-008.txt => test/correct-results-008.txt (100%) rename correct-results.csv => test/correct-results.csv (100%) mode change 100755 => 100644 rename correct-results.html => test/correct-results.html (100%) mode change 100755 => 100644 rename correct-results.txt => test/correct-results.txt (100%) mode change 100755 => 100644 create mode 100644 test/makefile rename no-ending-newline.c => test/no-ending-newline.c (100%) rename test-diff-005.patch => test/test-diff-005.patch (100%) rename test-patched.c => test/test-patched.c (100%) rename test.c => test/test.c (100%) rename test2.c => test/test2.c (100%) diff --git a/makefile b/makefile index e3b38d7..9257f61 100644 --- a/makefile +++ b/makefile @@ -12,9 +12,6 @@ VERSIONEDNAME=$(NAME)-$(VERSION) ARCH=noarch SAMPLE_DIR=/usr/src/linux-2.2.16 -PYTHON=python -PYTHON2=python2 -PYTHON3=python3 # Flawfinder has traditionally used INSTALL_DIR, INSTALL_DIR_BIN, and # INSTALL_DIR_MAN. Here we add support for GNU variables like prefix, etc.; @@ -49,6 +46,8 @@ RPMBUILD=rpmbuild DESTDIR= +TESTDIR=test/ + all: flawfinder.pdf flawfinder.1.gz chmod -R a+rX * @@ -100,7 +99,7 @@ distribute: clean flawfinder.pdf flawfinder.ps rm -fr build dist flawfinder.egg-info ,tempdir chmod -R a+rX * mkdir ,tempdir - cp -p [a-zA-Z]* ,tempdir + cp -r -p [a-zA-Z]* ,tempdir rm -f ,tempdir/*.tar.gz rm -f ,tempdir/*.rpm # We don't need both "flawfinder" and "flawfinder.py": @@ -133,88 +132,8 @@ time: echo "Lines examined:" wc -l $(SAMPLE_DIR)/*/*.[ch] | tail -2 -test_001: flawfinder test.c test2.c - @echo 'test_001 (text output)' - @# Omit time report so that results are always the same textually. - @$(PYTHON) ./flawfinder --omittime test.c test2.c > test-results.txt - @echo >> test-results.txt - @echo "Testing for no ending newline:" >> test-results.txt - @$(PYTHON) ./flawfinder --omittime no-ending-newline.c | \ - grep 'Lines analyzed' >> test-results.txt - @diff -u correct-results.txt test-results.txt - -test_002: flawfinder test.c test2.c - @echo 'test_002 (HTML output)' - @$(PYTHON) ./flawfinder --omittime --html --context test.c test2.c > test-results.html - @diff -u correct-results.html test-results.html - -test_003: flawfinder test.c test2.c - @echo 'test_003 (CSV output)' - @$(PYTHON) ./flawfinder --csv test.c test2.c > test-results.csv - @diff -u correct-results.csv test-results.csv - -test_004: flawfinder test.c - @echo 'test_004 (single-line)' - @$(PYTHON) ./flawfinder -m 5 -S -DC --quiet test.c > \ - test-results-004.txt - @diff -u correct-results-004.txt test-results-004.txt - -test_005: flawfinder test-diff-005.patch test-patched.c - @echo 'test_005 (diff)' - @$(PYTHON) ./flawfinder -SQDC -P test-diff-005.patch \ - test-patched.c > test-results-005.txt - @diff -u correct-results-005.txt test-results-005.txt - -test_006: flawfinder test.c - @echo 'test_006 (save/load hitlist)' - @$(PYTHON) ./flawfinder -S -DC --quiet \ - --savehitlist test-saved-hitlist-006.txt \ - test.c > test-junk-006.txt - @$(PYTHON) ./flawfinder -SQDC -m 5 \ - --loadhitlist test-saved-hitlist-006.txt > \ - test-results-006.txt - @diff -u correct-results-006.txt test-results-006.txt - -test_007: setup.py - @echo 'test_007 (setup.py sane)' - @test "`$(PYTHON) setup.py --name`" = 'flawfinder' - @test "`$(PYTHON) setup.py --license`" = 'GPL-2.0+' - @test "`$(PYTHON) setup.py --author`" = 'David A. Wheeler' - -test_008: flawfinder test.c - @echo 'test_008 (diff hitlist)' - @$(PYTHON) ./flawfinder -S -DC --quiet \ - --savehitlist test-saved-hitlist-008.txt \ - test.c > test-junk-008.txt - @$(PYTHON) ./flawfinder -S -C --quiet --omittime \ - --diffhitlist test-saved-hitlist-008.txt test.c > \ - test-results-008.txt - @diff -u correct-results-008.txt test-results-008.txt - -# Run all tests on *one* version of Python; -# output shows differences from expected results. -# If everything works as expected, it just prints test numbers. -# Set PYTHON as needed, including to "" -test: test_001 test_002 test_003 test_004 test_005 test_006 test_007 test_008 - @echo 'All tests pass!' - -# Usual check routine. Run all tests using *both* python2 and python3. -check: - @echo "Testing with $(PYTHON2)" - @PYTHON="$(PYTHON2)" $(MAKE) test - @echo - @echo "Testing with $(PYTHON3)" - @PYTHON="$(PYTHON3)" $(MAKE) test - -# Run "make test-is-correct" if the results are as expected. -test-is-correct: test-results.txt - cp -p test-results.txt correct-results.txt - cp -p test-results.html correct-results.html - cp -p test-results.csv correct-results.csv - cp -p test-results-004.txt correct-results-004.txt - cp -p test-results-005.txt correct-results-005.txt - cp -p test-results-006.txt correct-results-006.txt - cp -p test-results-008.txt correct-results-008.txt +test: + $(MAKE) -C $(TESTDIR) test profile: /usr/lib/python1.5/profile.py ./flawfinder > profile-results $(SAMPLE_DIR)/*/*.[ch] > profile-results @@ -245,7 +164,7 @@ my-install: flawfinder.pdf flawfinder.ps test cp -p $(VERSIONEDNAME).tar.gz \ flawfinder flawfinder.1 makefile \ flawfinder.pdf flawfinder.ps ChangeLog \ - test.c test2.c test-results.txt test-results.html \ + $(TESTDIR)/test.c $(TESTDIR)/test2.c $(TESTDIR)/test-results.txt $(TESTDIR)/test-results.html \ /home/dwheeler/dwheeler.com/flawfinder/ # This is intended to be a local capability to list CWEs @@ -261,7 +180,7 @@ show-cwes: cwe pylint: pylint flawfinder -.PHONY: install clean test check profile test-is-correct rpm \ +.PHONY: install clean test profile rpm \ uninstall distribute my-install show-cwes pylint # When I switch to using "DistUtils", I may need to move the MANIFEST.in diff --git a/correct-results-004.txt b/test/correct-results-004.txt old mode 100755 new mode 100644 similarity index 100% rename from correct-results-004.txt rename to test/correct-results-004.txt diff --git a/correct-results-005.txt b/test/correct-results-005.txt old mode 100755 new mode 100644 similarity index 100% rename from correct-results-005.txt rename to test/correct-results-005.txt diff --git a/correct-results-006.txt b/test/correct-results-006.txt old mode 100755 new mode 100644 similarity index 100% rename from correct-results-006.txt rename to test/correct-results-006.txt diff --git a/correct-results-008.txt b/test/correct-results-008.txt similarity index 100% rename from correct-results-008.txt rename to test/correct-results-008.txt diff --git a/correct-results.csv b/test/correct-results.csv old mode 100755 new mode 100644 similarity index 100% rename from correct-results.csv rename to test/correct-results.csv diff --git a/correct-results.html b/test/correct-results.html old mode 100755 new mode 100644 similarity index 100% rename from correct-results.html rename to test/correct-results.html diff --git a/correct-results.txt b/test/correct-results.txt old mode 100755 new mode 100644 similarity index 100% rename from correct-results.txt rename to test/correct-results.txt diff --git a/test/makefile b/test/makefile new file mode 100644 index 0000000..b4d4c5a --- /dev/null +++ b/test/makefile @@ -0,0 +1,94 @@ +# Flawfinder. +# Released under the General Public License (GPL) version 2 or later. +# (C) 2001-2017 David A. Wheeler. + +PYTHON=python +PYTHON2=python2 +PYTHON3=python3 +FLAWFINDER=../flawfinder +SETUPPY=../setup.py + +test_001: $(FLAWFINDER) test.c test2.c + @echo 'test_001 (text output)' + @# Omit time report so that results are always the same textually. + @$(PYTHON) $(FLAWFINDER) --omittime test.c test2.c > test-results.txt + @echo >> test-results.txt + @echo "Testing for no ending newline:" >> test-results.txt + @$(PYTHON) $(FLAWFINDER) --omittime no-ending-newline.c | \ + grep 'Lines analyzed' >> test-results.txt + @diff -u correct-results.txt test-results.txt + +test_002: $(FLAWFINDER) test.c test2.c + @echo 'test_002 (HTML output)' + @$(PYTHON) $(FLAWFINDER) --omittime --html --context test.c test2.c > test-results.html + @diff -u correct-results.html test-results.html + +test_003: $(FLAWFINDER) test.c test2.c + @echo 'test_003 (CSV output)' + @$(PYTHON) $(FLAWFINDER) --csv test.c test2.c > test-results.csv + @diff -u correct-results.csv test-results.csv + +test_004: $(FLAWFINDER) test.c + @echo 'test_004 (single-line)' + @$(PYTHON) $(FLAWFINDER) -m 5 -S -DC --quiet test.c > \ + test-results-004.txt + @diff -u correct-results-004.txt test-results-004.txt + +test_005: $(FLAWFINDER) test-diff-005.patch test-patched.c + @echo 'test_005 (diff)' + @$(PYTHON) $(FLAWFINDER) -SQDC -P test-diff-005.patch \ + test-patched.c > test-results-005.txt + @diff -u correct-results-005.txt test-results-005.txt + +test_006: $(FLAWFINDER) test.c + @echo 'test_006 (save/load hitlist)' + @$(PYTHON) $(FLAWFINDER) -S -DC --quiet \ + --savehitlist test-saved-hitlist-006.txt \ + test.c > test-junk-006.txt + @$(PYTHON) $(FLAWFINDER) -SQDC -m 5 \ + --loadhitlist test-saved-hitlist-006.txt > \ + test-results-006.txt + @diff -u correct-results-006.txt test-results-006.txt + +test_007: $(SETUPPY) + @echo 'test_007 (setup.py sane)' + @test "`$(PYTHON) $(SETUPPY) --name`" = 'flawfinder' + @test "`$(PYTHON) $(SETUPPY) --license`" = 'GPL-2.0+' + @test "`$(PYTHON) $(SETUPPY) --author`" = 'David A. Wheeler' + +test_008: $(FLAWFINDER) test.c + @echo 'test_008 (diff hitlist)' + @$(PYTHON) $(FLAWFINDER) -S -DC --quiet \ + --savehitlist test-saved-hitlist-008.txt \ + test.c > test-junk-008.txt + @$(PYTHON) $(FLAWFINDER) -S -C --quiet --omittime \ + --diffhitlist test-saved-hitlist-008.txt test.c > \ + test-results-008.txt + @diff -u correct-results-008.txt test-results-008.txt + +# Run all tests on *one* version of Python; +# output shows differences from expected results. +# If everything works as expected, it just prints test numbers. +# Set PYTHON as needed, including to "" +test: test_001 test_002 test_003 test_004 test_005 test_006 test_007 test_008 + @echo 'All tests pass!' + +# Usual check routine. Run all tests using *both* python2 and python3. +check: + @echo "Testing with $(PYTHON2)" + @PYTHON="$(PYTHON2)" $(MAKE) test + @echo + @echo "Testing with $(PYTHON3)" + @PYTHON="$(PYTHON3)" $(MAKE) test + +# Run "make test-is-correct" if the results are as expected. +test-is-correct: test-results.txt + cp -p test-results.txt correct-results.txt + cp -p test-results.html correct-results.html + cp -p test-results.csv correct-results.csv + cp -p test-results-004.txt correct-results-004.txt + cp -p test-results-005.txt correct-results-005.txt + cp -p test-results-006.txt correct-results-006.txt + cp -p test-results-008.txt correct-results-008.txt + +.PHONY: test check test-is-correct diff --git a/no-ending-newline.c b/test/no-ending-newline.c similarity index 100% rename from no-ending-newline.c rename to test/no-ending-newline.c diff --git a/test-diff-005.patch b/test/test-diff-005.patch similarity index 100% rename from test-diff-005.patch rename to test/test-diff-005.patch diff --git a/test-patched.c b/test/test-patched.c similarity index 100% rename from test-patched.c rename to test/test-patched.c diff --git a/test.c b/test/test.c similarity index 100% rename from test.c rename to test/test.c diff --git a/test2.c b/test/test2.c similarity index 100% rename from test2.c rename to test/test2.c From 717d0352060fd4eaae01751b49bbcf6a6195866b Mon Sep 17 00:00:00 2001 From: Julien Hachenberger Date: Tue, 24 Sep 2019 12:27:02 +0200 Subject: [PATCH 2/3] Make sub-'make' call POSIX compliant Pull request #15 introduced a `make` call using `-C` (short for: `--directory`) option which is not POSIX compliant. This commit will use the `cd` builtin instead to maintain POSIX compliance. Signed-off-by: Julien Hachenberger --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 9257f61..7e623bb 100644 --- a/makefile +++ b/makefile @@ -133,7 +133,7 @@ time: wc -l $(SAMPLE_DIR)/*/*.[ch] | tail -2 test: - $(MAKE) -C $(TESTDIR) test + cd $(TESTDIR); $(MAKE) test profile: /usr/lib/python1.5/profile.py ./flawfinder > profile-results $(SAMPLE_DIR)/*/*.[ch] > profile-results From be85f402915ab0f8527e2f125a0fb1d850a39c8e Mon Sep 17 00:00:00 2001 From: Julien Hachenberger Date: Tue, 24 Sep 2019 12:43:07 +0200 Subject: [PATCH 3/3] Recreate missing make targets Pull request #15 moved `check` and `test-is-correct` to test/makefile making them inaccessible by the main makefile. This commit recreates both targets again and simply passesthrough the calls to the corresponding sub makefile. Signed-off-by: Julien Hachenberger --- makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/makefile b/makefile index 7e623bb..81b97b4 100644 --- a/makefile +++ b/makefile @@ -135,6 +135,14 @@ time: test: cd $(TESTDIR); $(MAKE) test +# Usual check routine. Run all tests using *both* python2 and python3. +check: + cd $(TESTDIR); $(MAKE) check + +# Run "make test-is-correct" if the results are as expected. +test-is-correct: + cd $(TESTDIR); $(MAKE) test-is-correct + profile: /usr/lib/python1.5/profile.py ./flawfinder > profile-results $(SAMPLE_DIR)/*/*.[ch] > profile-results @@ -180,7 +188,7 @@ show-cwes: cwe pylint: pylint flawfinder -.PHONY: install clean test profile rpm \ +.PHONY: install clean test check profile test-is-correct rpm \ uninstall distribute my-install show-cwes pylint # When I switch to using "DistUtils", I may need to move the MANIFEST.in