Add support for generating CSV files

Signed-off-by: David A. Wheeler <dwheeler@dwheeler.com>

ChangeLog

@@ -1,5 +1,7 @@
 2017-07-29 David A. Wheeler <dwheeler, at, dwheeler.com>
 	* Change version numbers to use Semantic Versioning (x.y.z)
+	* Add support for generating CSV (comma-separated value) format,
+	  to make this tool easier to integrate into larger toolsuites.
 	* Change version number to 2.0.0, because we have a subtle
 	  interface change that won't affect most people but it
 	  *may* affect those who use postprocess

correct-results.csv

@@ -0,0 +1,37 @@
+File,Line,Column,Level,Category,Name,Warning,Suggestion,Note,CWEs,Context
+test.c,32,2,5,buffer,gets,"Does not check for buffer overflows (CWE-120, CWE-20)",Use fgets() instead,,"CWE-120, CWE-20", gets(f);
+test.c,56,3,5,buffer,strncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120)","Consider strcat_s, strlcat, snprintf, or automatically resizing strings","Risk is high; the length parameter appears to be a constant, instead of computing the number of characters left.",CWE-120,"  strncat(d,s,sizeof(d)); /* Misuse - this should be flagged as riskier. */"
+test.c,57,3,5,buffer,_tcsncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120)","Consider strcat_s, strlcat, or automatically resizing strings","Risk is high; the length parameter appears to be a constant, instead of computing the number of characters left.",CWE-120,"  _tcsncat(d,s,sizeof(d)); /* Misuse - flag as riskier */"
+test.c,60,3,5,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is high, it appears that the size is given as bytes, but the function requires size as characters.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof(wszUserName));"
+test.c,62,3,5,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is high, it appears that the size is given as bytes, but the function requires size as characters.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof wszUserName);"
+test.c,73,3,5,misc,SetSecurityDescriptorDacl,"Never create NULL ACLs; an attacker can set it to Everyone (Deny All Access), which would even forbid administrator access (CWE-732)",,,CWE-732,"  SetSecurityDescriptorDacl(&sd,TRUE,NULL,FALSE);"
+test.c,73,3,5,misc,SetSecurityDescriptorDacl,"Never create NULL ACLs; an attacker can set it to Everyone (Deny All Access), which would even forbid administrator access (CWE-732)",,,CWE-732,"  SetSecurityDescriptorDacl(&sd,TRUE,NULL,FALSE);"
+test.c,17,2,4,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused)",,CWE-120," strcpy(b, a);"
+test.c,20,2,4,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",,CWE-120," sprintf(s, ""hello %s"", bug);"
+test.c,21,2,4,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",,CWE-120," sprintf(s, gettext(""hello %s""), bug);"
+test.c,22,2,4,format,sprintf,Potential format string problem (CWE-134),Make format string constant,,CWE-134," sprintf(s, unknown, bug);"
+test.c,23,2,4,format,printf,"If format strings can be influenced by an attacker, they can be exploited (CWE-134)",Use a constant for the format specification,,CWE-134," printf(bf, x);"
+test.c,25,2,4,buffer,scanf,"The scanf() family's %s operation, without a limit specification, permits buffer overflows (CWE-120, CWE-20)","Specify a limit to %s, or use a different input function",,"CWE-120, CWE-20"," scanf(""%s"", s);"
+test.c,27,2,4,buffer,scanf,"The scanf() family's %s operation, without a limit specification, permits buffer overflows (CWE-120, CWE-20)","Specify a limit to %s, or use a different input function",,"CWE-120, CWE-20"," scanf(""%s"", s);"
+test.c,38,2,4,format,syslog,"If syslog's format strings can be influenced by an attacker, they can be exploited (CWE-134)",Use a constant format string for syslog,,CWE-134," syslog(LOG_ERR, attacker_string);"
+test.c,49,3,4,buffer,_mbscpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),Consider using a function version that stops copying at the end of the buffer,,CWE-120,"  _mbscpy(d,s); /* like strcpy, this doesn't check for buffer overflow */"
+test.c,52,3,4,buffer,lstrcat,Does not check for buffer overflows when concatenating to destination [MS-banned] (CWE-120),,,CWE-120,"  lstrcat(d,s);"
+test.c,75,3,3,shell,CreateProcess,This causes a new process to execute and is difficult to use safely (CWE-78),"Specify the application path in the first argument, NOT as part of the second, or embedded spaces could allow an attacker to force a different program to run",,CWE-78,"  CreateProcess(NULL, ""C:\\Program Files\\GoodGuy\\GoodGuy.exe -x"", """");"
+test.c,75,3,3,shell,CreateProcess,This causes a new process to execute and is difficult to use safely (CWE-78),"Specify the application path in the first argument, NOT as part of the second, or embedded spaces could allow an attacker to force a different program to run",,CWE-78,"  CreateProcess(NULL, ""C:\\Program Files\\GoodGuy\\GoodGuy.exe -x"", """");"
+test.c,91,20,3,buffer,getopt_long,"Some older implementations do not protect against internal buffer overflows (CWE-120, CWE-20)","Check implementation on installation, or limit the size of all string inputs",,"CWE-120, CWE-20","    while ((optc = getopt_long (argc, argv, ""a"",longopts, NULL )) != EOF) {"
+test.c,16,2,2,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused)",Risk is low because the source is a constant string.,CWE-120," strcpy(a, gettext(""Hello there"")); // Did this work?"
+test.c,19,2,2,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",Risk is low because the source has a constant maximum length.,CWE-120," sprintf(s, ""hello"");"
+test.c,45,3,2,buffer,char,"Statically-sized arrays can be improperly restricted, leading to potential overflows or other issues (CWE-119!/CWE-120)","Perform bounds checking, use functions that limit length, or ensure that the size is larger than the maximum possible length",,CWE-119!/CWE-120,  char d[20];
+test.c,46,3,2,buffer,char,"Statically-sized arrays can be improperly restricted, leading to potential overflows or other issues (CWE-119!/CWE-120)","Perform bounds checking, use functions that limit length, or ensure that the size is larger than the maximum possible length",,CWE-119!/CWE-120,  char s[20];
+test.c,50,3,2,buffer,memcpy,Does not check for buffer overflows when copying to destination (CWE-120),Make sure destination can always hold the source data,,CWE-120,"  memcpy(d,s);"
+test.c,51,3,2,buffer,CopyMemory,Does not check for buffer overflows when copying to destination (CWE-120),Make sure destination can always hold the source data,,CWE-120,"  CopyMemory(d,s);"
+test.c,97,7,2,misc,fopen,"Check when opening files - can an attacker redirect it (via symlinks), force the opening of special file type (e.g., device files), move things around to create a race condition, control its ancestors, or change its contents? (CWE-362)",,,CWE-362,"  f = fopen(""/etc/passwd"", ""r""); "
+test.c,15,2,1,buffer,strcpy,Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120),"Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused)",Risk is low because the source is a constant character.,CWE-120," strcpy(a, ""\n""); // Did this work?"
+test.c,18,2,1,buffer,sprintf,Does not check for buffer overflows (CWE-120),"Use sprintf_s, snprintf, or vsnprintf",Risk is low because the source is a constant character.,CWE-120," sprintf(s, ""\n"");"
+test.c,26,2,1,buffer,scanf,It's unclear if the %s limit in the format string is small enough (CWE-120),"Check that the limit is sufficiently small, or use a different input function",,CWE-120," scanf(""%10s"", s);"
+test.c,53,3,1,buffer,strncpy,Easily used incorrectly; doesn't always \0-terminate or check for invalid pointers [MS-banned] (CWE-120),,,CWE-120,"  strncpy(d,s);"
+test.c,54,3,1,buffer,_tcsncpy,Easily used incorrectly; doesn't always \0-terminate or check for invalid pointers [MS-banned] (CWE-120),,,CWE-120,"  _tcsncpy(d,s);"
+test.c,55,3,1,buffer,strncat,"Easily used incorrectly (e.g., incorrectly computing the correct maximum size to add) [MS-banned] (CWE-120)","Consider strcat_s, strlcat, snprintf, or automatically resizing strings",,CWE-120,"  strncat(d,s,10);"
+test.c,58,7,1,buffer,strlen,Does not handle strings that are not \0-terminated; if given one it may perform an over-read (it could cause a crash if unprotected) (CWE-126),,,CWE-126,  n = strlen(d);
+test.c,64,3,1,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is very low, the length appears to be in characters not bytes.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof(wszUserName)/sizeof(wszUserName[0]));"
+test.c,66,3,1,buffer,MultiByteToWideChar,"Requires maximum length in CHARACTERS, not bytes (CWE-120)",,"Risk is very low, the length appears to be in characters not bytes.",CWE-120,"  MultiByteToWideChar(CP_ACP,0,szName,-1,wszUserName,sizeof wszUserName /sizeof(wszUserName[0]));"

flawfinder

@@ -61,6 +61,7 @@ import pickle               # To support load/save/diff of hitlist
 import os, glob, operator   # To support filename expansion on Windows
 import os.path
 import time
+import csv                  # To support generating CSV format
 # import formatter
 
 # Program Options - these are the default values:
@@ -85,6 +86,8 @@ quiet = 0
 showheading = 1          # --dataonly turns this off
 output_format = 0        # 0 = normal, 1 = html.
 single_line = 0          # 1 = singleline (can 't be 0 if html)
+csv_output = 0           # 1 = Generate CSV
+csv_writer = None
 omit_time = 0            # 1 = omit time-to-run (needed for testing)
 required_regex = None    # If non-None, regex that must be met to report
 required_regex_compiled = None
@@ -317,6 +320,9 @@ def print_multi_line(text):
 # We don't refer to CWEs with one digit, so we'll only match on 2+ digits.
 link_cwe_pattern = re.compile(r'(CWE-([1-9][0-9]+))([,()])')
 
+# This matches the CWE data, including multiple entries.
+find_cwe_pattern = re.compile(r'\(CWE-[^)]*\)')
+
 class Hit:
   """
   Each instance of Hit is a warning of some kind in a source code file.
@@ -376,7 +382,24 @@ class Hit:
   def __getitem__(self, X):   # Define this so this works: "%(line)" % hit
     return getattr(self, X)
 
+  # return CWEs
+  def cwes(self):
+    result = find_cwe_pattern.search(self.warning)
+    if result:
+      return result.group()[1:-1]
+    else:
+      return ''
+
+  # Show as CSV format
+  def show_csv(self):
+    csv_writer.writerow([self.filename, self.line, self.column, self.level,
+      self.category, self.name, self.warning,
+      self.suggestion, self.note, self.cwes(), self.context_text ])
+
   def show(self):
+    if csv_output:
+        self.show_csv()
+        return
     if output_format: print "<li>",
     sys.stdout.write(h(self.filename))
 
@@ -411,8 +434,6 @@ class Hit:
       print h(self.context_text)
       if output_format: print "</pre>"
 
-
-
 # The "hitlist" is the list of all hits (warnings) found so far.
 # Use add_warning to add to it.
 
@@ -1507,6 +1528,11 @@ def initialize_ruleset():
 # Show the header, but only if it hasn't been shown yet.
 def display_header():
   global displayed_header
+  if csv_output:
+    csv_writer.writerow(['File', 'Line', 'Column', 'Level',
+      'Category', 'Name', 'Warning',
+      'Suggestion', 'Note', 'CWEs', 'Context'])
+    return
   if not showheading: return
   if not displayed_header:
     if output_format:
@@ -1709,6 +1735,7 @@ flawfinder [--help | -h] [--version] [--listrules]
 def process_options():
   global show_context, show_inputs, allowlink, skipdotdir, omit_time
   global output_format, minimum_level, show_immediately, single_line
+  global csv_output, csv_writer
   global required_regex, required_regex_compiled
   global falsepositive
   global show_columns, never_ignore, quiet, showheading, list_rules
@@ -1722,7 +1749,7 @@ def process_options():
                      "columns", "listrules", "omittime", "allowlink", "patch=",
                      "followdotdir",
                      "neverignore", "regex=",
-                     "quiet", "dataonly", "html", "singleline",
+                     "quiet", "dataonly", "html", "singleline", "csv",
                      "loadhitlist=", "savehitlist=", "diffhitlist=",
                      "version", "help" ])
     for (opt,value) in optlist:
@@ -1756,6 +1783,11 @@ def process_options():
         minimum_level = string.atoi(value)
       elif opt == "--singleline" or opt == "-S":
         single_line = 1
+      elif opt == "--csv":
+        csv_output = 1
+        quiet = 1
+        showheading = 0
+        csv_writer = csv.writer(sys.stdout)
       elif opt == "--immediate" or opt == "-i":
         show_immediately = 1
       elif opt == "-n" or opt == "--neverignore":

makefile

@@ -127,6 +127,7 @@ time:
 test: flawfinder test.c test2.c
 	# Omit time report so that results are always the same textually.
 	./flawfinder --omittime test.c test2.c > test-results.txt
+	./flawfinder --csv test.c test2.c > test-results.csv
 	echo >> test-results.txt
 	echo "Testing for no ending newline:" >> test-results.txt
 	./flawfinder --omittime no-ending-newline.c | \
@@ -135,6 +136,7 @@ test: flawfinder test.c test2.c
 	@echo "Differences from expected results:"
 	@diff -u correct-results.txt test-results.txt
 	@diff -u correct-results.html test-results.html
+	@diff -u correct-results.csv test-results.csv
 
 check: test