From f9819b48a563f67f01623aa123734b86a43eedb6 Mon Sep 17 00:00:00 2001 From: Yong Yan Date: Wed, 28 Apr 2021 10:58:32 -0700 Subject: [PATCH 1/3] export sarif report Fix functions/variables naming update function name --- flawfinder.py | 19 +++++- sariflogger.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 sariflogger.py diff --git a/flawfinder.py b/flawfinder.py index 3c72a3d..b3d001a 100644 --- a/flawfinder.py +++ b/flawfinder.py @@ -54,6 +54,7 @@ import time import csv # To support generating CSV format import hashlib # import formatter +from sariflogger import SarifLogger version = "2.0.15" @@ -87,6 +88,7 @@ output_format = 0 # 0 = normal, 1 = html. single_line = 0 # 1 = singleline (can 't be 0 if html) csv_output = 0 # 1 = Generate CSV csv_writer = None +sarif_output = 0 # 1 = Generate SARIF report omit_time = 0 # 1 = omit time-to-run (needed for testing) required_regex = None # If non-None, regex that must be met to report required_regex_compiled = None @@ -437,6 +439,8 @@ class Hit(object): if csv_output: self.show_csv() return + if sarif_output: + return if output_format: print("
  • ", end='') sys.stdout.write(h(self.filename)) @@ -1790,6 +1794,8 @@ def display_header(): 'Suggestion', 'Note', 'CWEs', 'Context', 'Fingerprint', 'ToolVersion', 'RuleId', 'HelpUri' ]) return + if sarif_output: + return if not showheading: return if not displayed_header: @@ -2044,7 +2050,7 @@ flawfinder [--help | -h] [--version] [--listrules] def process_options(): global show_context, show_inputs, allowlink, skipdotdir, omit_time global output_format, minimum_level, show_immediately, single_line - global csv_output, csv_writer + global csv_output, csv_writer, sarif_output global error_level global required_regex, required_regex_compiled global falsepositive @@ -2058,7 +2064,7 @@ def process_options(): "falsepositive", "falsepositives", "columns", "listrules", "omittime", "allowlink", "patch=", "followdotdir", "neverignore", "regex=", "quiet", "dataonly", "html", "singleline", "csv", - "error-level=", + "error-level=", "sarif", "loadhitlist=", "savehitlist=", "diffhitlist=", "version", "help" ]) for (opt, value) in optlist: @@ -2097,6 +2103,10 @@ def process_options(): quiet = 1 showheading = 0 csv_writer = csv.writer(sys.stdout) + elif opt == "--sarif": + sarif_output = 1 + quiet = 1 + showheading = 0 elif opt == "--error-level": error_level = int(value) elif opt == "--immediate" or opt == "-i": @@ -2349,7 +2359,10 @@ def flawfind(): display_header() initialize_ruleset() if process_files(): - show_final_results() + if sarif_output: + print(SarifLogger(hitlist).output_sarif()) + else: + show_final_results() save_if_desired() return 1 if error_level_exceeded else 0 diff --git a/sariflogger.py b/sariflogger.py new file mode 100644 index 0000000..6568f31 --- /dev/null +++ b/sariflogger.py @@ -0,0 +1,164 @@ +import json +import re + +def to_json(o): + return json.dumps(o, default=lambda o: o.__dict__, sort_keys=False, indent=2) + +class SarifLogger(object): + _hitlist = None + TOOL_NAME = "Flawfinder" + TOOL_URL = "https://dwheeler.com/flawfinder/" + TOOL_VERSION = "2.0.15" + URI_BASE_ID = "SRCROOT" + SARIF_SCHEMA = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json" + SARIF_SCHEMA_VERSION = "2.1.0" + CWE_TAXONOMY_NAME = "CWE" + CWE_TAXONOMY_URI = "https://raw.githubusercontent.com/sarif-standard/taxonomies/main/CWE_v4.4.sarif" + CWE_TAXONOMY_GUID = "FFC64C90-42B6-44CE-8BEB-F6B7DAE649E5" + + def __init__ (self, hits): + self._hitlist = hits + + def output_sarif(self): + tool = { + "driver": { + "name": self.TOOL_NAME, + "version": self.TOOL_VERSION, + "informationUri": self.TOOL_URL, + "rules": self._extract_rules(self._hitlist), + "supportedTaxonomies": [{ + "name": self.CWE_TAXONOMY_NAME, + "guid": self.CWE_TAXONOMY_GUID, + }], + } + } + + runs = [{ + "tool": tool, + "columnKind": "utf16CodeUnits", + "results": self._extract_results(self._hitlist), + "externalPropertyFileReferences": { + "taxonomies": [{ + "location": { + "uri": self.CWE_TAXONOMY_URI, + }, + "guid": self.CWE_TAXONOMY_GUID, + }], + }, + }] + + report = { + "$schema": self.SARIF_SCHEMA, + "version": self.SARIF_SCHEMA_VERSION, + "runs": runs, + } + + jsonstr = to_json(report) + return jsonstr + + def _extract_rules(self, hitlist): + rules = {} + for hit in hitlist: + if not hit.ruleid in rules: + rules[hit.ruleid] = self._to_sarif_rule(hit) + return list(rules.values()) + + def _extract_results(self, hitlist): + results = [] + for hit in hitlist: + results.append(self._to_sarif_result(hit)) + return results + + def _to_sarif_rule(self, hit): + return { + "id": hit.ruleid, + "name": "{0}/{1}".format(hit.category, hit.name), + "shortDescription": { + "text": self._append_period(hit.warning), + }, + "defaultConfiguration": { + "level": self._to_sarif_level(hit.defaultlevel), + }, + "helpUri": hit.helpuri(), + "relationships": self._extract_relationships(hit.cwes()), + } + + def _to_sarif_result(self, hit): + return { + "ruleId": hit.ruleid, + "level": self._to_sarif_level(hit.level), + "message": { + "text": self._append_period("{0}/{1}:{2}".format(hit.category, hit.name, hit.warning)), + }, + "locations": [{ + "physicalLocation": { + "artifactLocation": { + "uri": self._to_uri_path(hit.filename), + "uriBaseId": self.URI_BASE_ID, + }, + "region": { + "startLine": hit.line, + "startColumn": hit.column, + "endColumn": len(hit.context_text) + 1, + "snippet": { + "text": hit.context_text, + } + } + } + }], + "fingerprints": { + "contextHash/v1": hit.fingerprint() + }, + "rank": self._to_sarif_rank(hit.level), + } + + def _extract_relationships(self, cwestring): + # example cwe string "CWE-119!/ CWE-120", "CWE-829, CWE-20" + relationships = [] + for cwe in re.split(',|/',cwestring): + cwestr = cwe.strip() + if cwestr: + relationship = { + "target": { + "id": int(cwestr.replace("CWE-", "").replace("!", "")), + "toolComponent": { + "name": self.CWE_TAXONOMY_NAME, + "guid": self.CWE_TAXONOMY_GUID, + }, + }, + "kinds": [ + "relevant" if cwestr[-1] != '!' else "incomparable" + ], + } + relationships.append(relationship) + return relationships + + @staticmethod + def _to_sarif_level(level): + # level 4 & 5 + if level >= 4: + return "error" + # level 3 + if level == 3: + return "warning" + # level 0 1 2 + return "note" + + @staticmethod + def _to_sarif_rank(level): + #SARIF rank FF Level SARIF level Default Viewer Action + #0.0 0 note Does not display by default + #0.2 1 note Does not display by default + #0.4 2 note Does not display by default + #0.6 3 warning Displays by default, does not break build / other processes + #0.8 4 error Displays by default, breaks build/ other processes + #1.0 5 error Displays by default, breaks build/ other processes + return level * 0.2 + + @staticmethod + def _to_uri_path(path): + return path.replace("\\", "/") + + @staticmethod + def _append_period(text): + return text if text[-1] == '.' else text + "." From fd503914393ac09f8ca74fa5b5221b23c20fa126 Mon Sep 17 00:00:00 2001 From: "David A. Wheeler" Date: Sun, 30 May 2021 14:39:44 -0400 Subject: [PATCH 2/3] Move sariflogger.py into flawfinder.py Flawfinder has a project-specific rule to put all code in one file. That can be a pain for development, but the rule makes *deploying* flawfinder really easy in some settings. Worse comes to worse, just copy the file somewhere and you can run it! Signed-off-by: David A. Wheeler --- flawfinder.py | 171 ++++++++++++++++++++++++++++++++++++++++++++++++- sariflogger.py | 164 ----------------------------------------------- 2 files changed, 169 insertions(+), 166 deletions(-) delete mode 100644 sariflogger.py diff --git a/flawfinder.py b/flawfinder.py index b3d001a..8771dc3 100644 --- a/flawfinder.py +++ b/flawfinder.py @@ -53,8 +53,7 @@ import operator # To support filename expansion on Windows import time import csv # To support generating CSV format import hashlib -# import formatter -from sariflogger import SarifLogger +import json version = "2.0.15" @@ -127,6 +126,173 @@ def print_warning(message): sys.stderr.write("\n") sys.stderr.flush() +def to_json(o): + return json.dumps(o, default=lambda o: o.__dict__, sort_keys=False, indent=2) + + +# The following implements the SarifLogger. +# We intentionally merge all of flawfinder's functionality into 1 file +# so it's trivial to copy & use elsewhere. + +class SarifLogger(object): + _hitlist = None + TOOL_NAME = "Flawfinder" + TOOL_URL = "https://dwheeler.com/flawfinder/" + TOOL_VERSION = "2.0.15" + URI_BASE_ID = "SRCROOT" + SARIF_SCHEMA = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json" + SARIF_SCHEMA_VERSION = "2.1.0" + CWE_TAXONOMY_NAME = "CWE" + CWE_TAXONOMY_URI = "https://raw.githubusercontent.com/sarif-standard/taxonomies/main/CWE_v4.4.sarif" + CWE_TAXONOMY_GUID = "FFC64C90-42B6-44CE-8BEB-F6B7DAE649E5" + + def __init__ (self, hits): + self._hitlist = hits + + def output_sarif(self): + tool = { + "driver": { + "name": self.TOOL_NAME, + "version": self.TOOL_VERSION, + "informationUri": self.TOOL_URL, + "rules": self._extract_rules(self._hitlist), + "supportedTaxonomies": [{ + "name": self.CWE_TAXONOMY_NAME, + "guid": self.CWE_TAXONOMY_GUID, + }], + } + } + + runs = [{ + "tool": tool, + "columnKind": "utf16CodeUnits", + "results": self._extract_results(self._hitlist), + "externalPropertyFileReferences": { + "taxonomies": [{ + "location": { + "uri": self.CWE_TAXONOMY_URI, + }, + "guid": self.CWE_TAXONOMY_GUID, + }], + }, + }] + + report = { + "$schema": self.SARIF_SCHEMA, + "version": self.SARIF_SCHEMA_VERSION, + "runs": runs, + } + + jsonstr = to_json(report) + return jsonstr + + def _extract_rules(self, hitlist): + rules = {} + for hit in hitlist: + if not hit.ruleid in rules: + rules[hit.ruleid] = self._to_sarif_rule(hit) + return list(rules.values()) + + def _extract_results(self, hitlist): + results = [] + for hit in hitlist: + results.append(self._to_sarif_result(hit)) + return results + + def _to_sarif_rule(self, hit): + return { + "id": hit.ruleid, + "name": "{0}/{1}".format(hit.category, hit.name), + "shortDescription": { + "text": self._append_period(hit.warning), + }, + "defaultConfiguration": { + "level": self._to_sarif_level(hit.defaultlevel), + }, + "helpUri": hit.helpuri(), + "relationships": self._extract_relationships(hit.cwes()), + } + + def _to_sarif_result(self, hit): + return { + "ruleId": hit.ruleid, + "level": self._to_sarif_level(hit.level), + "message": { + "text": self._append_period("{0}/{1}:{2}".format(hit.category, hit.name, hit.warning)), + }, + "locations": [{ + "physicalLocation": { + "artifactLocation": { + "uri": self._to_uri_path(hit.filename), + "uriBaseId": self.URI_BASE_ID, + }, + "region": { + "startLine": hit.line, + "startColumn": hit.column, + "endColumn": len(hit.context_text) + 1, + "snippet": { + "text": hit.context_text, + } + } + } + }], + "fingerprints": { + "contextHash/v1": hit.fingerprint() + }, + "rank": self._to_sarif_rank(hit.level), + } + + def _extract_relationships(self, cwestring): + # example cwe string "CWE-119!/ CWE-120", "CWE-829, CWE-20" + relationships = [] + for cwe in re.split(',|/',cwestring): + cwestr = cwe.strip() + if cwestr: + relationship = { + "target": { + "id": int(cwestr.replace("CWE-", "").replace("!", "")), + "toolComponent": { + "name": self.CWE_TAXONOMY_NAME, + "guid": self.CWE_TAXONOMY_GUID, + }, + }, + "kinds": [ + "relevant" if cwestr[-1] != '!' else "incomparable" + ], + } + relationships.append(relationship) + return relationships + + @staticmethod + def _to_sarif_level(level): + # level 4 & 5 + if level >= 4: + return "error" + # level 3 + if level == 3: + return "warning" + # level 0 1 2 + return "note" + + @staticmethod + def _to_sarif_rank(level): + #SARIF rank FF Level SARIF level Default Viewer Action + #0.0 0 note Does not display by default + #0.2 1 note Does not display by default + #0.4 2 note Does not display by default + #0.6 3 warning Displays by default, does not break build / other processes + #0.8 4 error Displays by default, breaks build/ other processes + #1.0 5 error Displays by default, breaks build/ other processes + return level * 0.2 + + @staticmethod + def _to_uri_path(path): + return path.replace("\\", "/") + + @staticmethod + def _append_period(text): + return text if text[-1] == '.' else text + "." + # The following code accepts unified diff format from both subversion (svn) # and GNU diff, which aren't well-documented. It gets filenames from @@ -2018,6 +2184,7 @@ flawfinder [--help | -h] [--version] [--listrules] Display as HTML output. --immediate | -i Immediately display hits (don't just wait until the end). + --sarif Generate output in SARIF format. --singleline | -S Single-line output. --omittime Omit time to run. diff --git a/sariflogger.py b/sariflogger.py deleted file mode 100644 index 6568f31..0000000 --- a/sariflogger.py +++ /dev/null @@ -1,164 +0,0 @@ -import json -import re - -def to_json(o): - return json.dumps(o, default=lambda o: o.__dict__, sort_keys=False, indent=2) - -class SarifLogger(object): - _hitlist = None - TOOL_NAME = "Flawfinder" - TOOL_URL = "https://dwheeler.com/flawfinder/" - TOOL_VERSION = "2.0.15" - URI_BASE_ID = "SRCROOT" - SARIF_SCHEMA = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json" - SARIF_SCHEMA_VERSION = "2.1.0" - CWE_TAXONOMY_NAME = "CWE" - CWE_TAXONOMY_URI = "https://raw.githubusercontent.com/sarif-standard/taxonomies/main/CWE_v4.4.sarif" - CWE_TAXONOMY_GUID = "FFC64C90-42B6-44CE-8BEB-F6B7DAE649E5" - - def __init__ (self, hits): - self._hitlist = hits - - def output_sarif(self): - tool = { - "driver": { - "name": self.TOOL_NAME, - "version": self.TOOL_VERSION, - "informationUri": self.TOOL_URL, - "rules": self._extract_rules(self._hitlist), - "supportedTaxonomies": [{ - "name": self.CWE_TAXONOMY_NAME, - "guid": self.CWE_TAXONOMY_GUID, - }], - } - } - - runs = [{ - "tool": tool, - "columnKind": "utf16CodeUnits", - "results": self._extract_results(self._hitlist), - "externalPropertyFileReferences": { - "taxonomies": [{ - "location": { - "uri": self.CWE_TAXONOMY_URI, - }, - "guid": self.CWE_TAXONOMY_GUID, - }], - }, - }] - - report = { - "$schema": self.SARIF_SCHEMA, - "version": self.SARIF_SCHEMA_VERSION, - "runs": runs, - } - - jsonstr = to_json(report) - return jsonstr - - def _extract_rules(self, hitlist): - rules = {} - for hit in hitlist: - if not hit.ruleid in rules: - rules[hit.ruleid] = self._to_sarif_rule(hit) - return list(rules.values()) - - def _extract_results(self, hitlist): - results = [] - for hit in hitlist: - results.append(self._to_sarif_result(hit)) - return results - - def _to_sarif_rule(self, hit): - return { - "id": hit.ruleid, - "name": "{0}/{1}".format(hit.category, hit.name), - "shortDescription": { - "text": self._append_period(hit.warning), - }, - "defaultConfiguration": { - "level": self._to_sarif_level(hit.defaultlevel), - }, - "helpUri": hit.helpuri(), - "relationships": self._extract_relationships(hit.cwes()), - } - - def _to_sarif_result(self, hit): - return { - "ruleId": hit.ruleid, - "level": self._to_sarif_level(hit.level), - "message": { - "text": self._append_period("{0}/{1}:{2}".format(hit.category, hit.name, hit.warning)), - }, - "locations": [{ - "physicalLocation": { - "artifactLocation": { - "uri": self._to_uri_path(hit.filename), - "uriBaseId": self.URI_BASE_ID, - }, - "region": { - "startLine": hit.line, - "startColumn": hit.column, - "endColumn": len(hit.context_text) + 1, - "snippet": { - "text": hit.context_text, - } - } - } - }], - "fingerprints": { - "contextHash/v1": hit.fingerprint() - }, - "rank": self._to_sarif_rank(hit.level), - } - - def _extract_relationships(self, cwestring): - # example cwe string "CWE-119!/ CWE-120", "CWE-829, CWE-20" - relationships = [] - for cwe in re.split(',|/',cwestring): - cwestr = cwe.strip() - if cwestr: - relationship = { - "target": { - "id": int(cwestr.replace("CWE-", "").replace("!", "")), - "toolComponent": { - "name": self.CWE_TAXONOMY_NAME, - "guid": self.CWE_TAXONOMY_GUID, - }, - }, - "kinds": [ - "relevant" if cwestr[-1] != '!' else "incomparable" - ], - } - relationships.append(relationship) - return relationships - - @staticmethod - def _to_sarif_level(level): - # level 4 & 5 - if level >= 4: - return "error" - # level 3 - if level == 3: - return "warning" - # level 0 1 2 - return "note" - - @staticmethod - def _to_sarif_rank(level): - #SARIF rank FF Level SARIF level Default Viewer Action - #0.0 0 note Does not display by default - #0.2 1 note Does not display by default - #0.4 2 note Does not display by default - #0.6 3 warning Displays by default, does not break build / other processes - #0.8 4 error Displays by default, breaks build/ other processes - #1.0 5 error Displays by default, breaks build/ other processes - return level * 0.2 - - @staticmethod - def _to_uri_path(path): - return path.replace("\\", "/") - - @staticmethod - def _append_period(text): - return text if text[-1] == '.' else text + "." From 772c6f6448629d8d835a9fd5cff6789e7338b3ca Mon Sep 17 00:00:00 2001 From: "David A. Wheeler" Date: Sun, 30 May 2021 14:41:14 -0400 Subject: [PATCH 3/3] flawfinder.py should be executable Signed-off-by: David A. Wheeler --- flawfinder.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 flawfinder.py diff --git a/flawfinder.py b/flawfinder.py old mode 100644 new mode 100755