Loosen restrictions on MISRA rules text structure (#2157)

Version 1.88 changed the parsing of the MISRA rules file adding a
severity setting.  This caused a regression in rule parsing.

In particular the following format used to parse cleanly and produce
rule output that would show the severity as part of the rule text.

    Rule 1.2

    Advisory

    Rule text goes here.

    Rule 1.3

    Required

    More rule text goes here.

As of 1.88 a file structured like above would parse as having no rules.

The problem is the use of blank lines as a rule delimiter.   The
modified rule parser wants to see a rules formatted like below:

    Rule 3.1 Required
    R3.1 text.
    Rule 4.1 Required
    R4.1 text.

or:

    Rule 1.1
    Add this rule and parse to next, skipping empty lines.

    Rule 1.2
    Rule text.

Any rule text that did not fall into one of the above formats would
result in incomplete rule text parsing.

Change the parsing of the rule text file so that blank lines are ignored
instead of treating them as a delimiter between rules. Instead use the
start of the next rule as a delimiter for the end of the previous rule.
This allows both of the newer formats but also supports the behavior of
pre-1.88 versions.

Change units tests that were specifically forbidding the use of blank
lines to ones that allow blank lines.

addons/misra.py

@@ -2205,7 +2205,7 @@ class MisraChecker:
         expect_more = False
 
         Rule_pattern = re.compile(r'^Rule ([0-9]+).([0-9]+)')
-        Choice_pattern = re.compile(r'.*[ ]*(Advisory|Required|Mandatory)$')
+        severity_pattern = re.compile(r'.*[ ]*(Advisory|Required|Mandatory)$')
         xA_Z_pattern = re.compile(r'^[#A-Z].*')
         a_z_pattern = re.compile(r'^[a-z].*')
         # Try to detect the file encoding
@@ -2231,8 +2231,13 @@ class MisraChecker:
                 file_stream = open(filename, 'rt')
 
         rule = None
+        have_severity = False
+        severity_loc  = 0
+
         for line in file_stream:
+
             line = line.replace('\r', '').replace('\n', '')
+
             if not appendixA:
                 if line.find('Appendix A') >= 0 and line.find('Summary of guidelines') >= 10:
                     appendixA = True
@@ -2240,21 +2245,39 @@ class MisraChecker:
             if line.find('Appendix B') >= 0:
                 break
             if len(line) == 0:
-                expect_more = False
-                rule = None
                 continue
 
             # Parse rule declaration.
             res = Rule_pattern.match(line)
+
             if res:
+                have_severity = False
+                expect_more = False
+                severity_loc = 0
                 num1 = int(res.group(1))
                 num2 = int(res.group(2))
                 rule = Rule(num1, num2)
-                res = Choice_pattern.match(line)
+
+            if not have_severity and rule is not None:
+                res = severity_pattern.match(line)
+
                 if res:
                     rule.misra_severity = res.group(1)
-                expect_more = False
-                continue
+                    have_severity = True
+                else:
+                    severity_loc += 1;
+
+                # Only look for severity on the Rule line
+                # or the next non-blank line after
+                # If it's not in either of those locations then
+                # assume a severity was not provided.
+
+                if severity_loc < 2:
+                    continue
+                else:
+                    rule.misra_severity = ''
+                    have_severity = True
+
             if rule is None:
                 continue
 
@@ -2263,7 +2286,7 @@ class MisraChecker:
                 if a_z_pattern.match(line):
                     self.ruleTexts[rule.num].text += ' ' + line
                     continue
-                rule = None
+
                 expect_more = False
                 continue
 
@@ -2272,8 +2295,6 @@ class MisraChecker:
                 rule.text = line
                 self.ruleTexts[rule.num] = rule
                 expect_more = True
-            else:
-                rule = None
 
     def verifyRuleTexts(self):
         """Prints rule numbers without rule text."""

addons/test/misra/misra_rules_multiple_lines.txt

@@ -18,7 +18,7 @@ Rule 1.5
 Should
  starts from lowercase letter.
 Rule 1.6
-Should
+Can
 
-not contain empty lines.
+contain empty lines.
 

addons/test/test-misra.py

@@ -49,7 +49,7 @@ def test_loadRuleTexts_mutiple_lines(checker):
     assert(checker.ruleTexts[103].text == "Multiple lines text.")
     assert(checker.ruleTexts[104].text == "Should")
     assert(checker.ruleTexts[105].text == "Should")
-    assert(checker.ruleTexts[106].text == "Should")
+    assert(checker.ruleTexts[106].text == "Can contain empty lines.")
 
 
 def test_verifyRuleTexts(checker, capsys):