misra.py: Fix Rule 4.1 (#2216)

* misra.py: Use standard string module

* misra.py: Fixup R4.1

References:
* https://trac.cppcheck.net/ticket/9370
* https://sourceforge.net/p/cppcheck/discussion/development/thread/7274ed3842/?limit=25#799a

* Add more examples from @matzeschmid PR

* Add more out-of-bounds tests

* Fix R4.1.

* Compress hex condition

* Add more test cases

* Fix python3 zip import

addons/misra.py

@@ -21,6 +21,17 @@ import re
 import os
 import argparse
 import codecs
+import string
+
+try:
+    from itertools import izip as zip
+except ImportError:
+    pass
+
+
+def grouped(iterable, n):
+    "s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ..."
+    return zip(*[iter(iterable)]*n)
 
 
 typeBits = {
@@ -456,15 +467,52 @@ def getArguments(ftok):
 
 
 def isalnum(c):
-    return (c >= '0' and c <= '9') or (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z')
+    return c in string.digits or c in string.ascii_letters
 
 
-def isHexDigit(c):
-    return (c >= '0' and c <= '9') or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')
+def isHexEscapeSequence(symbols):
+    """Checks that given symbols are valid hex escape sequence.
+
+    hexademical-escape-sequence:
+            \\x hexademical-digit
+            hexademical-escape-sequence hexademical-digit
+
+    Reference: n1570 6.4.4.4"""
+    if len(symbols) < 3 or symbols[:2] != '\\x':
+        return False
+    return all([s in string.hexdigits for s in symbols[2:]])
 
 
-def isOctalDigit(c):
-    return c >= '0' and c <= '7'
+def isOctalEscapeSequence(symbols):
+    """Checks that given symbols are valid octal escape sequence:
+
+     octal-escape-sequence:
+             \ octal-digit
+             \ octal-digit octal-digit
+             \ octal-digit octal-digit octal-digit
+
+    Reference: n1570 6.4.4.4"""
+    if len(symbols) not in range(2, 5) or symbols[0] != '\\':
+        return False
+    return all([s in string.octdigits for s in symbols[1:]])
+
+
+def isSimpleEscapeSequence(symbols):
+    """Checks that given symbols are simple escape sequence.
+    Reference: n1570 6.4.4.4"""
+    if len(symbols) != 2 or symbols[0] != '\\':
+        return False
+    return symbols[1] in ("'", '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v')
+
+
+def hasNumericEscapeSequence(symbols):
+    """Check that given string contains octal or hexademical escape sequences."""
+    if '\\' not in symbols:
+        return False
+    for c, cn in grouped(symbols, 2):
+        if c == '\\' and cn in ('x' + string.octdigits):
+            return True
+    return False
 
 
 def isNoReturnScope(tok):
@@ -700,38 +748,34 @@ class MisraChecker:
 
     def misra_4_1(self, rawTokens):
         for token in rawTokens:
-            if ((token.str[0] != '"') and (token.str[0] != '\'')):
+            if (token.str[0] != '"') and (token.str[0] != '\''):
+                continue
+            if len(token.str) < 3:
                 continue
-            pos = 1
-            while pos < len(token.str) - 2:
-                pos1 = pos
-                pos = pos + 1
-                if token.str[pos1] != '\\':
-                    continue
-                if token.str[pos1 + 1] == '\\':
-                    pos = pos1 + 2
-                    continue
-                if token.str[pos1 + 1] == 'x':
-                    if not isHexDigit(token.str[pos1 + 2]):
-                        self.reportError(token, 4, 1)
-                        continue
-                    if not isHexDigit(token.str[pos1 + 3]):
-                        self.reportError(token, 4, 1)
-                        continue
-                elif isOctalDigit(token.str[pos1 + 1]):
-                    if not isOctalDigit(token.str[pos1 + 2]):
-                        self.reportError(token, 4, 1)
-                        continue
-                    if not isOctalDigit(token.str[pos1 + 2]):
-                        self.reportError(token, 4, 1)
-                        continue
-                else:
-                    continue
 
-                c = token.str[pos1 + 4]
-                if c != '"' and c != '\\':
+            delimiter = token.str[0]
+            symbols = token.str[1:-1]
+
+            # No closing delimiter. This will not compile.
+            if token.str[-1] != delimiter:
+                continue
+
+            if len(symbols) < 2:
+                continue
+
+            if not hasNumericEscapeSequence(symbols):
+                continue
+
+            # String literals that contains one or more escape sequences. All of them should be
+            # terminated.
+            for sequence in ['\\' + t for t in symbols.split('\\')][1:]:
+                if (isHexEscapeSequence(sequence) or isOctalEscapeSequence(sequence) or
+                    isSimpleEscapeSequence(sequence)):
+                    continue
+                else:
                     self.reportError(token, 4, 1)
 
+
     def misra_5_1(self, data):
         long_vars = {}
         for var in data.variables:

addons/test/misra/misra-test.c

@@ -68,9 +68,35 @@ int misra_5_2_field_hides_field1_31y;//5.2
 };
 const char *s41_1 = "\x41g"; // 4.1
 const char *s41_2 = "\x41\x42";
-const char *s41_3 = "\x41" "g";
+const char *s41_3 = "\x41" "\x42";
+const char *s41_4 = "\x41" "g";
+const char *s41_5 = "\x41\xA";
+const char *s41_6 = "\xA\x41";
+const char *s41_7 = "\xAA\xg\x41"; // 4.1
+const char *s41_8 = "\xAA\x\x41"; // 4.1
+const char *s41_9 = "unknown\gsequence";
+const char *s41_10 = "simple\nsequence";
+const char *s41_11 = "string";
 int c41_3         = '\141t'; // 4.1
 int c41_4         = '\141\t';
+int c41_5         = '\0';
+int c41_6         = '\0\t';
+int c41_7         = '\12\t';
+int c41_8         = '\0t';   // 4.1
+int c41_9         = '\12';
+int c41_10        = '\12\n';
+int c41_11        = '\12n';  // 4.1
+int c41_12         = '\12323'; // 4.1
+int c41_13         = '\123\3';
+int c41_14         = '\777\777';
+int c41_15         = 'a';
+
+void misra_4_1()
+{
+    (void)printf("\x41g"); // 4.1
+    (void)printf("\x41\x42");
+    (void)printf("\x41" "g");
+}
 
 extern int misra_5_3_var_hides_var______31x;
 void misra_5_3_var_hides_function_31x (void) {}