misra addon: Fix issue with unexpected encodings in loadRuleTexts() (trac 8946) (#1608)

https://trac.cppcheck.net/ticket/8946
Add tests to travis script for verifying rule text loading.
Add dummy rule text files.
misra.py: Try to find a suitable codec for rule texts file.

.travis.yml

@@ -85,6 +85,12 @@ matrix:
         - python3 ../misra.py -verify misra-test.c.dump
         - ${CPPCHECK} --dump misra-test.cpp
         - python3 ../misra.py -verify misra-test.cpp.dump
+        - python ../misra.py --rule-texts=misra2012_rules_dummy_ascii.txt -verify misra-test.cpp.dump
+        - python3 ../misra.py --rule-texts=misra2012_rules_dummy_ascii.txt -verify misra-test.cpp.dump
+        - python ../misra.py --rule-texts=misra2012_rules_dummy_utf8.txt -verify misra-test.cpp.dump
+        - python3 ../misra.py --rule-texts=misra2012_rules_dummy_utf8.txt -verify misra-test.cpp.dump
+        - python ../misra.py --rule-texts=misra2012_rules_dummy_windows1250.txt -verify misra-test.cpp.dump
+        - python3 ../misra.py --rule-texts=misra2012_rules_dummy_windows1250.txt -verify misra-test.cpp.dump
         - cd ../../
 # check addons/namingng.py
         - cd addons/test

addons/misra.py

@@ -20,6 +20,7 @@ import sys
 import re
 import os
 import argparse
+import codecs
 
 
 typeBits = {
@@ -1942,7 +1943,29 @@ class MisraChecker:
         Choice_pattern = re.compile(r'^[ ]*(Advisory|Required|Mandatory)$')
         xA_Z_pattern = re.compile(r'^[#A-Z].*')
         a_z_pattern = re.compile(r'^[a-z].*')
-        for line in open(filename, 'rt'):
+        # Try to detect the file encoding
+        file_stream = None
+        encodings = ['ascii', 'utf-8', 'windows-1250', 'windows-1252']
+        for e in encodings:
+            try:
+                file_stream = codecs.open(filename, 'r', encoding=e)
+                file_stream.readlines()
+                file_stream.seek(0)
+            except UnicodeDecodeError:
+                file_stream = None
+            else:
+                break
+        if not file_stream:
+            print('Could not find a suitable codec for "' + filename + '".')
+            print('If you know the codec please report it to the developers so the list can be enhanced.')
+            print('Trying with default codec now and ignoring errors if possible ...')
+            try:
+                file_stream = open(filename, 'rt', errors='ignore')
+            except TypeError:
+                # Python 2 does not support the errors parameter
+                file_stream = open(filename, 'rt')
+        # Parse the rule texts
+        for line in file_stream:
             line = line.replace('\r', '').replace('\n', '')
             if len(line) == 0:
                 if ruleText:

addons/test/misra2012_rules_dummy_ascii.txt

@@ -0,0 +1,5 @@
+Appendix A Summary of guidelines
+Rule 1.1
+Text of rule 1.1
+Rule 1.2
+Text of rule 1.2

addons/test/misra2012_rules_dummy_utf8.txt

@@ -0,0 +1,5 @@
+Appendix A Summary of guidelines
+Rule 1.1
+Text of rule 1.1, utf8 test: ∑
+Rule 1.2
+Text of rule 1.2

addons/test/misra2012_rules_dummy_windows1250.txt

@@ -0,0 +1,5 @@
+Appendix A Summary of guidelines
+Rule 1.1
+Text of rule 1.1, windows1250 test: ’
+Rule 1.2
+Text of rule 1.2