From c106dd2939a064e7eff0eca54433921768925094 Mon Sep 17 00:00:00 2001
From: Sebastian <versat@users.noreply.github.com>
Date: Fri, 18 Jan 2019 21:30:08 +0100
Subject: [PATCH] misra addon: Fix issue with unexpected encodings in
 loadRuleTexts() (trac 8946) (#1608)

https://trac.cppcheck.net/ticket/8946
Add tests to travis script for verifying rule text loading.
Add dummy rule text files.
misra.py: Try to find a suitable codec for rule texts file.
---
 .travis.yml                                   |  6 +++++
 addons/misra.py                               | 25 ++++++++++++++++++-
 addons/test/misra2012_rules_dummy_ascii.txt   |  5 ++++
 addons/test/misra2012_rules_dummy_utf8.txt    |  5 ++++
 .../misra2012_rules_dummy_windows1250.txt     |  5 ++++
 5 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 addons/test/misra2012_rules_dummy_ascii.txt
 create mode 100644 addons/test/misra2012_rules_dummy_utf8.txt
 create mode 100644 addons/test/misra2012_rules_dummy_windows1250.txt

diff --git a/.travis.yml b/.travis.yml
index 3982dc383..762429297 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -85,6 +85,12 @@ matrix:
         - python3 ../misra.py -verify misra-test.c.dump
         - ${CPPCHECK} --dump misra-test.cpp
         - python3 ../misra.py -verify misra-test.cpp.dump
+        - python ../misra.py --rule-texts=misra2012_rules_dummy_ascii.txt -verify misra-test.cpp.dump
+        - python3 ../misra.py --rule-texts=misra2012_rules_dummy_ascii.txt -verify misra-test.cpp.dump
+        - python ../misra.py --rule-texts=misra2012_rules_dummy_utf8.txt -verify misra-test.cpp.dump
+        - python3 ../misra.py --rule-texts=misra2012_rules_dummy_utf8.txt -verify misra-test.cpp.dump
+        - python ../misra.py --rule-texts=misra2012_rules_dummy_windows1250.txt -verify misra-test.cpp.dump
+        - python3 ../misra.py --rule-texts=misra2012_rules_dummy_windows1250.txt -verify misra-test.cpp.dump
         - cd ../../
 # check addons/namingng.py
         - cd addons/test
diff --git a/addons/misra.py b/addons/misra.py
index d57f690a9..57da6d5b3 100755
--- a/addons/misra.py
+++ b/addons/misra.py
@@ -20,6 +20,7 @@ import sys
 import re
 import os
 import argparse
+import codecs
 
 
 typeBits = {
@@ -1942,7 +1943,29 @@ class MisraChecker:
         Choice_pattern = re.compile(r'^[ ]*(Advisory|Required|Mandatory)$')
         xA_Z_pattern = re.compile(r'^[#A-Z].*')
         a_z_pattern = re.compile(r'^[a-z].*')
-        for line in open(filename, 'rt'):
+        # Try to detect the file encoding
+        file_stream = None
+        encodings = ['ascii', 'utf-8', 'windows-1250', 'windows-1252']
+        for e in encodings:
+            try:
+                file_stream = codecs.open(filename, 'r', encoding=e)
+                file_stream.readlines()
+                file_stream.seek(0)
+            except UnicodeDecodeError:
+                file_stream = None
+            else:
+                break
+        if not file_stream:
+            print('Could not find a suitable codec for "' + filename + '".')
+            print('If you know the codec please report it to the developers so the list can be enhanced.')
+            print('Trying with default codec now and ignoring errors if possible ...')
+            try:
+                file_stream = open(filename, 'rt', errors='ignore')
+            except TypeError:
+                # Python 2 does not support the errors parameter
+                file_stream = open(filename, 'rt')
+        # Parse the rule texts
+        for line in file_stream:
             line = line.replace('\r', '').replace('\n', '')
             if len(line) == 0:
                 if ruleText:
diff --git a/addons/test/misra2012_rules_dummy_ascii.txt b/addons/test/misra2012_rules_dummy_ascii.txt
new file mode 100644
index 000000000..8880ca560
--- /dev/null
+++ b/addons/test/misra2012_rules_dummy_ascii.txt
@@ -0,0 +1,5 @@
+Appendix A Summary of guidelines
+Rule 1.1
+Text of rule 1.1
+Rule 1.2
+Text of rule 1.2
diff --git a/addons/test/misra2012_rules_dummy_utf8.txt b/addons/test/misra2012_rules_dummy_utf8.txt
new file mode 100644
index 000000000..5f96010fa
--- /dev/null
+++ b/addons/test/misra2012_rules_dummy_utf8.txt
@@ -0,0 +1,5 @@
+Appendix A Summary of guidelines
+Rule 1.1
+Text of rule 1.1, utf8 test: ∑
+Rule 1.2
+Text of rule 1.2
diff --git a/addons/test/misra2012_rules_dummy_windows1250.txt b/addons/test/misra2012_rules_dummy_windows1250.txt
new file mode 100644
index 000000000..89480cc60
--- /dev/null
+++ b/addons/test/misra2012_rules_dummy_windows1250.txt
@@ -0,0 +1,5 @@
+Appendix A Summary of guidelines
+Rule 1.1
+Text of rule 1.1, windows1250 test: ’
+Rule 1.2
+Text of rule 1.2