Changed handling of unhandled characters:

- Don't abort checking (reverts 42140b6488)
- Modified error message: New Id unhandledCharacter, removed redundant line information, improved message text

lib/cppcheck.cpp

@@ -160,10 +160,6 @@ unsigned int CppCheck::processFile(const std::string& filename, const std::strin
             preprocessor.preprocess(fin, filedata, configurations, filename, _settings._includePaths);
         }
 
-        // Unhandled chars found during preprocessing => abort checking this file
-        if (preprocessor.foundUnhandledChars())
-            return 0;
-
         if (_settings.checkConfiguration) {
             return 0;
         }

lib/preprocessor.cpp

@@ -38,7 +38,7 @@ bool Preprocessor::missingSystemIncludeFlag;
 
 char Preprocessor::macroChar = char(1);
 
-Preprocessor::Preprocessor(Settings *settings, ErrorLogger *errorLogger) : _settings(settings), _errorLogger(errorLogger), _foundUnhandledChars(false)
+Preprocessor::Preprocessor(Settings *settings, ErrorLogger *errorLogger) : _settings(settings), _errorLogger(errorLogger)
 {
 
 }
@@ -482,11 +482,12 @@ std::string Preprocessor::removeComments(const std::string &str, const std::stri
         unsigned char ch = static_cast<unsigned char>(str[i]);
         if (ch & 0x80) {
             std::ostringstream errmsg;
-            errmsg << "The code contains characters that are unhandled. "
-                   << "Neither unicode nor extended ASCII are supported. "
-                   << "(line=" << lineno << ", character code=" << std::hex << (int(ch) & 0xff) << ")";
-            writeError(filename, lineno, _errorLogger, "syntaxError", errmsg.str());
-            _foundUnhandledChars = true;
+            errmsg << "(character code = 0x" << std::hex << (int(ch) & 0xff) << ")";
+            std::string info = errmsg.str();
+            errmsg.str("");
+            errmsg << "The code contains unhandled characters " << info << ". Checking continues, but do not expect valid results.\n"
+                   << "The code contains characters that are unhandled " << info << ". Neither unicode nor extended ASCII are supported. Checking continues, but do not expect valid results.";
+            writeError(filename, lineno, _errorLogger, "unhandledCharacters", errmsg.str());
         }
 
         if (_settings && _settings->terminated())

lib/preprocessor.h

@@ -251,10 +251,6 @@ public:
         file0 = f;
     }
 
-    bool foundUnhandledChars() const {
-        return _foundUnhandledChars;
-    }
-
 private:
     void missingInclude(const std::string &filename, unsigned int linenr, const std::string &header, HeaderTypes headerType);
 
@@ -278,11 +274,6 @@ private:
 
     /** filename for cpp/c file - useful when reporting errors */
     std::string file0;
-
-    /** set to true if unhandled chars are found in code. any char is ok
-     *  in comments and string literals, but variable/type names must
-     *  have plain ascii characters. */
-    bool _foundUnhandledChars;
 };
 
 /// @}

lib/tokenlist.cpp

@@ -296,9 +296,6 @@ bool TokenList::createTokens(std::istream &code, const std::string& file0)
             continue;
         }
 
-        // Preprocessor should ensure code doesn't contain any extended ascii / utf / etc.
-        assert(CurrentToken.empty() || (CurrentToken[0] & 0x80) == 0);
-
         if (ch == '.' &&
             !CurrentToken.empty() &&
             std::isdigit(CurrentToken[0])) {

test/testpreprocessor.cpp

@@ -2647,7 +2647,7 @@ private:
         Settings settings;
         Preprocessor preprocessor(&settings, this);
         preprocessor.read(istr, "test.cpp");
-        ASSERT_EQUALS("[test.cpp:1]: (error) The code contains characters that are unhandled. Neither unicode nor extended ASCII are supported. (line=1, character code=c8)\n", errout.str());
+        ASSERT_EQUALS("[test.cpp:1]: (error) The code contains unhandled characters (character code = 0xc8). Checking continues, but do not expect valid results.\n", errout.str());
     }
 
     void unicodeInComment() {