Fix ticket #417 (Crashes in Windows because of invalid char value)

8-bit characters and UTF-8 characters are now converted to spaces. They can only appear in the code files in comments or string/char constants. So we cannot just ignore them as that would change length.

src/preprocessor.cpp

@@ -261,6 +261,15 @@ std::string Preprocessor::read(std::istream &istr)
         if (ch == '\n')
             ++lineno;
 
+        // UTF / extended ASCII => The output from the preprocessor should
+        // only be standard ASCII.
+        // In C/C++ code the UTF and extented ASCII (8-bit ASCII) can only
+        // appear in comments, strings and char constants. So the safest thing
+        // to do is replace it with space char. We CAN'T change string lengths
+        // by ignoring characters.
+        if (ch < 0)
+            ch = ' ';
+
         // Replace assorted special chars with spaces..
         if ((ch != '\n') && (std::isspace(ch) || std::iscntrl(ch)))
             ch = ' ';

test/testpreprocessor.cpp

@@ -994,7 +994,7 @@ private:
     {
         const char filedata[] = {'a', (char)200, 0};
         std::istringstream istr(filedata);
-        ASSERT_THROW(Preprocessor::read(istr), std::runtime_error);
+        ASSERT_EQUALS(Preprocessor::read(istr), "a ");
     }
 
     void define_part_of_func()