From b3e8ef9d483ed98ade583cfb09b5b31193d3056e Mon Sep 17 00:00:00 2001 From: Erik Lax Date: Fri, 11 Feb 2011 06:30:42 +0100 Subject: [PATCH] Fixed #2559 (Refactoring Preprocessor::read) --- lib/preprocessor.cpp | 150 ++++++++++++++++++++++++++++---------- lib/preprocessor.h | 7 +- test/testpreprocessor.cpp | 76 ++++++++++++++++++- 3 files changed, 192 insertions(+), 41 deletions(-) diff --git a/lib/preprocessor.cpp b/lib/preprocessor.cpp index f3ab9e231..36d0d4177 100644 --- a/lib/preprocessor.cpp +++ b/lib/preprocessor.cpp @@ -109,39 +109,19 @@ static std::string join(const std::list &list, char separator) /** Just read the code into a string. Perform simple cleanup of the code */ std::string Preprocessor::read(std::istream &istr, const std::string &filename, Settings *settings) { - // Get filedata from stream.. - bool ignoreSpace = true; - - // need space.. #if( => #if ( - bool needSpace = false; - + // ------------------------------------------------------------------------------------------ + // // handling // when this is encountered the will be "skipped". // on the next , extra newlines will be added - unsigned int newlines = 0; - std::ostringstream code; + unsigned int newlines = 0; for (unsigned char ch = readChar(istr); istr.good(); ch = readChar(istr)) { // Replace assorted special chars with spaces.. if (((ch & 0x80) == 0) && (ch != '\n') && (std::isspace(ch) || std::iscntrl(ch))) ch = ' '; - // Skip spaces after ' ' and after '#' - if (ch == ' ' && ignoreSpace) - continue; - ignoreSpace = bool(ch == ' ' || ch == '#' || ch == '\n'); - - if (needSpace) - { - if (ch == '(' || ch == '!') - code << " "; - else if (!std::isalpha(ch)) - needSpace = false; - } - if (ch == '#') - needSpace = true; - // .. // for gcc-compatibility the trailing spaces should be ignored // for vs-compatibility the trailing spaces should be kept @@ -178,8 +158,6 @@ std::string Preprocessor::read(std::istream &istr, const std::string &filename, else code << "\\"; } - - // Just some code.. else { code << char(ch); @@ -192,8 +170,117 @@ std::string Preprocessor::read(std::istream &istr, const std::string &filename, } } } + std::string result = code.str(); + code.str(""); - return removeParantheses(removeComments(code.str(), filename, settings)); + // ------------------------------------------------------------------------------------------ + // + // Remove all comments.. + result = removeComments(result, filename, settings); + + // ------------------------------------------------------------------------------------------ + // + // Clean up all preprocessor statements + result = preprocessCleanupDirectives(result); + + // ------------------------------------------------------------------------------------------ + // + // Clean up preprocessor #if statements with Parantheses + result = removeParantheses(result); + + return result; +} + +std::string Preprocessor::preprocessCleanupDirectives(const std::string &processedFile) const +{ + std::ostringstream code; + std::istringstream sstr(processedFile); + + std::string line; + while (std::getline(sstr, line)) + { + // Trim lines.. + if (!line.empty() && line[0] == ' ') + line.erase(0, line.find_first_not_of(" ")); + if (!line.empty() && line[line.size()-1] == ' ') + line.erase(line.find_last_not_of(" ") + 1); + + // Preprocessor + if (!line.empty() && line[0] == '#') + { + enum { + ESC_NONE, + ESC_SINGLE, + ESC_DOUBLE + } escapeStatus = ESC_NONE; + + char prev = ' '; // hack to make it skip spaces between # and the directive + code << "#"; + std::string::const_iterator i = line.begin(); + i++; + + // need space.. #if( => #if ( + bool needSpace = true; + while(i != line.end()) + { + // disable esc-mode + if (escapeStatus != ESC_NONE) + { + if (prev != '\\' && escapeStatus == ESC_SINGLE && *i == '\'') + { + escapeStatus = ESC_NONE; + } + if (prev != '\\' && escapeStatus == ESC_DOUBLE && *i == '"') + { + escapeStatus = ESC_NONE; + } + } else { + // enable esc-mode + if (escapeStatus == ESC_NONE && *i == '"') + escapeStatus = ESC_DOUBLE; + if (escapeStatus == ESC_NONE && *i == '\'') + escapeStatus = ESC_SINGLE; + } + // skip double whitespace between arguments + if (escapeStatus == ESC_NONE && prev == ' ' && *i == ' ') + { + i++; + continue; + } + // Convert #if( to "#if (" + if (escapeStatus == ESC_NONE) + { + if (needSpace) + { + if (*i == '(' || *i == '!') + code << " "; + else if (!std::isalpha(*i)) + needSpace = false; + } + if (*i == '#') + needSpace = true; + } + code << *i; + if (escapeStatus != ESC_NONE && prev == '\\' && *i == '\\') + { + prev = ' '; + } else { + prev = *i; + } + i++; + } + if (escapeStatus != ESC_NONE) + { + // unmatched quotes.. compiler should probably complain about this.. + } + } else { + // Do not mess with regular code.. + code << line; + } + code << (sstr.eof()?"":"\n"); + } + + return code.str(); } static bool hasbom(const std::string &str) @@ -668,9 +755,6 @@ void Preprocessor::preprocess(std::istream &srcCodeStream, std::string &processe processedFile = read(srcCodeStream, filename, _settings); - // normalize the whitespaces of the file - preprocessWhitespaces(processedFile); - // Remove asm(...) removeAsm(processedFile); @@ -1635,15 +1719,7 @@ void Preprocessor::handleIncludes(std::string &code, const std::string &filePath if (!processedFile.empty()) { - // Replace all tabs with spaces.. - std::replace(processedFile.begin(), processedFile.end(), '\t', ' '); - - // Remove all indentation.. - if (!processedFile.empty() && processedFile[0] == ' ') - processedFile.erase(0, processedFile.find_first_not_of(" ")); - // Remove space characters that are after or before new line character - processedFile = removeSpaceNearNL(processedFile); processedFile = "#file \"" + filename + "\"\n" + processedFile + "\n#endfile"; code.insert(pos, processedFile); diff --git a/lib/preprocessor.h b/lib/preprocessor.h index 2d9bc14cb..a6829efea 100644 --- a/lib/preprocessor.h +++ b/lib/preprocessor.h @@ -106,7 +106,6 @@ public: * @param processedFile The data to be processed */ static void preprocessWhitespaces(std::string &processedFile); - protected: /** @@ -152,6 +151,12 @@ protected: */ static std::string removeParantheses(const std::string &str); + /** + * clean up #-preprocessor lines (only) + * @param processedFile The data to be processed + */ + std::string preprocessCleanupDirectives(const std::string &processedFile) const; + /** * Returns the string between double quote characters or \< \> characters. * @param str e.g. \code#include "menu.h"\endcode or \code#include \endcode diff --git a/test/testpreprocessor.cpp b/test/testpreprocessor.cpp index 7163b3af2..fbd012632 100644 --- a/test/testpreprocessor.cpp +++ b/test/testpreprocessor.cpp @@ -148,6 +148,7 @@ private: TEST_CASE(macro_simple11); TEST_CASE(macro_simple12); TEST_CASE(macro_simple13); + TEST_CASE(macro_simple14); TEST_CASE(macroInMacro); TEST_CASE(macro_mismatch); TEST_CASE(macro_linenumbers); @@ -198,6 +199,11 @@ private: TEST_CASE(endfile); TEST_CASE(redundant_config); + + TEST_CASE(testPreprocessorRead1); + TEST_CASE(testPreprocessorRead2); + TEST_CASE(testPreprocessorRead3); + TEST_CASE(testPreprocessorRead4); } @@ -209,7 +215,7 @@ private: Preprocessor preprocessor(&settings, this); std::istringstream istr(code); std::string codestr(preprocessor.read(istr,"test.c",0)); - ASSERT_EQUALS("a \n#aa b \n", codestr); + ASSERT_EQUALS("a\n#aa b\n", codestr); } void readCode2() @@ -323,7 +329,7 @@ private: preprocessor.preprocess(istr, actual, "file.c"); // Compare results.. - ASSERT_EQUALS("\n\" #ifdef WIN32\"\n\n\n\n", actual[""]); + ASSERT_EQUALS("\n\" # ifdef WIN32\"\n\n\n\n", actual[""]); ASSERT_EQUALS("\n\n\nqwerty\n\n", actual["WIN32"]); ASSERT_EQUALS(2, static_cast(actual.size())); } @@ -1363,7 +1369,7 @@ private: std::istringstream istr(filedata); Settings settings; Preprocessor preprocessor(&settings, this); - ASSERT_EQUALS("#define str \"abc\" \"def\" \n\nabcdef = str;\n", preprocessor.read(istr, "test.c", 0)); + ASSERT_EQUALS("#define str \"abc\" \"def\"\n\nabcdef = str;\n", preprocessor.read(istr, "test.c", 0)); } void multiline2() @@ -1576,6 +1582,13 @@ private: ASSERT_EQUALS("\n\n", OurPreprocessor::expandMacros(filedata)); } + void macro_simple14() + { + const char filedata[] = "#define A \" a \"\n" + "printf(A);\n"; + ASSERT_EQUALS("\nprintf(\" a \");\n", OurPreprocessor::expandMacros(filedata)); + } + void macroInMacro() { { @@ -2591,6 +2604,63 @@ private: ASSERT_EQUALS("A;A;B is NOT checked", "A;A;B is NOT checked"); } } + + void testPreprocessorRead1() + { + const std::string filedata("/*\n*/ # /*\n*/ defi\\\nne FO\\\nO 10\\\n20"); + std::istringstream istr(filedata.c_str()); + Settings settings; + Preprocessor preprocessor(&settings, this); + ASSERT_EQUALS("#define FOO 1020", preprocessor.read(istr, "test.cpp", 0)); + } + + void testPreprocessorRead2() + { + const std::string filedata("\"foo\\\\\nbar\""); + std::istringstream istr(filedata.c_str()); + Settings settings; + Preprocessor preprocessor(&settings, this); + ASSERT_EQUALS("\"foo\\bar\"", preprocessor.read(istr, "test.cpp", 0)); + } + + void testPreprocessorRead3() + { + const std::string filedata("#define A \" a \"\n\" b\""); + std::istringstream istr(filedata.c_str()); + Settings settings; + Preprocessor preprocessor(&settings, this); + ASSERT_EQUALS(filedata, preprocessor.read(istr, "test.cpp", 0)); + } + + void testPreprocessorRead4() + { + { + // test < \\> < > (unescaped) + const std::string filedata("#define A \" \\\\\"/*space*/ \" \""); + std::istringstream istr(filedata.c_str()); + Settings settings; + Preprocessor preprocessor(&settings, this); + ASSERT_EQUALS("#define A \" \\\\\" \" \"", preprocessor.read(istr, "test.cpp", 0)); + } + + { + // test <" \\\" "> (unescaped) + const std::string filedata("#define A \" \\\\\\\" \""); + std::istringstream istr(filedata.c_str()); + Settings settings; + Preprocessor preprocessor(&settings, this); + ASSERT_EQUALS("#define A \" \\\\\\\" \"", preprocessor.read(istr, "test.cpp", 0)); + } + + { + // test <" \\\\"> <" "> (unescaped) + const std::string filedata("#define A \" \\\\\\\\\"/*space*/ \" \""); + std::istringstream istr(filedata.c_str()); + Settings settings; + Preprocessor preprocessor(&settings, this); + ASSERT_EQUALS("#define A \" \\\\\\\\\" \" \"", preprocessor.read(istr, "test.cpp", 0)); + } + } }; REGISTER_TEST(TestPreprocessor)