Tokenizer: Add special tokenize method for the Preprocessor with only basic simplifications
This commit is contained in:
parent
0a88a136ca
commit
4391f0880f
|
@ -1335,8 +1335,7 @@ std::list<std::string> Preprocessor::getcfgs(const std::string &filedata, const
|
|||
|
||||
if (s.find("&&") != std::string::npos) {
|
||||
Tokenizer tokenizer(_settings, _errorLogger);
|
||||
std::istringstream tempIstr(s);
|
||||
if (!tokenizer.tokenize(tempIstr, filename.c_str(), "", true)) {
|
||||
if (!tokenizer.tokenizeCondition(s)) {
|
||||
std::ostringstream lineStream;
|
||||
lineStream << __LINE__;
|
||||
|
||||
|
@ -1444,8 +1443,7 @@ void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &c
|
|||
{
|
||||
const Settings settings;
|
||||
Tokenizer tokenizer(&settings, _errorLogger);
|
||||
std::istringstream istr("(" + condition + ")");
|
||||
if (!tokenizer.tokenize(istr, "", "", true)) {
|
||||
if (!tokenizer.tokenizeCondition("(" + condition + ")")) {
|
||||
// If tokenize returns false, then there is syntax error in the
|
||||
// code which we can't handle. So stop here.
|
||||
return;
|
||||
|
@ -1505,8 +1503,7 @@ void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &c
|
|||
if (!it->second.empty()) {
|
||||
// Tokenize the value
|
||||
Tokenizer tokenizer2(&settings,NULL);
|
||||
std::istringstream istr2(it->second);
|
||||
tokenizer2.tokenize(istr2,"","",true);
|
||||
tokenizer2.tokenizeCondition(it->second);
|
||||
|
||||
// Copy the value tokens
|
||||
std::stack<Token *> link;
|
||||
|
|
156
lib/tokenize.cpp
156
lib/tokenize.cpp
|
@ -1554,8 +1554,7 @@ void Tokenizer::simplifyMulAndParens()
|
|||
|
||||
bool Tokenizer::tokenize(std::istream &code,
|
||||
const char FileName[],
|
||||
const std::string &configuration,
|
||||
const bool preprocessorCondition)
|
||||
const std::string &configuration)
|
||||
{
|
||||
// make sure settings specified
|
||||
assert(_settings);
|
||||
|
@ -1743,12 +1742,10 @@ bool Tokenizer::tokenize(std::istream &code,
|
|||
// ";a+=b;" => ";a=a+b;"
|
||||
simplifyCompoundAssignment();
|
||||
|
||||
if (!preprocessorCondition) {
|
||||
if (hasComplicatedSyntaxErrorsInTemplates()) {
|
||||
list.deallocateTokens();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
simplifyDefaultAndDeleteInsideClass();
|
||||
|
||||
|
@ -2002,7 +1999,6 @@ bool Tokenizer::tokenize(std::istream &code,
|
|||
// Split up variable declarations.
|
||||
simplifyVarDecl(false);
|
||||
|
||||
if (!preprocessorCondition) {
|
||||
if (m_timerResults) {
|
||||
Timer t("Tokenizer::tokenize::setVarId", _settings->_showtime, m_timerResults);
|
||||
setVarId();
|
||||
|
@ -2014,7 +2010,6 @@ bool Tokenizer::tokenize(std::istream &code,
|
|||
|
||||
// Change initialisation of variable to assignment
|
||||
simplifyInitVar();
|
||||
}
|
||||
|
||||
// Convert e.g. atol("0") into 0
|
||||
simplifyMathFunctions();
|
||||
|
@ -2040,6 +2035,155 @@ bool Tokenizer::tokenize(std::istream &code,
|
|||
}
|
||||
//---------------------------------------------------------------------------
|
||||
|
||||
bool Tokenizer::tokenizeCondition(const std::string &code) {
|
||||
assert(_settings);
|
||||
|
||||
// Fill the map _typeSize..
|
||||
_typeSize.clear();
|
||||
_typeSize["char"] = 1;
|
||||
_typeSize["bool"] = _settings->sizeof_bool;
|
||||
_typeSize["short"] = _settings->sizeof_short;
|
||||
_typeSize["int"] = _settings->sizeof_int;
|
||||
_typeSize["long"] = _settings->sizeof_long;
|
||||
_typeSize["float"] = _settings->sizeof_float;
|
||||
_typeSize["double"] = _settings->sizeof_double;
|
||||
_typeSize["wchar_t"] = _settings->sizeof_wchar_t;
|
||||
_typeSize["size_t"] = _settings->sizeof_size_t;
|
||||
_typeSize["*"] = _settings->sizeof_pointer;
|
||||
|
||||
{
|
||||
std::istringstream istr(code);
|
||||
if (!list.createTokens(istr, "")) {
|
||||
cppcheckError(0);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Combine wide strings
|
||||
for (Token *tok = list.front();
|
||||
tok;
|
||||
tok = tok->next()) {
|
||||
while (tok->str() == "L" && tok->next() && tok->next()->type() == Token::eString) {
|
||||
// Combine 'L "string"'
|
||||
tok->str(tok->next()->str());
|
||||
tok->deleteNext();
|
||||
tok->isLong(true);
|
||||
}
|
||||
}
|
||||
|
||||
// Combine strings
|
||||
for (Token *tok = list.front();
|
||||
tok;
|
||||
tok = tok->next()) {
|
||||
if (tok->str()[0] != '"')
|
||||
continue;
|
||||
|
||||
tok->str(simplifyString(tok->str()));
|
||||
while (tok->next() && tok->next()->type() == Token::eString) {
|
||||
tok->next()->str(simplifyString(tok->next()->str()));
|
||||
|
||||
// Two strings after each other, combine them
|
||||
tok->concatStr(tok->next()->str());
|
||||
tok->deleteNext();
|
||||
}
|
||||
}
|
||||
|
||||
// Remove "volatile", "inline", "register", and "restrict"
|
||||
simplifyKeyword();
|
||||
|
||||
// convert platform dependent types to standard types
|
||||
// 32 bits: size_t -> unsigned long
|
||||
// 64 bits: size_t -> unsigned long long
|
||||
simplifyPlatformTypes();
|
||||
|
||||
// collapse compound standard types into a single token
|
||||
// unsigned long long int => long _isUnsigned=true,_isLong=true
|
||||
simplifyStdType();
|
||||
|
||||
// Concatenate double sharp: 'a ## b' -> 'ab'
|
||||
concatenateDoubleSharp();
|
||||
|
||||
if (!createLinks()) {
|
||||
// Source has syntax errors, can't proceed
|
||||
return false;
|
||||
}
|
||||
|
||||
// replace 'NULL' and similar '0'-defined macros with '0'
|
||||
simplifyNull();
|
||||
|
||||
// replace 'sin(0)' to '0' and other similar math expressions
|
||||
simplifyMathExpressions();
|
||||
|
||||
// combine "- %num%"
|
||||
concatenateNegativeNumberAndAnyPositive();
|
||||
|
||||
// simplify simple calculations
|
||||
for (Token *tok = list.front() ? list.front()->next() : NULL;
|
||||
tok;
|
||||
tok = tok->next()) {
|
||||
if (tok->isNumber())
|
||||
TemplateSimplifier::simplifyNumericCalculations(tok->previous());
|
||||
}
|
||||
|
||||
// Combine tokens..
|
||||
for (Token *tok = list.front();
|
||||
tok && tok->next();
|
||||
tok = tok->next()) {
|
||||
const char c1 = tok->str()[0];
|
||||
|
||||
if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
|
||||
const char c2 = tok->next()->str()[0];
|
||||
|
||||
// combine +-*/ and =
|
||||
if (c2 == '=' && (std::strchr("+-*/%&|^=!<>", c1))) {
|
||||
tok->str(tok->str() + c2);
|
||||
tok->deleteNext();
|
||||
continue;
|
||||
}
|
||||
|
||||
// replace "->" with "."
|
||||
else if (c1 == '-' && c2 == '>') {
|
||||
tok->str(".");
|
||||
tok->deleteNext();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
else if (tok->str() == ">>" && tok->next()->str() == "=") {
|
||||
tok->str(">>=");
|
||||
tok->deleteNext();
|
||||
}
|
||||
|
||||
else if (tok->str() == "<<" && tok->next()->str() == "=") {
|
||||
tok->str("<<=");
|
||||
tok->deleteNext();
|
||||
}
|
||||
|
||||
else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
|
||||
if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
|
||||
tok->str(tok->str() + ":");
|
||||
tok->deleteNext();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
simplifyRedundantParentheses();
|
||||
for (Token *tok = list.front();
|
||||
tok;
|
||||
tok = tok->next())
|
||||
while (TemplateSimplifier::simplifyNumericCalculations(tok));
|
||||
|
||||
while (simplifyLogicalOperators()) { }
|
||||
|
||||
// Convert e.g. atol("0") into 0
|
||||
simplifyMathFunctions();
|
||||
|
||||
simplifyDoublePlusAndDoubleMinus();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Tokenizer::hasComplicatedSyntaxErrorsInTemplates()
|
||||
{
|
||||
const Token *tok = TemplateSimplifier::hasComplicatedSyntaxErrorsInTemplates(list.front());
|
||||
|
|
|
@ -89,8 +89,14 @@ public:
|
|||
*/
|
||||
bool tokenize(std::istream &code,
|
||||
const char FileName[],
|
||||
const std::string &configuration = "",
|
||||
const bool preprocessorCondition = false);
|
||||
const std::string &configuration = "");
|
||||
|
||||
/**
|
||||
* tokenize condition and run simple simplifications on it
|
||||
* @param code code
|
||||
* @return true if success.
|
||||
*/
|
||||
bool tokenizeCondition(const std::string &code);
|
||||
|
||||
/** Set variable id */
|
||||
void setVarId();
|
||||
|
|
|
@ -5285,7 +5285,7 @@ private:
|
|||
Settings settings;
|
||||
Tokenizer tokenizer(&settings, this);
|
||||
std::istringstream istr(code);
|
||||
tokenizer.tokenize(istr, "test.cpp", "", false);
|
||||
tokenizer.tokenize(istr, "test.cpp", "");
|
||||
ASSERT_EQUALS(true, tokenizer.validate());
|
||||
}
|
||||
|
||||
|
@ -5296,7 +5296,7 @@ private:
|
|||
Settings settings;
|
||||
Tokenizer tokenizer(&settings, this);
|
||||
std::istringstream istr(code);
|
||||
tokenizer.tokenize(istr, "test.cpp", "", false);
|
||||
tokenizer.tokenize(istr, "test.cpp", "");
|
||||
ASSERT_EQUALS(true, tokenizer.validate());
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue