diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index d42e6e0d4..6fd4d7ec4 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -1935,26 +1935,35 @@ bool Tokenizer::tokenize(std::istream &code, tok->str(MathLib::toString(tok->linenr())); } - // token concatenation - for (Token *tok = _tokens; tok; tok = tok->next()) { - // TODO: pattern should be "%var%|%num% ## %var%|%num%" - if (Token::Match(tok, "%any% ## %any%") && - (tok->isName() || tok->isNumber()) && - (tok->tokAt(2)->isName() || tok->tokAt(2)->isNumber())) { - tok->str(tok->str() + tok->strAt(2)); - tok->deleteNext(2); - if (tok->previous()) + // 'double sharp' token concatenation + { + bool goback = false; + for (Token *tok = _tokens; tok; tok = tok->next()) { + if (goback) { + goback = false; tok = tok->previous(); + } + // TODO: pattern should be "%var%|%num% ## %var%|%num%" + if (Token::Match(tok, "%any% ## %any%") && + (tok->isName() || tok->isNumber()) && + (tok->tokAt(2)->isName() || tok->tokAt(2)->isNumber())) { + tok->str(tok->str() + tok->strAt(2)); + tok->deleteNext(2); + goback = true; + } } } // Convert C# code if (_files[0].find(".cs")) { for (Token *tok = _tokens; tok; tok = tok->next()) { - if (Token::Match(tok, "[;{}] %type% [ ] %var% [=;]")) { + if (Token::Match(tok, "%type% [ ] %var% [=;]") && + (!tok->previous() || Token::Match(tok->previous(), "[;{}]"))) { + tok->deleteNext(2); + tok->insertToken("*"); tok = tok->tokAt(2); - tok->str("*"); - tok->deleteNext(); + if (tok->next()->str() == "=") + tok = tok->next(); } } } @@ -2620,9 +2629,12 @@ void Tokenizer::simplifyLabelsCaseDefault() // Simplify labels in the executable scope.. unsigned int indentlevel = 0; while (NULL != (tok = tok->next())) { - if (tok->str() == "{") - ++indentlevel; - else if (tok->str() == "}") { + if (tok->str() == "{") { + if (tok->previous() && tok->previous()->str() == "=") + tok = tok->link(); + else + ++indentlevel; + } else if (tok->str() == "}") { --indentlevel; if (!indentlevel) break; @@ -5703,18 +5715,15 @@ void Tokenizer::simplifyVarDecl(bool only_k_r_fpar) for (Token *tok = _tokens; tok; tok = tok->next()) { if (Token::simpleMatch(tok, "= {")) { tok = tok->next()->link(); - if (!tok) - break; } if (only_k_r_fpar) { - if (tok->str() == "(" || tok->str() == "{") { + if (tok->link()) { tok = tok->link(); - if (!tok) - break; if (tok->next() && Token::Match(tok, ") !!{")) tok = tok->next(); - else continue; + else + continue; } else continue; } @@ -5768,10 +5777,9 @@ void Tokenizer::simplifyVarDecl(bool only_k_r_fpar) if (Token::Match(tok2, "%type% *| %var% ,|=")) { const bool isPointer = (tok2->next()->str() == "*"); const Token *varName = tok2->tokAt((isPointer ? 2 : 1)); - Token *endDeclaration = varName->next(); if (varName->str() != "operator") { - tok2 = endDeclaration; // The ',' or '=' token + tok2 = varName->next(); // The ',' or '=' token if (isstatic && tok2->str() == "=") { if (Token::Match(tok2->next(), "%num% ,")) @@ -5866,70 +5874,69 @@ void Tokenizer::simplifyVarDecl(bool only_k_r_fpar) --typelen; } else { tok2 = NULL; - typelen = 0; } } else { tok2 = NULL; - typelen = 0; } - if (tok2) { - if (tok2->str() == ",") { - tok2->str(";"); - insertTokens(tok2, type0, typelen); - std::stack link1; - std::stack link2; - while (((typelen--) > 0) && (NULL != (tok2 = tok2->next()))) { - if (tok2->str() == "(") - link1.push(tok2); - else if (tok2->str() == ")" && !link1.empty()) { - Token::createMutualLinks(tok2, link1.top()); - link1.pop(); - } + if (!tok2) + continue; - else if (tok2->str() == "[") - link2.push(tok2); - else if (tok2->str() == "]" && !link2.empty()) { - Token::createMutualLinks(tok2, link2.top()); - link2.pop(); - } + if (tok2->str() == ",") { + tok2->str(";"); + insertTokens(tok2, type0, typelen); + std::stack link1; + std::stack link2; + while (((typelen--) > 0) && (NULL != (tok2 = tok2->next()))) { + if (tok2->str() == "(") + link1.push(tok2); + else if (tok2->str() == ")" && !link1.empty()) { + Token::createMutualLinks(tok2, link1.top()); + link1.pop(); + } + + else if (tok2->str() == "[") + link2.push(tok2); + else if (tok2->str() == "]" && !link2.empty()) { + Token::createMutualLinks(tok2, link2.top()); + link2.pop(); } } + } - else { - Token *eq = tok2; + else { + Token *eq = tok2; - unsigned int level = 0; - while (tok2) { - if (Token::Match(tok2, "[{(]")) - tok2 = tok2->link(); + unsigned int level = 0; + while (tok2) { + if (Token::Match(tok2, "[{(]")) + tok2 = tok2->link(); - else if (tok2->str() == "<") { - if (tok2->previous()->isName() && !tok2->previous()->varId()) - ++level; - } - - else if (level > 0 && tok2->str() == ">") - --level; - - else if (level == 0 && strchr(";,", tok2->str()[0])) { - // "type var =" => "type var; var =" - Token *VarTok = type0->tokAt((int)typelen); - while (Token::Match(VarTok, "*|&|const")) - VarTok = VarTok->next(); - insertTokens(eq, VarTok, 2); - eq->str(";"); - - // "= x, " => "= x; type " - if (tok2->str() == ",") { - tok2->str(";"); - insertTokens(tok2, type0, typelen); - } - break; - } - - tok2 = tok2->next(); + else if (tok2->str() == "<") { + if (tok2->previous()->isName() && !tok2->previous()->varId()) + ++level; } + + else if (level > 0 && tok2->str() == ">") + --level; + + else if (level == 0 && strchr(";,", tok2->str()[0])) { + // "type var =" => "type var; var =" + Token *VarTok = type0->tokAt((int)typelen); + while (Token::Match(VarTok, "*|&|const")) + VarTok = VarTok->next(); + insertTokens(eq, VarTok, 2); + eq->str(";"); + + // "= x, " => "= x; type " + if (tok2->str() == ",") { + tok2->str(";"); + insertTokens(tok2, type0, typelen); + } + break; + } + + tok2 = tok2->next(); } } } diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 09eead795..0f623ad7a 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -3882,7 +3882,7 @@ private: void doublesharp() { - const char code[] = "TEST(var,val) var##_##val = val\n"; + const char code[] = "a##_##b TEST(var,val) var##_##val = val\n"; errout.str(""); @@ -3898,7 +3898,7 @@ private: for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) ostr << tok->str() << " "; - ASSERT_EQUALS("TEST ( var , val ) var_val = val ", ostr.str()); + ASSERT_EQUALS("a_b TEST ( var , val ) var_val = val ", ostr.str()); } void macrodoublesharp() { @@ -5876,6 +5876,7 @@ private: } void cs() { + ASSERT_EQUALS("int * i ;", tokenizeAndStringify("int [] i;")); ASSERT_EQUALS("; int * i ;", tokenizeAndStringify("; int [] i;")); }