From e5220bdf0c9f2f4a7fadb66082622b45ce0a3b72 Mon Sep 17 00:00:00 2001 From: IOBYTE Date: Wed, 4 Sep 2019 02:07:30 -0400 Subject: [PATCH] make ellipsis ... a single token (#2143) * make ellipsis ... a single token Using cppcheck -E to preprocess code with ellipsis produces output that can't be compiled because ... is split into 3 tokens. * try to fix addon --- addons/misc.py | 2 +- externals/simplecpp/simplecpp.cpp | 18 ++++++++++-------- lib/checkother.cpp | 4 ++-- lib/symboldatabase.cpp | 4 ++-- lib/templatesimplifier.cpp | 20 +++++++------------- lib/token.cpp | 2 ++ lib/token.h | 1 + lib/tokenize.cpp | 29 ++++++++++++++--------------- lib/tokenlist.cpp | 7 +++++-- test/testsimplifytemplate.cpp | 20 ++++++++++---------- test/testtokenize.cpp | 20 ++++++++++---------- test/testvarid.cpp | 2 +- 12 files changed, 65 insertions(+), 64 deletions(-) diff --git a/addons/misc.py b/addons/misc.py index 6e2c59f4b..84cb6c238 100644 --- a/addons/misc.py +++ b/addons/misc.py @@ -104,7 +104,7 @@ def ellipsisStructArg(data): for argnr, argvar in tok.astOperand1.function.argument.items(): if argnr < 1: continue - if not simpleMatch(argvar.typeStartToken, '. . .'): + if not simpleMatch(argvar.typeStartToken, '...'): continue callArgs = getArguments(tok) for i in range(argnr-1, len(callArgs)): diff --git a/externals/simplecpp/simplecpp.cpp b/externals/simplecpp/simplecpp.cpp index 2493a0a0b..47a0b631a 100644 --- a/externals/simplecpp/simplecpp.cpp +++ b/externals/simplecpp/simplecpp.cpp @@ -745,10 +745,14 @@ void simplecpp::TokenList::combineOperators() } if (tok->op == '.') { - if (tok->previous && tok->previous->op == '.') - continue; - if (tok->next && tok->next->op == '.') + // ellipsis ... + if (tok->next && tok->next->op == '.' && tok->next->location.col == (tok->location.col + 1) && + tok->next->next && tok->next->next->op == '.' && tok->next->next->location.col == (tok->location.col + 2)) { + tok->setstr("..."); + deleteToken(tok->next); + deleteToken(tok->next); continue; + } // float literals.. if (tok->previous && tok->previous->number) { tok->setstr(tok->previous->str() + '.'); @@ -1376,14 +1380,12 @@ namespace simplecpp { args.clear(); const Token *argtok = nameTokDef->next->next; while (sameline(nametoken, argtok) && argtok->op != ')') { - if (argtok->op == '.' && - argtok->next && argtok->next->op == '.' && - argtok->next->next && argtok->next->next->op == '.' && - argtok->next->next->next && argtok->next->next->next->op == ')') { + if (argtok->str() == "..." && + argtok->next && argtok->next->op == ')') { variadic = true; if (!argtok->previous->name) args.push_back("__VA_ARGS__"); - argtok = argtok->next->next->next; // goto ')' + argtok = argtok->next; // goto ')' break; } if (argtok->op != ',') diff --git a/lib/checkother.cpp b/lib/checkother.cpp index d321299b1..98b34708c 100644 --- a/lib/checkother.cpp +++ b/lib/checkother.cpp @@ -1229,7 +1229,7 @@ void CheckOther::checkPassByReference() if (!var || !var->isArgument() || !var->isClass() || var->isPointer() || var->isArray() || var->isReference() || var->isEnumType()) continue; - if (var->scope() && var->scope()->function->arg->link()->strAt(-1) == ".") + if (var->scope() && var->scope()->function->arg->link()->strAt(-1) == "...") continue; // references could not be used as va_start parameters (#5824) bool inconclusive = false; @@ -2447,7 +2447,7 @@ void CheckOther::checkVarFuncNullUB() if (f && f->argCount() <= argnr) { const Token *tok2 = f->argDef; tok2 = tok2 ? tok2->link() : nullptr; // goto ')' - if (tok2 && Token::simpleMatch(tok2->tokAt(-3), ". . .")) + if (tok2 && Token::simpleMatch(tok2->tokAt(-1), "...")) varFuncNullUBError(tok); } } diff --git a/lib/symboldatabase.cpp b/lib/symboldatabase.cpp index b3827646a..63fe4d965 100644 --- a/lib/symboldatabase.cpp +++ b/lib/symboldatabase.cpp @@ -1776,7 +1776,7 @@ void Variable::evaluate(const Settings* settings) std::string strtype = mTypeStartToken->str(); for (const Token *typeToken = mTypeStartToken; Token::Match(typeToken, "%type% :: %type%"); typeToken = typeToken->tokAt(2)) strtype += "::" + typeToken->strAt(2); - setFlag(fIsClass, !lib->podtype(strtype) && !mTypeStartToken->isStandardType() && !isEnumType() && !isPointer() && !isReference()); + setFlag(fIsClass, !lib->podtype(strtype) && !mTypeStartToken->isStandardType() && !isEnumType() && !isPointer() && !isReference() && strtype != "..."); setFlag(fIsStlType, Token::simpleMatch(mTypeStartToken, "std ::")); setFlag(fIsStlString, isStlType() && (Token::Match(mTypeStartToken->tokAt(2), "string|wstring|u16string|u32string !!::") || (Token::simpleMatch(mTypeStartToken->tokAt(2), "basic_string <") && !Token::simpleMatch(mTypeStartToken->linkAt(3), "> ::")))); setFlag(fIsSmartPointer, lib->isSmartPointer(mTypeStartToken)); @@ -3306,7 +3306,7 @@ void Function::addArguments(const SymbolDatabase *symbolDatabase, const Scope *s if (tok->str() == ")") { // check for a variadic function - if (Token::simpleMatch(startTok, ". . .")) + if (Token::simpleMatch(startTok, "...")) isVariadic(true); break; diff --git a/lib/templatesimplifier.cpp b/lib/templatesimplifier.cpp index 023be4170..96b7c55c5 100644 --- a/lib/templatesimplifier.cpp +++ b/lib/templatesimplifier.cpp @@ -113,7 +113,7 @@ TemplateSimplifier::TokenAndName::TokenAndName(Token *token, const std::string & isClass(Token::Match(next, "class|struct|union %name% <|{|:|;|::")); if (mToken->strAt(1) == "<" && !isSpecialization()) { const Token *end = mToken->next()->findClosingBracket(); - isVariadic(end && Token::findmatch(mToken->tokAt(2), "typename|class . . .", end)); + isVariadic(end && Token::findmatch(mToken->tokAt(2), "typename|class ...", end)); } const Token *tok1 = mNameToken->next(); if (tok1->str() == "<") { @@ -455,11 +455,11 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok) tok = tok->next(); // Skip variadic types (Ticket #5774, #6059, #6172) - if (Token::simpleMatch(tok, ". . .")) { + if (Token::simpleMatch(tok, "...")) { if ((tok->previous()->isName() && !Token::Match(tok->tokAt(-2), "<|,|::")) || (!tok->previous()->isName() && tok->strAt(-1) != ">")) return 0; // syntax error - tok = tok->tokAt(3); + tok = tok->next(); if (!tok) return 0; if (tok->str() == ">") { @@ -726,7 +726,7 @@ bool TemplateSimplifier::getTemplateDeclarations() if (!tok->tokAt(2)) syntaxError(tok->next()); if (tok->strAt(2)=="typename" && - !Token::Match(tok->tokAt(3), "%name%|.|,|=|>")) + !Token::Match(tok->tokAt(3), "%name%|...|,|=|>")) syntaxError(tok->next()); codeWithTemplates = true; const Token * const parmEnd = tok1->next()->findClosingBracket(); @@ -1630,10 +1630,8 @@ void TemplateSimplifier::expandTemplate( for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token(); typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>")); typetok = typetok->next()) { - if (Token::simpleMatch(typetok, ". . .")) { - typetok = typetok->tokAt(2); + if (Token::simpleMatch(typetok, "...")) continue; - } if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next()))) ++typeindentlevel; else if (typeindentlevel > 0 && typetok->str() == ">") @@ -1856,9 +1854,7 @@ void TemplateSimplifier::expandTemplate( for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token(); typetok && (typeindentlevel>0 || !Token::Match(typetok, ",|>")); typetok = typetok->next()) { - if (Token::simpleMatch(typetok, ". . .")) { - typetok = typetok->tokAt(2); - } else { + if (!Token::simpleMatch(typetok, "...")) { if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next()))) ++typeindentlevel; else if (typeindentlevel > 0 && typetok->str() == ">") @@ -1960,10 +1956,8 @@ void TemplateSimplifier::expandTemplate( for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token(); typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>")); typetok = typetok->next()) { - if (Token::simpleMatch(typetok, ". . .")) { - typetok = typetok->tokAt(2); + if (Token::simpleMatch(typetok, "...")) continue; - } if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next()))) { brackets1.push(typetok->next()); diff --git a/lib/token.cpp b/lib/token.cpp index 79590a974..fa4f92cf1 100644 --- a/lib/token.cpp +++ b/lib/token.cpp @@ -128,6 +128,8 @@ void Token::update_property_info() tokType(eIncDecOp); else if (mStr.size() == 1 && (mStr.find_first_of("{}") != std::string::npos || (mLink && mStr.find_first_of("<>") != std::string::npos))) tokType(eBracket); + else if (mStr == "...") + tokType(eEllipsis); else tokType(eOther); } else { diff --git a/lib/token.h b/lib/token.h index 7fb8e412d..bd1efee8b 100644 --- a/lib/token.h +++ b/lib/token.h @@ -176,6 +176,7 @@ public: eArithmeticalOp, eComparisonOp, eAssignmentOp, eLogicalOp, eBitOp, eIncDecOp, eExtendedOp, // Operators: Arithmetical, Comparison, Assignment, Logical, Bitwise, ++/--, Extended eBracket, // {, }, <, >: < and > only if link() is set. Otherwise they are comparison operators. eLambda, // A function without a name + eEllipsis, // "..." eOther, eNone }; diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 43ce74237..5865e8ead 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -2797,29 +2797,27 @@ void Tokenizer::simplifyLabelsCaseDefault() void Tokenizer::simplifyCaseRange() { for (Token* tok = list.front(); tok; tok = tok->next()) { - if (Token::Match(tok, "case %num% . . . %num% :")) { + if (Token::Match(tok, "case %num% ... %num% :")) { const MathLib::bigint start = MathLib::toLongNumber(tok->strAt(1)); - MathLib::bigint end = MathLib::toLongNumber(tok->strAt(5)); + MathLib::bigint end = MathLib::toLongNumber(tok->strAt(3)); end = std::min(start + 50, end); // Simplify it 50 times at maximum if (start < end) { tok = tok->tokAt(2); tok->str(":"); - tok->deleteNext(); - tok->next()->str("case"); + tok->insertToken("case"); for (MathLib::bigint i = end-1; i > start; i--) { tok->insertToken(":"); tok->insertToken(MathLib::toString(i)); tok->insertToken("case"); } } - } else if (Token::Match(tok, "case %char% . . . %char% :")) { + } else if (Token::Match(tok, "case %char% ... %char% :")) { const char start = tok->strAt(1)[1]; - const char end = tok->strAt(5)[1]; + const char end = tok->strAt(3)[1]; if (start < end) { tok = tok->tokAt(2); tok->str(":"); - tok->deleteNext(); - tok->next()->str("case"); + tok->insertToken("case"); for (char i = end - 1; i > start; i--) { tok->insertToken(":"); if (i == '\\') { @@ -3970,13 +3968,13 @@ void Tokenizer::createLinks2() continue; if (token->next() && !Token::Match(token->next(), "%name%|%comp%|&|&&|*|::|,|(|)|{|}|;|[|:|.|=") && - !Token::simpleMatch(token->next(), ". . .") && + !Token::simpleMatch(token->next(), "...") && !Token::Match(token->next(), "&& %name% =")) continue; } // if > is followed by [ .. "new a[" is expected // unless this is from varidiac expansion - if (token->strAt(1) == "[" && !Token::simpleMatch(token->tokAt(-3), ". . . >")) { + if (token->strAt(1) == "[" && !Token::simpleMatch(token->tokAt(-1), "... >")) { Token *prev = type.top()->previous(); while (prev && Token::Match(prev->previous(), ":: %name%")) prev = prev->tokAt(-2); @@ -4088,8 +4086,9 @@ bool Tokenizer::simplifySizeof() if (tok->str() != "sizeof") continue; - if (Token::simpleMatch(tok->next(), ". . .")) { - tok->deleteNext(3); + if (Token::simpleMatch(tok->next(), "...")) { + //tok->deleteNext(3); + tok->deleteNext(); } // sizeof('x') @@ -4991,9 +4990,9 @@ void Tokenizer::simplifyHeaders() if (removeUnusedTemplates || (isIncluded && removeUnusedIncludedTemplates)) { if (Token::Match(tok->next(), "template < %name%")) { const Token *tok2 = tok->tokAt(3); - while (Token::Match(tok2, "%name% %name% [,=>]") || Token::Match(tok2, "typename . . . %name% [,>]")) { - if (Token::simpleMatch(tok2, "typename . . .")) - tok2 = tok2->tokAt(5); + while (Token::Match(tok2, "%name% %name% [,=>]") || Token::Match(tok2, "typename ... %name% [,>]")) { + if (Token::simpleMatch(tok2, "typename ...")) + tok2 = tok2->tokAt(3); else tok2 = tok2->tokAt(2); if (Token::Match(tok2, "= %name% [,>]")) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index ad78358cb..8353c4438 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -688,8 +688,8 @@ static void compileTerm(Token *&tok, AST_state& state) state.op.push(tok); if (Token::Match(tok, "%name% <") && tok->linkAt(1)) tok = tok->linkAt(1); - else if (Token::Match(tok, "%name% . . .")) - tok = tok->tokAt(3); + else if (Token::Match(tok, "%name% ...")) + tok = tok->next(); tok = tok->next(); if (Token::Match(tok, "%str%")) { while (Token::Match(tok, "%name%|%str%")) @@ -767,6 +767,9 @@ static void compilePrecedence2(Token *&tok, AST_state& state) while (tok) { if (tok->tokType() == Token::eIncDecOp && !isPrefixUnary(tok, state.cpp)) { compileUnaryOp(tok, state, compileScope); + } else if (tok->str() == "...") { + state.op.push(tok); + break; } else if (tok->str() == "." && tok->strAt(1) != "*") { if (tok->strAt(1) == ".") { state.op.push(tok); diff --git a/test/testsimplifytemplate.cpp b/test/testsimplifytemplate.cpp index 542f3a5ee..376ddc7e0 100644 --- a/test/testsimplifytemplate.cpp +++ b/test/testsimplifytemplate.cpp @@ -2908,11 +2908,11 @@ private: "struct b, d<>>;\n" "}\n" "void e() { using c = a<>; }"; - const char exp[] = "template < class . . . > struct a ; " + const char exp[] = "template < class ... > struct a ; " "namespace { " "template < class , class > struct b ; " - "template < template < class > class c , class . . . f , template < class . . . > class d > " - "struct b < c < f . . . > , d < > > ; " + "template < template < class > class c , class ... f , template < class ... > class d > " + "struct b < c < f ... > , d < > > ; " "} " "void e ( ) { }"; ASSERT_EQUALS(exp, tok(code)); @@ -2930,12 +2930,12 @@ private: " using c = a<>;\n" " using e = a<>;\n" "}"; - const char exp[] = "template < class . . . > struct a ; " + const char exp[] = "template < class ... > struct a ; " "namespace { " "template < class , class , class , class > " "struct b ; " - "template < template < class > class c , class . . . d , template < class > class e , class . . . f > " - "struct b < c < d . . . > , e < f . . . > > ; " + "template < template < class > class c , class ... d , template < class > class e , class ... f > " + "struct b < c < d ... > , e < f ... > > ; " "} " "void fn1 ( ) { " "}"; @@ -2953,7 +2953,7 @@ private: "template < bool b > using c = typename a < b > :: d ; " "template < typename > struct e ; " "template < typename > struct h { " - "template < typename . . . f , c < h < e < typename f :: d . . . > > :: g > > void i ( ) ; " + "template < typename ... f , c < h < e < typename f :: d ... > > :: g > > void i ( ) ; " "} ;"; ASSERT_EQUALS(exp, tok(code)); } @@ -3271,9 +3271,9 @@ private: const char exp[] = "template < bool > struct a ; " "template < bool b , class > using c = typename a < b > :: d ; " "template < class , template < class > class , class > struct e ; " - "template < class f , class g , class . . . h > " - "using i = typename e < f , g :: template fn , h . . . > :: d ; " - "template < class . . . j > struct k : c < sizeof . . . ( j ) , int > :: template fn < j . . . > { } ;"; + "template < class f , class g , class ... h > " + "using i = typename e < f , g :: template fn , h ... > :: d ; " + "template < class ... j > struct k : c < sizeof ... ( j ) , int > :: template fn < j ... > { } ;"; ASSERT_EQUALS(exp, tok(code)); } diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 003506c67..ea2cb983d 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -1488,7 +1488,7 @@ private: " for(;;) try { } catch (...) { }\n" "}"; const char expected[] = "void f ( ) {\n" - "for ( ; ; ) { try { } catch ( . . . ) { } }\n" + "for ( ; ; ) { try { } catch ( ... ) { } }\n" "}"; ASSERT_EQUALS(expected, tokenizeAndStringify(code, true)); } @@ -5277,27 +5277,27 @@ private: void cpp0xtemplate5() { // #9154 { const char *code = "struct s>;"; - ASSERT_EQUALS("struct s < x < u . . . > > ;", + ASSERT_EQUALS("struct s < x < u ... > > ;", tokenizeAndStringify(code)); } { const char *code = "template using c = e,b...>>;"; - ASSERT_EQUALS("template < class f > using c = e < i < q < f , r > , b . . . > > ;", + ASSERT_EQUALS("template < class f > using c = e < i < q < f , r > , b ... > > ;", tokenizeAndStringify(code)); } { const char *code = "struct s> { };"; - ASSERT_EQUALS("struct s < x < u . . . > > { } ;", + ASSERT_EQUALS("struct s < x < u ... > > { } ;", tokenizeAndStringify(code)); } { const char *code = "struct q : s> { };"; - ASSERT_EQUALS("struct q : s < x < u . . . > > { } ;", + ASSERT_EQUALS("struct q : s < x < u ... > > { } ;", tokenizeAndStringify(code)); } { const char *code = "struct q : private s> { };"; - ASSERT_EQUALS("struct q : private s < x < u . . . > > { } ;", + ASSERT_EQUALS("struct q : private s < x < u ... > > { } ;", tokenizeAndStringify(code)); } } @@ -7115,11 +7115,11 @@ private: void simplifyCaseRange() { ASSERT_EQUALS("void f ( ) { switch ( x ) { case 1 : case 2 : case 3 : case 4 : ; } }", tokenizeAndStringify("void f() { switch(x) { case 1 ... 4: } }")); - ASSERT_EQUALS("void f ( ) { switch ( x ) { case 4 . . . 1 : ; } }", tokenizeAndStringify("void f() { switch(x) { case 4 ... 1: } }")); + ASSERT_EQUALS("void f ( ) { switch ( x ) { case 4 ... 1 : ; } }", tokenizeAndStringify("void f() { switch(x) { case 4 ... 1: } }")); tokenizeAndStringify("void f() { switch(x) { case 1 ... 1000000: } }"); // Do not run out of memory ASSERT_EQUALS("void f ( ) { switch ( x ) { case 'a' : case 'b' : case 'c' : ; } }", tokenizeAndStringify("void f() { switch(x) { case 'a' ... 'c': } }")); - ASSERT_EQUALS("void f ( ) { switch ( x ) { case 'c' . . . 'a' : ; } }", tokenizeAndStringify("void f() { switch(x) { case 'c' ... 'a': } }")); + ASSERT_EQUALS("void f ( ) { switch ( x ) { case 'c' ... 'a' : ; } }", tokenizeAndStringify("void f() { switch(x) { case 'c' ... 'a': } }")); ASSERT_EQUALS("void f ( ) { switch ( x ) { case '[' : case '\\\\' : case ']' : ; } }", tokenizeAndStringify("void f() { switch(x) { case '[' ... ']': } }")); } @@ -7275,7 +7275,7 @@ private: ASSERT_EQUALS("abc.1:?1+bd.1:?+=", testAst("a =(b.c ? : 1) + 1 + (b.d ? : 1);")); - ASSERT_EQUALS("catch.(", testAst("try {} catch (...) {}")); + ASSERT_EQUALS("catch...(", testAst("try {} catch (...) {}")); ASSERT_EQUALS("FooBar(", testAst("void Foo(Bar&);")); ASSERT_EQUALS("FooBar(", testAst("void Foo(Bar& &);")); // Rvalue reference - simplified from && to & & by real tokenizer @@ -7536,7 +7536,7 @@ private: ASSERT_EQUALS("1f2a&,(+", testAst("1+f(2,&a)")); ASSERT_EQUALS("fargv[(", testAst("int f(char argv[]);")); ASSERT_EQUALS("fchar(", testAst("extern unsigned f(const char *);")); - ASSERT_EQUALS("fcharformat*.,(", testAst("extern void f(const char *format, ...);")); + ASSERT_EQUALS("fcharformat*...,(", testAst("extern void f(const char *format, ...);")); ASSERT_EQUALS("for_each_commit_graftint((void,(", testAst("extern int for_each_commit_graft(int (*)(int*), void *);")); ASSERT_EQUALS("for;;(", testAst("for (;;) {}")); ASSERT_EQUALS("xsizeofvoid(=", testAst("x=sizeof(void*)")); diff --git a/test/testvarid.cpp b/test/testvarid.cpp index 271fc30ac..ae10e84f1 100644 --- a/test/testvarid.cpp +++ b/test/testvarid.cpp @@ -2225,7 +2225,7 @@ private: } void varid_variadicFunc() { - ASSERT_EQUALS("1: int foo ( . . . ) ;\n", tokenize("int foo(...);")); + ASSERT_EQUALS("1: int foo ( ... ) ;\n", tokenize("int foo(...);")); } void varid_typename() {