make ellipsis ... a single token (#2143)

* make ellipsis ... a single token

Using cppcheck -E to preprocess code with ellipsis produces output that
can't be compiled because ... is split into 3 tokens.

* try to fix addon

addons/misc.py

@@ -104,7 +104,7 @@ def ellipsisStructArg(data):
             for argnr, argvar in tok.astOperand1.function.argument.items():
                 if argnr < 1:
                     continue
-                if not simpleMatch(argvar.typeStartToken, '. . .'):
+                if not simpleMatch(argvar.typeStartToken, '...'):
                     continue
                 callArgs = getArguments(tok)
                 for i in range(argnr-1, len(callArgs)):

externals/simplecpp/simplecpp.cpp

@@ -745,10 +745,14 @@ void simplecpp::TokenList::combineOperators()
         }
 
         if (tok->op == '.') {
-            if (tok->previous && tok->previous->op == '.')
+            // ellipsis ...
-                continue;
+            if (tok->next && tok->next->op == '.' && tok->next->location.col == (tok->location.col + 1) &&
-            if (tok->next && tok->next->op == '.')
+                tok->next->next && tok->next->next->op == '.' && tok->next->next->location.col == (tok->location.col + 2)) {
+                tok->setstr("...");
+                deleteToken(tok->next);
+                deleteToken(tok->next);
                 continue;
+            }
             // float literals..
             if (tok->previous && tok->previous->number) {
                 tok->setstr(tok->previous->str() + '.');
@@ -1376,14 +1380,12 @@ namespace simplecpp {
                 args.clear();
                 const Token *argtok = nameTokDef->next->next;
                 while (sameline(nametoken, argtok) && argtok->op != ')') {
-                    if (argtok->op == '.' &&
+                    if (argtok->str() == "..." &&
-                        argtok->next && argtok->next->op == '.' &&
+                        argtok->next && argtok->next->op == ')') {
-                        argtok->next->next && argtok->next->next->op == '.' &&
-                        argtok->next->next->next && argtok->next->next->next->op == ')') {
                         variadic = true;
                         if (!argtok->previous->name)
                             args.push_back("__VA_ARGS__");
-                        argtok = argtok->next->next->next; // goto ')'
+                        argtok = argtok->next; // goto ')'
                         break;
                     }
                     if (argtok->op != ',')

lib/checkother.cpp

@@ -1229,7 +1229,7 @@ void CheckOther::checkPassByReference()
         if (!var || !var->isArgument() || !var->isClass() || var->isPointer() || var->isArray() || var->isReference() || var->isEnumType())
             continue;
 
-        if (var->scope() && var->scope()->function->arg->link()->strAt(-1) == ".")
+        if (var->scope() && var->scope()->function->arg->link()->strAt(-1) == "...")
             continue; // references could not be used as va_start parameters (#5824)
 
         bool inconclusive = false;
@@ -2447,7 +2447,7 @@ void CheckOther::checkVarFuncNullUB()
                     if (f && f->argCount() <= argnr) {
                         const Token *tok2 = f->argDef;
                         tok2 = tok2 ? tok2->link() : nullptr; // goto ')'
-                        if (tok2 && Token::simpleMatch(tok2->tokAt(-3), ". . ."))
+                        if (tok2 && Token::simpleMatch(tok2->tokAt(-1), "..."))
                             varFuncNullUBError(tok);
                     }
                 }

lib/symboldatabase.cpp

@@ -1776,7 +1776,7 @@ void Variable::evaluate(const Settings* settings)
         std::string strtype = mTypeStartToken->str();
         for (const Token *typeToken = mTypeStartToken; Token::Match(typeToken, "%type% :: %type%"); typeToken = typeToken->tokAt(2))
             strtype += "::" + typeToken->strAt(2);
-        setFlag(fIsClass, !lib->podtype(strtype) && !mTypeStartToken->isStandardType() && !isEnumType() && !isPointer() && !isReference());
+        setFlag(fIsClass, !lib->podtype(strtype) && !mTypeStartToken->isStandardType() && !isEnumType() && !isPointer() && !isReference() && strtype != "...");
         setFlag(fIsStlType, Token::simpleMatch(mTypeStartToken, "std ::"));
         setFlag(fIsStlString, isStlType() && (Token::Match(mTypeStartToken->tokAt(2), "string|wstring|u16string|u32string !!::") || (Token::simpleMatch(mTypeStartToken->tokAt(2), "basic_string <") && !Token::simpleMatch(mTypeStartToken->linkAt(3), "> ::"))));
         setFlag(fIsSmartPointer, lib->isSmartPointer(mTypeStartToken));
@@ -3306,7 +3306,7 @@ void Function::addArguments(const SymbolDatabase *symbolDatabase, const Scope *s
 
         if (tok->str() == ")") {
             // check for a variadic function
-            if (Token::simpleMatch(startTok, ". . ."))
+            if (Token::simpleMatch(startTok, "..."))
                 isVariadic(true);
 
             break;

lib/templatesimplifier.cpp

@@ -113,7 +113,7 @@ TemplateSimplifier::TokenAndName::TokenAndName(Token *token, const std::string &
         isClass(Token::Match(next, "class|struct|union %name% <|{|:|;|::"));
         if (mToken->strAt(1) == "<" && !isSpecialization()) {
             const Token *end = mToken->next()->findClosingBracket();
-            isVariadic(end && Token::findmatch(mToken->tokAt(2), "typename|class . . .", end));
+            isVariadic(end && Token::findmatch(mToken->tokAt(2), "typename|class ...", end));
         }
         const Token *tok1 = mNameToken->next();
         if (tok1->str() == "<") {
@@ -455,11 +455,11 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok)
             tok = tok->next();
 
         // Skip variadic types (Ticket #5774, #6059, #6172)
-        if (Token::simpleMatch(tok, ". . .")) {
+        if (Token::simpleMatch(tok, "...")) {
             if ((tok->previous()->isName() && !Token::Match(tok->tokAt(-2), "<|,|::")) ||
                 (!tok->previous()->isName() && tok->strAt(-1) != ">"))
                 return 0; // syntax error
-            tok = tok->tokAt(3);
+            tok = tok->next();
             if (!tok)
                 return 0;
             if (tok->str() == ">") {
@@ -726,7 +726,7 @@ bool TemplateSimplifier::getTemplateDeclarations()
         if (!tok->tokAt(2))
             syntaxError(tok->next());
         if (tok->strAt(2)=="typename" &&
-            !Token::Match(tok->tokAt(3), "%name%|.|,|=|>"))
+            !Token::Match(tok->tokAt(3), "%name%|...|,|=|>"))
             syntaxError(tok->next());
         codeWithTemplates = true;
         const Token * const parmEnd = tok1->next()->findClosingBracket();
@@ -1630,10 +1630,8 @@ void TemplateSimplifier::expandTemplate(
                 for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
                      typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>"));
                      typetok = typetok->next()) {
-                    if (Token::simpleMatch(typetok, ". . .")) {
+                    if (Token::simpleMatch(typetok, "..."))
-                        typetok = typetok->tokAt(2);
                         continue;
-                    }
                     if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next())))
                         ++typeindentlevel;
                     else if (typeindentlevel > 0 && typetok->str() == ">")
@@ -1856,9 +1854,7 @@ void TemplateSimplifier::expandTemplate(
                             for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
                                  typetok && (typeindentlevel>0 || !Token::Match(typetok, ",|>"));
                                  typetok = typetok->next()) {
-                                if (Token::simpleMatch(typetok, ". . .")) {
+                                if (!Token::simpleMatch(typetok, "...")) {
-                                    typetok = typetok->tokAt(2);
-                                } else {
                                     if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next())))
                                         ++typeindentlevel;
                                     else if (typeindentlevel > 0 && typetok->str() == ">")
@@ -1960,10 +1956,8 @@ void TemplateSimplifier::expandTemplate(
                     for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
                          typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>"));
                          typetok = typetok->next()) {
-                        if (Token::simpleMatch(typetok, ". . .")) {
+                        if (Token::simpleMatch(typetok, "..."))
-                            typetok = typetok->tokAt(2);
                             continue;
-                        }
                         if (Token::Match(typetok, "%name% <") &&
                             (typetok->strAt(2) == ">" || templateParameters(typetok->next()))) {
                             brackets1.push(typetok->next());

lib/token.cpp

@@ -128,6 +128,8 @@ void Token::update_property_info()
             tokType(eIncDecOp);
         else if (mStr.size() == 1 && (mStr.find_first_of("{}") != std::string::npos || (mLink && mStr.find_first_of("<>") != std::string::npos)))
             tokType(eBracket);
+        else if (mStr == "...")
+            tokType(eEllipsis);
         else
             tokType(eOther);
     } else {

lib/token.h

@@ -176,6 +176,7 @@ public:
         eArithmeticalOp, eComparisonOp, eAssignmentOp, eLogicalOp, eBitOp, eIncDecOp, eExtendedOp, // Operators: Arithmetical, Comparison, Assignment, Logical, Bitwise, ++/--, Extended
         eBracket, // {, }, <, >: < and > only if link() is set. Otherwise they are comparison operators.
         eLambda, // A function without a name
+        eEllipsis, // "..."
         eOther,
         eNone
     };

lib/tokenize.cpp

@@ -2797,29 +2797,27 @@ void Tokenizer::simplifyLabelsCaseDefault()
 void Tokenizer::simplifyCaseRange()
 {
     for (Token* tok = list.front(); tok; tok = tok->next()) {
-        if (Token::Match(tok, "case %num% . . . %num% :")) {
+        if (Token::Match(tok, "case %num% ... %num% :")) {
             const MathLib::bigint start = MathLib::toLongNumber(tok->strAt(1));
-            MathLib::bigint end = MathLib::toLongNumber(tok->strAt(5));
+            MathLib::bigint end = MathLib::toLongNumber(tok->strAt(3));
             end = std::min(start + 50, end); // Simplify it 50 times at maximum
             if (start < end) {
                 tok = tok->tokAt(2);
                 tok->str(":");
-                tok->deleteNext();
+                tok->insertToken("case");
-                tok->next()->str("case");
                 for (MathLib::bigint i = end-1; i > start; i--) {
                     tok->insertToken(":");
                     tok->insertToken(MathLib::toString(i));
                     tok->insertToken("case");
                 }
             }
-        } else if (Token::Match(tok, "case %char% . . . %char% :")) {
+        } else if (Token::Match(tok, "case %char% ... %char% :")) {
             const char start = tok->strAt(1)[1];
-            const char end = tok->strAt(5)[1];
+            const char end = tok->strAt(3)[1];
             if (start < end) {
                 tok = tok->tokAt(2);
                 tok->str(":");
-                tok->deleteNext();
+                tok->insertToken("case");
-                tok->next()->str("case");
                 for (char i = end - 1; i > start; i--) {
                     tok->insertToken(":");
                     if (i == '\\') {
@@ -3970,13 +3968,13 @@ void Tokenizer::createLinks2()
                     continue;
                 if (token->next() &&
                     !Token::Match(token->next(), "%name%|%comp%|&|&&|*|::|,|(|)|{|}|;|[|:|.|=") &&
-                    !Token::simpleMatch(token->next(), ". . .") &&
+                    !Token::simpleMatch(token->next(), "...") &&
                     !Token::Match(token->next(), "&& %name% ="))
                     continue;
             }
             // if > is followed by [ .. "new a<b>[" is expected
             // unless this is from varidiac expansion
-            if (token->strAt(1) == "[" && !Token::simpleMatch(token->tokAt(-3), ". . . >")) {
+            if (token->strAt(1) == "[" && !Token::simpleMatch(token->tokAt(-1), "... >")) {
                 Token *prev = type.top()->previous();
                 while (prev && Token::Match(prev->previous(), ":: %name%"))
                     prev = prev->tokAt(-2);
@@ -4088,8 +4086,9 @@ bool Tokenizer::simplifySizeof()
         if (tok->str() != "sizeof")
             continue;
 
-        if (Token::simpleMatch(tok->next(), ". . .")) {
+        if (Token::simpleMatch(tok->next(), "...")) {
-            tok->deleteNext(3);
+            //tok->deleteNext(3);
+            tok->deleteNext();
         }
 
         // sizeof('x')
@@ -4991,9 +4990,9 @@ void Tokenizer::simplifyHeaders()
             if (removeUnusedTemplates || (isIncluded && removeUnusedIncludedTemplates)) {
                 if (Token::Match(tok->next(), "template < %name%")) {
                     const Token *tok2 = tok->tokAt(3);
-                    while (Token::Match(tok2, "%name% %name% [,=>]") || Token::Match(tok2, "typename . . . %name% [,>]")) {
+                    while (Token::Match(tok2, "%name% %name% [,=>]") || Token::Match(tok2, "typename ... %name% [,>]")) {
-                        if (Token::simpleMatch(tok2, "typename . . ."))
+                        if (Token::simpleMatch(tok2, "typename ..."))
-                            tok2 = tok2->tokAt(5);
+                            tok2 = tok2->tokAt(3);
                         else
                             tok2 = tok2->tokAt(2);
                         if (Token::Match(tok2, "= %name% [,>]"))

lib/tokenlist.cpp

@@ -688,8 +688,8 @@ static void compileTerm(Token *&tok, AST_state& state)
             state.op.push(tok);
             if (Token::Match(tok, "%name% <") && tok->linkAt(1))
                 tok = tok->linkAt(1);
-            else if (Token::Match(tok, "%name% . . ."))
+            else if (Token::Match(tok, "%name% ..."))
-                tok = tok->tokAt(3);
+                tok = tok->next();
             tok = tok->next();
             if (Token::Match(tok, "%str%")) {
                 while (Token::Match(tok, "%name%|%str%"))
@@ -767,6 +767,9 @@ static void compilePrecedence2(Token *&tok, AST_state& state)
     while (tok) {
         if (tok->tokType() == Token::eIncDecOp && !isPrefixUnary(tok, state.cpp)) {
             compileUnaryOp(tok, state, compileScope);
+        } else if (tok->str() == "...") {
+            state.op.push(tok);
+            break;
         } else if (tok->str() == "." && tok->strAt(1) != "*") {
             if (tok->strAt(1) == ".") {
                 state.op.push(tok);

test/testsimplifytemplate.cpp

@@ -2908,11 +2908,11 @@ private:
                             "struct b<c<f...>, d<>>;\n"
                             "}\n"
                             "void e() { using c = a<>; }";
-        const char exp[] = "template < class . . . > struct a ; "
+        const char exp[] = "template < class ... > struct a ; "
                            "namespace { "
                            "template < class , class > struct b ; "
-                           "template < template < class > class c , class . . . f , template < class . . . > class d > "
+                           "template < template < class > class c , class ... f , template < class ... > class d > "
-                           "struct b < c < f . . . > , d < > > ; "
+                           "struct b < c < f ... > , d < > > ; "
                            "} "
                            "void e ( ) { }";
         ASSERT_EQUALS(exp, tok(code));
@@ -2930,12 +2930,12 @@ private:
                             "  using c = a<>;\n"
                             "  using e = a<>;\n"
                             "}";
-        const char exp[] = "template < class . . . > struct a ; "
+        const char exp[] = "template < class ... > struct a ; "
                            "namespace { "
                            "template < class , class , class , class > "
                            "struct b ; "
-                           "template < template < class > class c , class . . . d , template < class > class e , class . . . f > "
+                           "template < template < class > class c , class ... d , template < class > class e , class ... f > "
-                           "struct b < c < d . . . > , e < f . . . > > ; "
+                           "struct b < c < d ... > , e < f ... > > ; "
                            "} "
                            "void fn1 ( ) { "
                            "}";
@@ -2953,7 +2953,7 @@ private:
                            "template < bool b > using c = typename a < b > :: d ; "
                            "template < typename > struct e ; "
                            "template < typename > struct h { "
-                           "template < typename . . . f , c < h < e < typename f :: d . . . > > :: g > > void i ( ) ; "
+                           "template < typename ... f , c < h < e < typename f :: d ... > > :: g > > void i ( ) ; "
                            "} ;";
         ASSERT_EQUALS(exp, tok(code));
     }
@@ -3271,9 +3271,9 @@ private:
         const char exp[] = "template < bool > struct a ; "
                            "template < bool b , class > using c = typename a < b > :: d ; "
                            "template < class , template < class > class , class > struct e ; "
-                           "template < class f , class g , class . . . h > "
+                           "template < class f , class g , class ... h > "
-                           "using i = typename e < f , g :: template fn , h . . . > :: d ; "
+                           "using i = typename e < f , g :: template fn , h ... > :: d ; "
-                           "template < class . . . j > struct k : c < sizeof . . . ( j ) , int > :: template fn < j . . . > { } ;";
+                           "template < class ... j > struct k : c < sizeof ... ( j ) , int > :: template fn < j ... > { } ;";
         ASSERT_EQUALS(exp, tok(code));
     }
 

test/testtokenize.cpp

@@ -1488,7 +1488,7 @@ private:
                             "    for(;;) try { } catch (...) { }\n"
                             "}";
         const char expected[] = "void f ( ) {\n"
-                                "for ( ; ; ) { try { } catch ( . . . ) { } }\n"
+                                "for ( ; ; ) { try { } catch ( ... ) { } }\n"
                                 "}";
         ASSERT_EQUALS(expected, tokenizeAndStringify(code, true));
     }
@@ -5277,27 +5277,27 @@ private:
     void cpp0xtemplate5() { // #9154
         {
             const char *code = "struct s<x<u...>>;";
-            ASSERT_EQUALS("struct s < x < u . . . > > ;",
+            ASSERT_EQUALS("struct s < x < u ... > > ;",
                           tokenizeAndStringify(code));
         }
         {
             const char *code = "template <class f> using c = e<i<q<f,r>,b...>>;";
-            ASSERT_EQUALS("template < class f > using c = e < i < q < f , r > , b . . . > > ;",
+            ASSERT_EQUALS("template < class f > using c = e < i < q < f , r > , b ... > > ;",
                           tokenizeAndStringify(code));
         }
         {
             const char *code = "struct s<x<u...>> { };";
-            ASSERT_EQUALS("struct s < x < u . . . > > { } ;",
+            ASSERT_EQUALS("struct s < x < u ... > > { } ;",
                           tokenizeAndStringify(code));
         }
         {
             const char *code = "struct q : s<x<u...>> { };";
-            ASSERT_EQUALS("struct q : s < x < u . . . > > { } ;",
+            ASSERT_EQUALS("struct q : s < x < u ... > > { } ;",
                           tokenizeAndStringify(code));
         }
         {
             const char *code = "struct q : private s<x<u...>> { };";
-            ASSERT_EQUALS("struct q : private s < x < u . . . > > { } ;",
+            ASSERT_EQUALS("struct q : private s < x < u ... > > { } ;",
                           tokenizeAndStringify(code));
         }
     }
@@ -7115,11 +7115,11 @@ private:
 
     void simplifyCaseRange() {
         ASSERT_EQUALS("void f ( ) { switch ( x ) { case 1 : case 2 : case 3 : case 4 : ; } }", tokenizeAndStringify("void f() { switch(x) { case 1 ... 4: } }"));
-        ASSERT_EQUALS("void f ( ) { switch ( x ) { case 4 . . . 1 : ; } }", tokenizeAndStringify("void f() { switch(x) { case 4 ... 1: } }"));
+        ASSERT_EQUALS("void f ( ) { switch ( x ) { case 4 ... 1 : ; } }", tokenizeAndStringify("void f() { switch(x) { case 4 ... 1: } }"));
         tokenizeAndStringify("void f() { switch(x) { case 1 ... 1000000: } }"); // Do not run out of memory
 
         ASSERT_EQUALS("void f ( ) { switch ( x ) { case 'a' : case 'b' : case 'c' : ; } }", tokenizeAndStringify("void f() { switch(x) { case 'a' ... 'c': } }"));
-        ASSERT_EQUALS("void f ( ) { switch ( x ) { case 'c' . . . 'a' : ; } }", tokenizeAndStringify("void f() { switch(x) { case 'c' ... 'a': } }"));
+        ASSERT_EQUALS("void f ( ) { switch ( x ) { case 'c' ... 'a' : ; } }", tokenizeAndStringify("void f() { switch(x) { case 'c' ... 'a': } }"));
 
         ASSERT_EQUALS("void f ( ) { switch ( x ) { case '[' : case '\\\\' : case ']' : ; } }", tokenizeAndStringify("void f() { switch(x) { case '[' ... ']': } }"));
     }
@@ -7275,7 +7275,7 @@ private:
 
         ASSERT_EQUALS("abc.1:?1+bd.1:?+=", testAst("a =(b.c ? : 1) + 1 + (b.d ? : 1);"));
 
-        ASSERT_EQUALS("catch.(", testAst("try {} catch (...) {}"));
+        ASSERT_EQUALS("catch...(", testAst("try {} catch (...) {}"));
 
         ASSERT_EQUALS("FooBar(", testAst("void Foo(Bar&);"));
         ASSERT_EQUALS("FooBar(", testAst("void Foo(Bar& &);")); // Rvalue reference - simplified from && to & & by real tokenizer
@@ -7536,7 +7536,7 @@ private:
         ASSERT_EQUALS("1f2a&,(+", testAst("1+f(2,&a)"));
         ASSERT_EQUALS("fargv[(", testAst("int f(char argv[]);"));
         ASSERT_EQUALS("fchar(", testAst("extern unsigned f(const char *);"));
-        ASSERT_EQUALS("fcharformat*.,(", testAst("extern void f(const char *format, ...);"));
+        ASSERT_EQUALS("fcharformat*...,(", testAst("extern void f(const char *format, ...);"));
         ASSERT_EQUALS("for_each_commit_graftint((void,(", testAst("extern int for_each_commit_graft(int (*)(int*), void *);"));
         ASSERT_EQUALS("for;;(", testAst("for (;;) {}"));
         ASSERT_EQUALS("xsizeofvoid(=", testAst("x=sizeof(void*)"));

test/testvarid.cpp

@@ -2225,7 +2225,7 @@ private:
     }
 
     void varid_variadicFunc() {
-        ASSERT_EQUALS("1: int foo ( . . . ) ;\n", tokenize("int foo(...);"));
+        ASSERT_EQUALS("1: int foo ( ... ) ;\n", tokenize("int foo(...);"));
     }
 
     void varid_typename() {