make ellipsis ... a single token (#2143)

* make ellipsis ... a single token

Using cppcheck -E to preprocess code with ellipsis produces output that
can't be compiled because ... is split into 3 tokens.

* try to fix addon

externals/simplecpp/simplecpp.cpp

@@ -745,10 +745,14 @@ void simplecpp::TokenList::combineOperators()
         }
 
         if (tok->op == '.') {
-            if (tok->previous && tok->previous->op == '.')
+            // ellipsis ...
-                continue;
+            if (tok->next && tok->next->op == '.' && tok->next->location.col == (tok->location.col + 1) &&
-            if (tok->next && tok->next->op == '.')
+                tok->next->next && tok->next->next->op == '.' && tok->next->next->location.col == (tok->location.col + 2)) {
+                tok->setstr("...");
+                deleteToken(tok->next);
+                deleteToken(tok->next);
                 continue;
+            }
             // float literals..
             if (tok->previous && tok->previous->number) {
                 tok->setstr(tok->previous->str() + '.');
@@ -1376,14 +1380,12 @@ namespace simplecpp {
                 args.clear();
                 const Token *argtok = nameTokDef->next->next;
                 while (sameline(nametoken, argtok) && argtok->op != ')') {
-                    if (argtok->op == '.' &&
+                    if (argtok->str() == "..." &&
-                        argtok->next && argtok->next->op == '.' &&
+                        argtok->next && argtok->next->op == ')') {
-                        argtok->next->next && argtok->next->next->op == '.' &&
-                        argtok->next->next->next && argtok->next->next->next->op == ')') {
                         variadic = true;
                         if (!argtok->previous->name)
                             args.push_back("__VA_ARGS__");
-                        argtok = argtok->next->next->next; // goto ')'
+                        argtok = argtok->next; // goto ')'
                         break;
                     }
                     if (argtok->op != ',')

lib/checkother.cpp

@@ -1229,7 +1229,7 @@ void CheckOther::checkPassByReference()
         if (!var || !var->isArgument() || !var->isClass() || var->isPointer() || var->isArray() || var->isReference() || var->isEnumType())
             continue;
 
-        if (var->scope() && var->scope()->function->arg->link()->strAt(-1) == ".")
+        if (var->scope() && var->scope()->function->arg->link()->strAt(-1) == "...")
             continue; // references could not be used as va_start parameters (#5824)
 
         bool inconclusive = false;
@@ -2447,7 +2447,7 @@ void CheckOther::checkVarFuncNullUB()
                     if (f && f->argCount() <= argnr) {
                         const Token *tok2 = f->argDef;
                         tok2 = tok2 ? tok2->link() : nullptr; // goto ')'
-                        if (tok2 && Token::simpleMatch(tok2->tokAt(-3), ". . ."))
+                        if (tok2 && Token::simpleMatch(tok2->tokAt(-1), "..."))
                             varFuncNullUBError(tok);
                     }
                 }

lib/symboldatabase.cpp

@@ -1776,7 +1776,7 @@ void Variable::evaluate(const Settings* settings)
         std::string strtype = mTypeStartToken->str();
         for (const Token *typeToken = mTypeStartToken; Token::Match(typeToken, "%type% :: %type%"); typeToken = typeToken->tokAt(2))
             strtype += "::" + typeToken->strAt(2);
-        setFlag(fIsClass, !lib->podtype(strtype) && !mTypeStartToken->isStandardType() && !isEnumType() && !isPointer() && !isReference());
+        setFlag(fIsClass, !lib->podtype(strtype) && !mTypeStartToken->isStandardType() && !isEnumType() && !isPointer() && !isReference() && strtype != "...");
         setFlag(fIsStlType, Token::simpleMatch(mTypeStartToken, "std ::"));
         setFlag(fIsStlString, isStlType() && (Token::Match(mTypeStartToken->tokAt(2), "string|wstring|u16string|u32string !!::") || (Token::simpleMatch(mTypeStartToken->tokAt(2), "basic_string <") && !Token::simpleMatch(mTypeStartToken->linkAt(3), "> ::"))));
         setFlag(fIsSmartPointer, lib->isSmartPointer(mTypeStartToken));

lib/templatesimplifier.cpp

@@ -459,7 +459,7 @@ unsigned int TemplateSimplifier::templateParameters(const Token *tok)
             if ((tok->previous()->isName() && !Token::Match(tok->tokAt(-2), "<|,|::")) ||
                 (!tok->previous()->isName() && tok->strAt(-1) != ">"))
                 return 0; // syntax error
-            tok = tok->tokAt(3);
+            tok = tok->next();
             if (!tok)
                 return 0;
             if (tok->str() == ">") {
@@ -726,7 +726,7 @@ bool TemplateSimplifier::getTemplateDeclarations()
         if (!tok->tokAt(2))
             syntaxError(tok->next());
         if (tok->strAt(2)=="typename" &&
-            !Token::Match(tok->tokAt(3), "%name%|.|,|=|>"))
+            !Token::Match(tok->tokAt(3), "%name%|...|,|=|>"))
             syntaxError(tok->next());
         codeWithTemplates = true;
         const Token * const parmEnd = tok1->next()->findClosingBracket();
@@ -1630,10 +1630,8 @@ void TemplateSimplifier::expandTemplate(
                 for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
                      typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>"));
                      typetok = typetok->next()) {
-                    if (Token::simpleMatch(typetok, ". . .")) {
+                    if (Token::simpleMatch(typetok, "..."))
-                        typetok = typetok->tokAt(2);
                         continue;
-                    }
                     if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next())))
                         ++typeindentlevel;
                     else if (typeindentlevel > 0 && typetok->str() == ">")
@@ -1856,9 +1854,7 @@ void TemplateSimplifier::expandTemplate(
                             for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
                                  typetok && (typeindentlevel>0 || !Token::Match(typetok, ",|>"));
                                  typetok = typetok->next()) {
-                                if (Token::simpleMatch(typetok, ". . .")) {
+                                if (!Token::simpleMatch(typetok, "...")) {
-                                    typetok = typetok->tokAt(2);
-                                } else {
                                     if (Token::Match(typetok, "%name% <") && (typetok->strAt(2) == ">" || templateParameters(typetok->next())))
                                         ++typeindentlevel;
                                     else if (typeindentlevel > 0 && typetok->str() == ">")
@@ -1960,10 +1956,8 @@ void TemplateSimplifier::expandTemplate(
                     for (const Token *typetok = mTypesUsedInTemplateInstantiation[itype].token();
                          typetok && (typeindentlevel > 0 || !Token::Match(typetok, ",|>"));
                          typetok = typetok->next()) {
-                        if (Token::simpleMatch(typetok, ". . .")) {
+                        if (Token::simpleMatch(typetok, "..."))
-                            typetok = typetok->tokAt(2);
                             continue;
-                        }
                         if (Token::Match(typetok, "%name% <") &&
                             (typetok->strAt(2) == ">" || templateParameters(typetok->next()))) {
                             brackets1.push(typetok->next());

lib/token.cpp

@@ -128,6 +128,8 @@ void Token::update_property_info()
             tokType(eIncDecOp);
         else if (mStr.size() == 1 && (mStr.find_first_of("{}") != std::string::npos || (mLink && mStr.find_first_of("<>") != std::string::npos)))
             tokType(eBracket);
+        else if (mStr == "...")
+            tokType(eEllipsis);
         else
             tokType(eOther);
     } else {

lib/token.h

@@ -176,6 +176,7 @@ public:
         eArithmeticalOp, eComparisonOp, eAssignmentOp, eLogicalOp, eBitOp, eIncDecOp, eExtendedOp, // Operators: Arithmetical, Comparison, Assignment, Logical, Bitwise, ++/--, Extended
         eBracket, // {, }, <, >: < and > only if link() is set. Otherwise they are comparison operators.
         eLambda, // A function without a name
+        eEllipsis, // "..."
         eOther,
         eNone
     };

lib/tokenize.cpp

@@ -2799,13 +2799,12 @@ void Tokenizer::simplifyCaseRange()
     for (Token* tok = list.front(); tok; tok = tok->next()) {
         if (Token::Match(tok, "case %num% ... %num% :")) {
             const MathLib::bigint start = MathLib::toLongNumber(tok->strAt(1));
-            MathLib::bigint end = MathLib::toLongNumber(tok->strAt(5));
+            MathLib::bigint end = MathLib::toLongNumber(tok->strAt(3));
             end = std::min(start + 50, end); // Simplify it 50 times at maximum
             if (start < end) {
                 tok = tok->tokAt(2);
                 tok->str(":");
-                tok->deleteNext();
+                tok->insertToken("case");
-                tok->next()->str("case");
                 for (MathLib::bigint i = end-1; i > start; i--) {
                     tok->insertToken(":");
                     tok->insertToken(MathLib::toString(i));
@@ -2814,12 +2813,11 @@ void Tokenizer::simplifyCaseRange()
             }
         } else if (Token::Match(tok, "case %char% ... %char% :")) {
             const char start = tok->strAt(1)[1];
-            const char end = tok->strAt(5)[1];
+            const char end = tok->strAt(3)[1];
             if (start < end) {
                 tok = tok->tokAt(2);
                 tok->str(":");
-                tok->deleteNext();
+                tok->insertToken("case");
-                tok->next()->str("case");
                 for (char i = end - 1; i > start; i--) {
                     tok->insertToken(":");
                     if (i == '\\') {
@@ -3976,7 +3974,7 @@ void Tokenizer::createLinks2()
             }
             // if > is followed by [ .. "new a<b>[" is expected
             // unless this is from varidiac expansion
-            if (token->strAt(1) == "[" && !Token::simpleMatch(token->tokAt(-3), ". . . >")) {
+            if (token->strAt(1) == "[" && !Token::simpleMatch(token->tokAt(-1), "... >")) {
                 Token *prev = type.top()->previous();
                 while (prev && Token::Match(prev->previous(), ":: %name%"))
                     prev = prev->tokAt(-2);
@@ -4089,7 +4087,8 @@ bool Tokenizer::simplifySizeof()
             continue;
 
         if (Token::simpleMatch(tok->next(), "...")) {
-            tok->deleteNext(3);
+            //tok->deleteNext(3);
+            tok->deleteNext();
         }
 
         // sizeof('x')
@@ -4993,7 +4992,7 @@ void Tokenizer::simplifyHeaders()
                     const Token *tok2 = tok->tokAt(3);
                     while (Token::Match(tok2, "%name% %name% [,=>]") || Token::Match(tok2, "typename ... %name% [,>]")) {
                         if (Token::simpleMatch(tok2, "typename ..."))
-                            tok2 = tok2->tokAt(5);
+                            tok2 = tok2->tokAt(3);
                         else
                             tok2 = tok2->tokAt(2);
                         if (Token::Match(tok2, "= %name% [,>]"))

lib/tokenlist.cpp

@@ -689,7 +689,7 @@ static void compileTerm(Token *&tok, AST_state& state)
             if (Token::Match(tok, "%name% <") && tok->linkAt(1))
                 tok = tok->linkAt(1);
             else if (Token::Match(tok, "%name% ..."))
-                tok = tok->tokAt(3);
+                tok = tok->next();
             tok = tok->next();
             if (Token::Match(tok, "%str%")) {
                 while (Token::Match(tok, "%name%|%str%"))
@@ -767,6 +767,9 @@ static void compilePrecedence2(Token *&tok, AST_state& state)
     while (tok) {
         if (tok->tokType() == Token::eIncDecOp && !isPrefixUnary(tok, state.cpp)) {
             compileUnaryOp(tok, state, compileScope);
+        } else if (tok->str() == "...") {
+            state.op.push(tok);
+            break;
         } else if (tok->str() == "." && tok->strAt(1) != "*") {
             if (tok->strAt(1) == ".") {
                 state.op.push(tok);

test/testtokenize.cpp

@@ -7275,7 +7275,7 @@ private:
 
         ASSERT_EQUALS("abc.1:?1+bd.1:?+=", testAst("a =(b.c ? : 1) + 1 + (b.d ? : 1);"));
 
-        ASSERT_EQUALS("catch.(", testAst("try {} catch (...) {}"));
+        ASSERT_EQUALS("catch...(", testAst("try {} catch (...) {}"));
 
         ASSERT_EQUALS("FooBar(", testAst("void Foo(Bar&);"));
         ASSERT_EQUALS("FooBar(", testAst("void Foo(Bar& &);")); // Rvalue reference - simplified from && to & & by real tokenizer
@@ -7536,7 +7536,7 @@ private:
         ASSERT_EQUALS("1f2a&,(+", testAst("1+f(2,&a)"));
         ASSERT_EQUALS("fargv[(", testAst("int f(char argv[]);"));
         ASSERT_EQUALS("fchar(", testAst("extern unsigned f(const char *);"));
-        ASSERT_EQUALS("fcharformat*.,(", testAst("extern void f(const char *format, ...);"));
+        ASSERT_EQUALS("fcharformat*...,(", testAst("extern void f(const char *format, ...);"));
         ASSERT_EQUALS("for_each_commit_graftint((void,(", testAst("extern int for_each_commit_graft(int (*)(int*), void *);"));
         ASSERT_EQUALS("for;;(", testAst("for (;;) {}"));
         ASSERT_EQUALS("xsizeofvoid(=", testAst("x=sizeof(void*)"));