#11134 Fix broken AST with (designated) initializers (#4550)

* Make control flow a bit easier, and more similar to previous code

Made similar to around line 790

* In a cpp11init, always parse only the corresponding } (#11134)

- _always_, because in some cases this was omitted (around line 790) or too strict (around line 860)
- _only_, and not following tokens which happen to be } as well (around line 1030)

* Fix unit tests: AST was incorrect, now is fixed

auto var{ {{},{}}, {} };

Old AST:
```
{
|-var
`-{
  `-,
    |-,
    | |-{
    | `-{
    `-{
```
New AST:
```
{
|-var
`-,
  |-{
  | `-,
  | | |-{
  | | `-{
  `-{
```
Compare the same example, but with `X{}` instead of just `{}`:
`auto var{ a{b{},c{}}, d{} };`
```
{
|-var
`-,
  |-{
  | |-a
  | `-,
  | | |-{
  | | | `-b
  | | `-{
  | | | `-c
  `-{
    `-d
```
This structure is similar to that of the new AST, not the old AST

* Fix unit tests: another AST was incorrect, now is fixed

Code: `auto var{{1,a::b{2,3}}, {4,a::b{5,6}}};`

Old AST:
```
{
|-var
`-{
  `-,
    |-,
    | |-1 'signed int'
    | `-{
    | | |-::
    | | | |-a
    | | | `-b
    | | `-,
    | | | |-2 'signed int'
    | | | `-3 'signed int'
    `-{
      `-,
        |-4 'signed int'
        `-{
          |-::
          | |-a
          | `-b
          `-,
            |-5 'signed int'
            `-6 'signed int'
```
New AST:
```
{
|-var
`-,
  |-{
  | `-,
  | | |-1 'signed int'
  | | `-{
  | | | |-::
  | | | | |-a
  | | | | `-b
  | | | `-,
  | | | | |-2 'signed int'
  | | | | `-3 'signed int'
  `-{
    `-,
      |-4 'signed int'
      `-{
        |-::
        | |-a
        | `-b
        `-,
          |-5 'signed int'
          `-6 'signed int'
```

* Fix unit tests: missing ; after class, resulting in incorrectly being marked as cpp11init

Because of the missing `;` after the class declaration, it was marked as a cpp11init block.
Which it isn't, and which now throws an exception

* Fix cpp11init to let unit tests pass again

The following unit tests failed on the newly introduced throws, because the code for these tests incorrectly marked some tokens as cpp11init:
TestVarID::varid_cpp11initialization
TestTokenizer::checkRefQualifiers

* Fix typo

* Improve check for void trailing return type

Observation: the only function body _not_ containing a semicolon, is a void function: something like
   auto make_zero(ini& i) -> void {
     while(--i > 0) {}
   }
Non-void function? Then it must return a value, and thus contain a semicolon, which is checked for a few lines later.

* Fix cpp11init with templated trailing return type

In the following example, vector was marked as cpp11init due to the mismatch of `%any% {`
auto f() -> std::vector<int> { return {}; }

I made the assumption that whenever "%any% {" matches, endtok must be set too.
If this assumtion doesn't hold (so "%any% {" matches, but endtok == nullptr), then the for-loop would search all the way to the end of stream. Which I guess was not the intention.

* Remove comments

Co-authored-by: Gerbo Engels <gerbo.engels@ortec-finance.com>

lib/token.cpp

@@ -2482,7 +2482,7 @@ Token* findLambdaEndScope(Token* tok)
     if (!Token::simpleMatch(tok, ")"))
         return nullptr;
     tok = tok->next();
-    while (Token::Match(tok, "mutable|constexpr|constval|noexcept|.")) {
+    while (Token::Match(tok, "mutable|constexpr|consteval|noexcept|.")) {
         if (Token::simpleMatch(tok, "noexcept ("))
             tok = tok->linkAt(1);
         if (Token::simpleMatch(tok, ".")) {

lib/tokenlist.cpp

@@ -652,9 +652,11 @@ static bool iscpp11init_impl(const Token * const tok)
         return false;
     if (Token::Match(nameToken, "else|try|do|const|constexpr|override|volatile|&|&&"))
         return false;
+    if (Token::simpleMatch(nameToken->previous(), ". void {") && nameToken->previous()->originalName() == "->")
+        return false; // trailing return type. The only function body that can contain no semicolon is a void function.
     if (Token::simpleMatch(nameToken->previous(), "namespace"))
         return false;
-    if (Token::Match(nameToken, "%any% {") && !Token::Match(nameToken, "return|:")) {
+    if (endtok != nullptr && !Token::Match(nameToken, "return|:")) {
         // If there is semicolon between {..} this is not a initlist
         for (const Token *tok2 = nameToken->next(); tok2 != endtok; tok2 = tok2->next()) {
             if (tok2->str() == ";")
@@ -668,7 +670,7 @@ static bool iscpp11init_impl(const Token * const tok)
     if (!Token::simpleMatch(endtok, "} ;"))
         return true;
     const Token *prev = nameToken;
-    while (Token::Match(prev, "%name%|::|:|<|>")) {
+    while (Token::Match(prev, "%name%|::|:|<|>|,")) {
         if (Token::Match(prev, "class|struct"))
             return false;
 
@@ -785,10 +787,15 @@ static void compileTerm(Token *&tok, AST_state& state)
                 tok = tok->link()->next();
 
             if (Token::Match(tok, "{ . %name% =|{")) {
+                const Token* end = tok->link();
                 const int inArrayAssignment = state.inArrayAssignment;
                 state.inArrayAssignment = 1;
                 compileBinOp(tok, state, compileExpression);
                 state.inArrayAssignment = inArrayAssignment;
+                if (tok == end)
+                    tok = tok->next();
+                else
+                    throw InternalError(tok, "Syntax error. Unexpected tokens in designated initializer.", InternalError::AST);
             } else if (Token::simpleMatch(tok, "{ }")) {
                 tok->astOperand1(state.op.top());
                 state.op.pop();
@@ -834,26 +841,26 @@ static void compileTerm(Token *&tok, AST_state& state)
         if (Token::simpleMatch(tok->link(),"} [")) {
             tok = tok->next();
         } else if (state.cpp && iscpp11init(tok)) {
-            if (state.op.empty() || Token::Match(tok->previous(), "[{,]") || Token::Match(tok->tokAt(-2), "%name% (")) {
-                if (Token::Match(tok, "{ !!}")) {
             Token *const end = tok->link();
+            if (state.op.empty() || Token::Match(tok->previous(), "[{,]") || Token::Match(tok->tokAt(-2), "%name% (")) {
                 if (Token::Match(tok, "{ . %name% =|{")) {
                     const int inArrayAssignment = state.inArrayAssignment;
                     state.inArrayAssignment = 1;
                     compileBinOp(tok, state, compileExpression);
                     state.inArrayAssignment = inArrayAssignment;
+                } else if (Token::simpleMatch(tok, "{ }")) {
+                    state.op.push(tok);
+                    tok = tok->next();
                 } else {
                     compileUnaryOp(tok, state, compileExpression);
-                    }
+                    if (precedes(tok,end)) // typically for something like `MACRO(x, { if (c) { ... } })`, where end is the last curly, and tok is the open curly for the if
-                    if (precedes(tok,end))
                         tok = end;
-                } else {
-                    state.op.push(tok);
-                    tok = tok->tokAt(2);
                 }
             } else
                 compileBinOp(tok, state, compileExpression);
-            if (Token::Match(tok, "} ,|:|)"))
+            if (tok != end)
+                throw InternalError(tok, "Syntax error. Unexpected tokens in initializer.", InternalError::AST);
+            if (tok->next())
                 tok = tok->next();
         } else if (state.cpp && Token::Match(tok->tokAt(-2), "%name% ( {") && !Token::findsimplematch(tok, ";", tok->link())) {
             if (Token::simpleMatch(tok, "{ }"))
@@ -966,7 +973,7 @@ static void compilePrecedence2(Token *&tok, AST_state& state)
                 if (Token::simpleMatch(squareBracket->link(), "] (")) {
                     Token* const roundBracket = squareBracket->link()->next();
                     Token* curlyBracket = roundBracket->link()->next();
-                    while (Token::Match(curlyBracket, "mutable|const|constexpr"))
+                    while (Token::Match(curlyBracket, "mutable|const|constexpr|consteval"))
                         curlyBracket = curlyBracket->next();
                     if (Token::simpleMatch(curlyBracket, "noexcept ("))
                         curlyBracket = curlyBracket->linkAt(1)->next();
@@ -1025,12 +1032,20 @@ static void compilePrecedence2(Token *&tok, AST_state& state)
             cast->astOperand1(tok1);
             tok = tok1->link()->next();
         } else if (state.cpp && tok->str() == "{" && iscpp11init(tok)) {
+            const Token* end = tok->link();
             if (Token::simpleMatch(tok, "{ }"))
-                compileUnaryOp(tok, state, compileExpression);
+            {
-            else
+                compileUnaryOp(tok, state, nullptr);
-                compileBinOp(tok, state, compileExpression);
-            while (Token::simpleMatch(tok, "}"))
                 tok = tok->next();
+            }
+            else
+            {
+                compileBinOp(tok, state, compileExpression);
+            }
+            if (tok == end)
+                tok = end->next();
+            else
+                throw InternalError(tok, "Syntax error. Unexpected tokens in initializer.", InternalError::AST);
         } else break;
     }
 }

test/testsimplifytemplate.cpp

@@ -4854,14 +4854,14 @@ private:
         //         "no constructor" false positives
         const char code[] = "class Fred {\n"
                             "    template<class T> explicit Fred(T t) { }\n"
-                            "}";
+                            "};";
-        ASSERT_EQUALS("class Fred { template < class T > explicit Fred ( T t ) { } }", tok(code));
+        ASSERT_EQUALS("class Fred { template < class T > explicit Fred ( T t ) { } } ;", tok(code));
 
         // #3532
         const char code2[] = "class Fred {\n"
                              "    template<class T> Fred(T t) { }\n"
-                             "}";
+                             "};";
-        ASSERT_EQUALS("class Fred { template < class T > Fred ( T t ) { } }", tok(code2));
+        ASSERT_EQUALS("class Fred { template < class T > Fred ( T t ) { } } ;", tok(code2));
     }
 
     void syntax_error_templates_1() {

test/testsimplifytypedef.cpp

@@ -1427,14 +1427,14 @@ private:
                             "typedef const Class & Const_Reference;\n"
                             "void some_method (Const_Reference x) const {}\n"
                             "void another_method (Const_Reference x) const {}\n"
-                            "}";
+                            "};";
 
         // The expected result..
         const char expected[] = "class Class2 { "
                                 ""
                                 "void some_method ( const Class & x ) const { } "
                                 "void another_method ( const Class & x ) const { } "
-                                "}";
+                                "} ;";
         ASSERT_EQUALS(expected, tok(code));
     }
 

test/testtokenize.cpp

@@ -461,6 +461,8 @@ private:
         TEST_CASE(simplifyIfSwitchForInit5);
 
         TEST_CASE(cpp20_default_bitfield_initializer);
+
+        TEST_CASE(cpp11init);
     }
 
 #define tokenizeAndStringify(...) tokenizeAndStringify_(__FILE__, __LINE__, __VA_ARGS__)
@@ -6228,8 +6230,8 @@ private:
         ASSERT_EQUALS("abR{{,P(,((", testAst("a(b(R{},{},P()));"));
         ASSERT_EQUALS("f1{2{,3{,{x,(", testAst("f({{1},{2},{3}},x);"));
         ASSERT_EQUALS("a1{ b2{", testAst("auto a{1}; auto b{2};"));
-        ASSERT_EQUALS("var1ab::23,{,4ab::56,{,{,{{", testAst("auto var{{1,a::b{2,3}}, {4,a::b{5,6}}};"));
+        ASSERT_EQUALS("var1ab::23,{,{4ab::56,{,{,{", testAst("auto var{{1,a::b{2,3}}, {4,a::b{5,6}}};"));
-        ASSERT_EQUALS("var{{,{,{{", testAst("auto var{ {{},{}}, {} };"));
+        ASSERT_EQUALS("var{{,{{,{", testAst("auto var{ {{},{}}, {} };"));
         ASSERT_EQUALS("fXYabcfalse==CD:?,{,{(", testAst("f({X, {Y, abc == false ? C : D}});"));
         ASSERT_EQUALS("stdvector::p0[{(return", testAst("return std::vector<int>({ p[0] });"));
 
@@ -6465,7 +6467,7 @@ private:
                               "}"));
         ASSERT_EQUALS("{(=[{return ab=",
                       testAst("return {\n"
-                              "  [=]() mutable -> int {\n"
+                              "  [=]() mutable consteval -> int {\n"
                               "    a=b;\n"
                               "  }\n"
                               "}"));
@@ -6690,12 +6692,31 @@ private:
                                              "}; "
                                              "struct poc p = { .port[0] = {.d = 3} };"));
 
-        // op op
-        ASSERT_THROW_EQUALS(tokenizeAndStringify("void f() { dostuff (x==>y); }"), InternalError, "syntax error: == >");
-
         // Ticket #9664
         ASSERT_NO_THROW(tokenizeAndStringify("S s = { .x { 2 }, .y[0] { 3 } };"));
 
+        // Ticket #11134
+        ASSERT_NO_THROW(tokenizeAndStringify("struct my_struct { int x; }; "
+                                             "std::string s; "
+                                             "func(my_struct{ .x=42 }, s.size());"));
+        ASSERT_NO_THROW(tokenizeAndStringify("struct my_struct { int x; int y; }; "
+                                             "std::string s; "
+                                             "func(my_struct{ .x{42}, .y=3 }, s.size());"));
+        ASSERT_NO_THROW(tokenizeAndStringify("struct my_struct { int x; int y; }; "
+                                             "std::string s; "
+                                             "func(my_struct{ .x=42, .y{3} }, s.size());"));
+        ASSERT_NO_THROW(tokenizeAndStringify("struct my_struct { int x; }; "
+                                             "void h() { "
+                                             "  for (my_struct ms : { my_struct{ .x=5 } }) {} "
+                                             "}"));
+        ASSERT_NO_THROW(tokenizeAndStringify("struct my_struct { int x; int y; }; "
+                                             "void h() { "
+                                             "  for (my_struct ms : { my_struct{ .x=5, .y{42} } }) {} "
+                                             "}"));
+
+        // op op
+        ASSERT_THROW_EQUALS(tokenizeAndStringify("void f() { dostuff (x==>y); }"), InternalError, "syntax error: == >");
+
         ASSERT_THROW_EQUALS(tokenizeAndStringify("void f() { assert(a==()); }"), InternalError, "syntax error: ==()");
         ASSERT_THROW_EQUALS(tokenizeAndStringify("void f() { assert(a+()); }"), InternalError, "syntax error: +()");
 
@@ -7382,6 +7403,51 @@ private:
         settings.standards.cpp = Standards::CPP17;
         ASSERT_THROW(tokenizeAndStringify(code, settings), InternalError);
     }
+
+    void cpp11init() {
+        #define testIsCpp11init(...) testIsCpp11init_(__FILE__, __LINE__, __VA_ARGS__)
+        auto testIsCpp11init_ = [this](const char* file, int line, const char* code, const char* find, TokenImpl::Cpp11init expected) {
+            Settings settings;
+            Tokenizer tokenizer(&settings, this);
+            std::istringstream istr(code);
+            ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+
+            const Token* tok = Token::findsimplematch(tokenizer.tokens(), find, strlen(find));
+            ASSERT_LOC(tok, file, line);
+            ASSERT_LOC(tok->isCpp11init() == expected, file, line);
+        };
+
+        testIsCpp11init("class X : public A<int>, C::D {};",
+                        "D {",
+                        TokenImpl::Cpp11init::NOINIT);
+
+        testIsCpp11init("auto f() -> void {}",
+                        "void {",
+                        TokenImpl::Cpp11init::NOINIT);
+        testIsCpp11init("auto f() & -> void {}",
+                        "void {",
+                        TokenImpl::Cpp11init::NOINIT);
+        testIsCpp11init("auto f() const noexcept(false) -> void {}",
+                        "void {",
+                        TokenImpl::Cpp11init::NOINIT);
+        testIsCpp11init("auto f() -> std::vector<int> { return {}; }",
+                        "{ return",
+                        TokenImpl::Cpp11init::NOINIT);
+        testIsCpp11init("auto f() -> std::vector<int> { return {}; }",
+                        "vector",
+                        TokenImpl::Cpp11init::NOINIT);
+        testIsCpp11init("auto f() -> std::vector<int> { return {}; }",
+                        "std ::",
+                        TokenImpl::Cpp11init::NOINIT);
+
+        testIsCpp11init("class X{};",
+                        "{ }",
+                        TokenImpl::Cpp11init::NOINIT);
+        testIsCpp11init("class X{}", // forgotten ; so not properly recognized as a class
+                        "{ }",
+                        TokenImpl::Cpp11init::CPP11INIT);
+        #undef testIsCpp11init
+    }
 };
 
 REGISTER_TEST(TestTokenizer)

test/testvarid.cpp

@@ -790,14 +790,14 @@ private:
                              "        : ExecutionPath(c, id)\n"
                              "    {\n"
                              "    }\n"
-                             "}\n";
+                             "};\n";
         const char expected3[] = "1: class Nullpointer : public ExecutionPath\n"
                                  "2: {\n"
                                  "3: Nullpointer ( Check * c@1 , const unsigned int id@2 , const std :: string & name@3 )\n"
                                  "4: : ExecutionPath ( c@1 , id@2 )\n"
                                  "5: {\n"
                                  "6: }\n"
-                                 "7: }\n";
+                                 "7: } ;\n";
         ASSERT_EQUALS(expected3, tokenize(code3));
     }