AST: Rewrite using standard approach

lib/tokenlist.cpp

@@ -357,82 +357,246 @@ bool TokenList::createTokens(std::istream &code, const std::string& file0)
 
 //---------------------------------------------------------------------------
 
-void TokenList::createAst() const
+static void compileUnaryOp(Token *&tok, void (*f)(Token *&, std::stack<Token*> &), std::stack<Token*> &op)
 {
-    // operators that must be ordered according to C-precedence
+    Token *unaryop = tok;
-    const char * const operators[] = {
-        " :: ",
-        " ++ -- . ",
-        "> ++ -- + - ! ~ * & sizeof ",  // prefix unary operators, from right to left
-        " * / % ",
-        " + - ",
-        " << >> ",
-        " < <= > >= ",
-        " == != ",
-        " & ",
-        " ^ ",
-        " | ",
-        " && ",
-        " || ",
-        " = ? : ",
-        " throw ",
-        " , "
-        " [ "
-    };
-
-    // No tokens => bail out
-    if (!_front)
-        return;
-
-    for (unsigned int i = 0; i < sizeof(operators) / sizeof(*operators); ++i) {
-        // TODO: extract operators to std::set - that should be faster
-        if (*operators[i] == '>') {  // Unary operators, parse from right to left
-            const std::string op(1+operators[i]);
-            Token *tok = _front;
-            while (tok->next())
     tok = tok->next();
-            for (; tok; tok = tok->previous()) {
+    f(tok,op);
-                if (tok->isOp() &&
-                    (!tok->previous() ||
-                     tok->previous()->isOp() ||
-                     tok->previous()->str() == "," ||
-                     tok->previous()->type() == Token::eOther) &&
-                    op.find(" "+tok->str()+" ")!=std::string::npos) {
-                    tok->astOperand1(tok->next());
-                }
-            }
-        } else {  // parse from left to right
-            const std::string op(operators[i]);
-            for (Token *tok = _front; tok; tok = tok->next()) {
-                if (tok->astOperand1()==NULL && op.find(" "+tok->str()+" ")!=std::string::npos) {
-                    // Don't create AST for "..."
-                    if (tok->str() == "." && (tok->previous()->str() == "." || tok->next()->str() == "."))
-                        continue;
 
-                    if (Token::Match(tok, "* [)]]"))
+    if (!op.empty()) {
-                        continue;
+        unaryop->astOperand1(op.top());
+        op.pop();
+    }
+    op.push(unaryop);
+}
 
-                    if (tok->type() != Token::eIncDecOp) {
+static void compileBinOp(Token *&tok, void (*f)(Token *&, std::stack<Token*> &), std::stack<Token*> &op)
-                        tok->astOperand1(tok->previous());
+{
-                        tok->astOperand2(tok->next());
+    Token *binop = tok;
-                    } else if (tok->previous() && !tok->previous()->isOp()) {
+    tok = tok->next();
+    f(tok,op);
+
+    // TODO: Should we check if op is empty.
+    // * Is it better to add assertion that it isn't?
+    // * Write debug warning if it's empty?
+    if (!op.empty()) {
+        binop->astOperand2(op.top());
+        op.pop();
+    }
+    if (!op.empty()) {
+        binop->astOperand1(op.top());
+        op.pop();
+    }
+    op.push(binop);
+}
+
+static void compileExpression(Token *&tok, std::stack<Token*> &op);
+
+static void compileTerm(Token *& tok, std::stack<Token*> &op)
+{
+    if (tok->isLiteral()) {
+        op.push(tok);
+        tok = tok->next();
+    } else if (tok->isName()) {
+        if (Token::Match(tok->next(), "++|--")) {  // post increment / decrement
+            tok = tok->next();
             tok->astOperand1(tok->previous());
+            op.push(tok);
+            tok = tok->next();
+        } else if (!Token::Match(tok->next(), "(|[")) {
+            op.push(tok);
+            tok = tok->next();
+        } else {
+            Token *name = tok;
+            tok = tok->tokAt(2);
+            if (Token::Match(tok, ")|]")) {
+                name->next()->astOperand1(name);
+                tok = tok->next();
+            } else {
+                compileExpression(tok,op);
+                tok = tok->next(); // skip ')' or ']'
+                if (!op.empty()) {
+                    name->next()->astOperand2(op.top());
+                    op.pop();
                 }
+                name->next()->astOperand1(name);
             }
+            op.push(name->next());
         }
+    } else if (Token::Match(tok, "+|-|~|*|&|!")) {
+        compileUnaryOp(tok, compileExpression, op);
+    } else if (Token::Match(tok, "++|--")) {
+        if (!op.empty() && op.top()->isOp()) {
+            // post increment/decrement
+            tok->astOperand1(op.top());
+            op.pop();
+            op.push(tok);
+            tok = tok->next();
+        } else {
+            // pre increment/decrement
+            compileUnaryOp(tok, compileExpression, op);
         }
+    } else if (tok->str() == "(") {
+        // Parenthesized sub-expression
+        tok = tok->next();
+        compileExpression(tok,op);
+        tok = tok->next();
     }
+}
 
-    // function calls..
+static void compileScope(Token *&tok, std::stack<Token*> &op)
-    for (Token *tok = _front; tok; tok = tok->next()) {
+{
-        if (Token::Match(tok, "%var% ("))
+    compileTerm(tok,op);
-            tok->astFunctionCall();
+    while (tok) {
+        if (tok->str() == "::") {
+            compileBinOp(tok, compileTerm, op);
+        } else break;
     }
+}
 
-    // parentheses..
+static void compileDot(Token *&tok, std::stack<Token*> &op)
-    for (Token *tok = _front; tok; tok = tok->next()) {
+{
-        if (Token::Match(tok, "(|)|]")) {
+    compileScope(tok,op);
-            tok->astHandleParentheses();
+    while (tok) {
+        if (tok->str() == ".") {
+            compileBinOp(tok, compileScope, op);
+        } else break;
+    }
+}
+
+static void compileMulDiv(Token *&tok, std::stack<Token*> &op)
+{
+    compileDot(tok,op);
+    while (tok) {
+        if (Token::Match(tok, "[*/%]")) {
+            if (Token::Match(tok, "* [,)]"))
+                break;
+            compileBinOp(tok, compileDot, op);
+        } else break;
+    }
+}
+
+static void compileAddSub(Token *&tok, std::stack<Token*> &op)
+{
+    compileMulDiv(tok,op);
+    while (tok) {
+        if (Token::Match(tok, "+|-")) {
+            compileBinOp(tok, compileMulDiv, op);
+        } else break;
+    }
+}
+
+static void compileShift(Token *&tok, std::stack<Token*> &op)
+{
+    compileAddSub(tok,op);
+    while (tok) {
+        if (Token::Match(tok, "<<|>>")) {
+            compileBinOp(tok, compileAddSub, op);
+        } else break;
+    }
+}
+
+static void compileRelComp(Token *&tok, std::stack<Token*> &op)
+{
+    compileShift(tok,op);
+    while (tok) {
+        if (Token::Match(tok, "<|<=|>=|>")) {
+            compileBinOp(tok, compileShift, op);
+        } else break;
+    }
+}
+
+static void compileEqComp(Token *&tok, std::stack<Token*> &op)
+{
+    compileRelComp(tok,op);
+    while (tok) {
+        if (Token::Match(tok, "==|!=")) {
+            compileBinOp(tok, compileRelComp, op);
+        } else break;
+    }
+}
+
+static void compileAnd(Token *&tok, std::stack<Token*> &op)
+{
+    compileEqComp(tok,op);
+    while (tok) {
+        if (tok->str() == "&") {
+            compileBinOp(tok, compileEqComp, op);
+        } else break;
+    }
+}
+
+static void compileXor(Token *&tok, std::stack<Token*> &op)
+{
+    compileAnd(tok,op);
+    while (tok) {
+        if (tok->str() == "^") {
+            compileBinOp(tok, compileAnd, op);
+        } else break;
+    }
+}
+
+static void compileOr(Token *&tok, std::stack<Token*> &op)
+{
+    compileXor(tok,op);
+    while (tok) {
+        if (tok->str() == "|") {
+            compileBinOp(tok, compileXor, op);
+        } else break;
+    }
+}
+
+static void compileLogicAnd(Token *&tok, std::stack<Token*> &op)
+{
+    compileOr(tok,op);
+    while (tok) {
+        if (tok->str() == "&&") {
+            compileBinOp(tok, compileOr, op);
+        } else break;
+    }
+}
+
+static void compileLogicOr(Token *&tok, std::stack<Token*> &op)
+{
+    compileLogicAnd(tok,op);
+    while (tok) {
+        if (tok->str() == "||") {
+            compileBinOp(tok, compileLogicAnd, op);
+        } else break;
+    }
+}
+
+static void compileAssign(Token *&tok, std::stack<Token*> &op)
+{
+    compileLogicOr(tok,op);
+    while (tok) {
+        if (tok->str() == "=") {
+            compileBinOp(tok, compileLogicOr, op);
+        } else break;
+    }
+}
+
+static void compileComma(Token *&tok, std::stack<Token*> &op)
+{
+    compileAssign(tok,op);
+    while (tok) {
+        if (tok->str() == ",") {
+            compileBinOp(tok, compileAssign, op);
+        } else break;
+    }
+}
+
+static void compileExpression(Token *&tok, std::stack<Token*> &op)
+{
+    compileComma(tok,op);
+}
+
+void TokenList::createAst()
+{
+    for (Token *tok = _front; tok; tok = tok ? tok->next() : NULL) {
+        if (!tok->previous() || Token::Match(tok, "%var% (|[|.|=")) {
+            std::stack<Token *> operands;
+            compileExpression(tok, operands);
         }
     }
 }

lib/tokenlist.h

@@ -104,7 +104,7 @@ public:
      */
     std::string fileLine(const Token *tok) const;
 
-    void createAst() const;
+    void createAst();
 
 private:
     /** Disable copy constructor, no implementation */

test/testtokenize.cpp

@@ -9962,17 +9962,21 @@ private:
         ASSERT_EQUALS("abc+=", testAst("a=b+c"));
         ASSERT_EQUALS("abc=,", testAst("a,b=c"));
 
+        ASSERT_EQUALS("a\"\"=", testAst("a=\"\""));
+        ASSERT_EQUALS("a\'\'=", testAst("a=\'\'"));
+        testAst("char a[1]=\"\";"); // don't crash
+        testAst("int f(char argv[]);"); // don't crash
     }
 
     void astpar() const { // parentheses
         ASSERT_EQUALS("12+3*", testAst("(1+2)*3"));
         ASSERT_EQUALS("123+*", testAst("1*(2+3)"));
         ASSERT_EQUALS("123+*4*", testAst("1*(2+3)*4"));
-        ASSERT_EQUALS("ab.c&d==if", testAst("if((a.b&c)==d){}"));
+        ASSERT_EQUALS("ifab.c&d==(", testAst("if((a.b&c)==d){}"));
     }
 
     void astbrackets() const { // []
-        ASSERT_EQUALS("123+[4+", testAst("1[2+3]+4"));
+        ASSERT_EQUALS("a23+[4+", testAst("a[2+3]+4"));
     }
 
     void astunaryop() const { // unary operators
@@ -9982,10 +9986,14 @@ private:
     }
 
     void astfunction() const { // function calls
-        ASSERT_EQUALS("1(f+2+", testAst("1+f()+2"));
+        ASSERT_EQUALS("1f(+2+", testAst("1+f()+2"));
-        ASSERT_EQUALS("12f+3+", testAst("1+f(2)+3"));
+        ASSERT_EQUALS("1f2(+3+", testAst("1+f(2)+3"));
-        ASSERT_EQUALS("123,f+4+", testAst("1+f(2,3)+4"));
+        ASSERT_EQUALS("1f23,(+4+", testAst("1+f(2,3)+4"));
-        ASSERT_EQUALS("12a&,f+", testAst("1+f(2,&a)"));
+        ASSERT_EQUALS("1f2a&,(+", testAst("1+f(2,&a)"));
+        testAst("extern unsigned f(const char *);"); // don't crash
+        testAst("extern void f(const char *format, ...);"); // don't crash
+        testAst("extern int for_each_commit_graft(int (*)(int*), void *);"); // don't crash
+        testAst("for (;;) {}"); // don't crash
     }
 
     void asttemplate() const { // uninstantiated templates will have <,>,etc.. how do we handle them?