From 2ace3d31447be28d410d052dced441d7937bac5e Mon Sep 17 00:00:00 2001 From: PKEuS Date: Thu, 27 Aug 2015 14:34:00 +0200 Subject: [PATCH] Rewritten handling of declaration specifiers: - Removed simplifyConst() because it did the opposite of the (superior) simplifyStaticConst() - Execute simplifyStaticConst() in simplifyTokenList1() - there is no reason to defer it, and it is required to properly parse declarations like "unsigned static int i;" - Fixed simplifyStaticConst() to handle more patterns. It did not work at the beginning of the token list and for function arguments - Reimplemented Tokenizer::simplifyStdType() -> properly support all possible ways to declare integers as requested by the standard, instead of only a few common permutations of "unsigned|signed", "short|char|long|int" -> Fixed parsing of _Complex/complex types --- lib/token.cpp | 2 + lib/tokenize.cpp | 185 ++++++++++++++--------------------- lib/tokenize.h | 5 - test/testsimplifytypedef.cpp | 2 +- test/testtokenize.cpp | 53 +++++++--- 5 files changed, 114 insertions(+), 133 deletions(-) diff --git a/lib/token.cpp b/lib/token.cpp index 845d6dacb..a05af8cc2 100644 --- a/lib/token.cpp +++ b/lib/token.cpp @@ -1008,6 +1008,8 @@ void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro) os << "unsigned "; else if (isSigned()) os << "signed "; + if (isComplex()) + os << "_Complex "; if (isLong()) { if (_tokType == eString || _tokType == eChar) os << "L"; diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 890b89362..cb511f573 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -1754,20 +1754,24 @@ bool Tokenizer::tokenizeCondition(const std::string &code) // Remove "volatile", "inline", "register", and "restrict" simplifyKeyword(); + // Concatenate double sharp: 'a ## b' -> 'ab' + concatenateDoubleSharp(); + + // Link brackets (, [ and { + createLinks(); + + // Order keywords "static" and "const" + simplifyStaticConst(); + // convert platform dependent types to standard types // 32 bits: size_t -> unsigned long // 64 bits: size_t -> unsigned long long simplifyPlatformTypes(); // collapse compound standard types into a single token - // unsigned long long int => long _isUnsigned=true,_isLong=true + // unsigned long long int => long (with _isUnsigned=true,_isLong=true) simplifyStdType(); - // Concatenate double sharp: 'a ## b' -> 'ab' - concatenateDoubleSharp(); - - createLinks(); - // replace 'NULL' and similar '0'-defined macros with '0' simplifyNull(); @@ -3422,13 +3426,16 @@ bool Tokenizer::simplifyTokenList1(const char FileName[]) // Put ^{} statements in asm() simplifyAsm2(); + // Order keywords "static" and "const" + simplifyStaticConst(); + // convert platform dependent types to standard types // 32 bits: size_t -> unsigned long // 64 bits: size_t -> unsigned long long simplifyPlatformTypes(); // collapse compound standard types into a single token - // unsigned long long int => long _isUnsigned=true,_isLong=true + // unsigned long long int => long (with _isUnsigned=true,_isLong=true) simplifyStdType(); if (_settings->terminated()) @@ -3440,8 +3447,6 @@ bool Tokenizer::simplifyTokenList1(const char FileName[]) if (_settings->terminated()) return false; - simplifyConst(); - // struct simplification "struct S {} s; => struct S { } ; S s ; simplifyStructDecl(); @@ -3665,8 +3670,6 @@ bool Tokenizer::simplifyTokenList2() simplifyEmptyNamespaces(); - simplifyStaticConst(); - simplifyMathFunctions(); validate(); @@ -5644,88 +5647,59 @@ void Tokenizer::simplifyPlatformTypes() void Tokenizer::simplifyStdType() { for (Token *tok = list.front(); tok; tok = tok->next()) { - // long unsigned => unsigned long - if (Token::Match(tok, "char|short|int|long unsigned|signed")) { - const bool isUnsigned = tok->next()->str() == "unsigned"; - tok->deleteNext(); - tok->isUnsigned(isUnsigned); - tok->isSigned(!isUnsigned); - } + if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (_settings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) { + bool isFloat= false; + bool isSigned = false; + bool isUnsigned = false; + bool isComplex = false; + unsigned int countLong = 0; + Token* typeSpec = nullptr; - else if (Token::Match(tok, "float|double complex|_Complex")) { - tok->deleteNext(); - tok->isComplex(true); - } - - else if (!Token::Match(tok, "unsigned|signed|char|short|int|long")) - continue; - - // check if signed or unsigned specified - if (Token::Match(tok, "unsigned|signed")) { - const bool isUnsigned = tok->str() == "unsigned"; - - // unsigned i => unsigned int i - if (!Token::Match(tok->next(), "char|short|int|long")) - tok->str("int"); - else - tok->deleteThis(); - tok->isUnsigned(isUnsigned); - tok->isSigned(!isUnsigned); - } - - if (tok->str() == "int") { - if (tok->strAt(1) == "long") { - tok->str("long"); - tok->deleteNext(); - } else if (tok->strAt(1) == "short") { - tok->str("short"); - tok->deleteNext(); + Token* tok2 = tok; + for (; tok2->next(); tok2 = tok2->next()) { + if (tok2->str() == "long") { + countLong++; + if (!isFloat) + typeSpec = tok2; + } else if (tok2->str() == "short") { + typeSpec = tok2; + } else if (tok2->str() == "unsigned") + isUnsigned = true; + else if (tok2->str() == "signed") + isSigned = true; + else if (Token::Match(tok2, "float|double")) { + isFloat = true; + typeSpec = tok2; + } else if (_settings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex")) + isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name + else if (Token::Match(tok2, "char|int")) { + if (!typeSpec) + typeSpec = tok2; + } else + break; } - if (tok->strAt(1) == "long") { - tok->isLong(true); - tok->deleteNext(); - } - if (Token::Match(tok->next(), "unsigned|signed")) { - tok->isUnsigned(tok->next()->str() == "unsigned"); - tok->isSigned(tok->next()->str() == "signed"); - tok->deleteNext(); - if (tok->strAt(1) == "long") - tok->deleteNext(); - else if (tok->strAt(1) == "short") - tok->deleteNext(); - } - } else if (tok->str() == "long") { - if (tok->strAt(1) == "long") { - tok->isLong(true); - tok->deleteNext(); - } - if (tok->strAt(1) == "int") { - tok->deleteNext(); - if (Token::Match(tok->next(), "unsigned|signed")) { - tok->isUnsigned(tok->next()->str() == "unsigned"); - tok->isSigned(tok->next()->str() == "signed"); - tok->deleteNext(); + + if (!typeSpec) { // unsigned i; or similar declaration + if (!isComplex) { // Ensure that "complex" is not the variables name + tok->str("int"); + tok->isSigned(isSigned); + tok->isUnsigned(isUnsigned); + } + } else { + typeSpec->isLong(typeSpec->isLong() || (isFloat && countLong == 1) || countLong > 1); + typeSpec->isComplex(typeSpec->isComplex() || (isFloat && isComplex)); + typeSpec->isSigned(typeSpec->isSigned() || isSigned); + typeSpec->isUnsigned(typeSpec->isUnsigned() || isUnsigned); + + // Remove specifiers + const Token* tok3 = tok->previous(); + tok2 = tok2->previous(); + while (tok3 != tok2) { + if (tok2 != typeSpec && + (isComplex || !Token::Match(tok2, "complex|_Complex"))) // Ensure that "complex" is not the variables name + tok2->deleteThis(); + tok2 = tok2->previous(); } - } else if (tok->strAt(1) == "double") { - tok->str("double"); - tok->isLong(true); - tok->deleteNext(); - } else if (Token::Match(tok->next(), "unsigned|signed")) { - tok->isUnsigned(tok->next()->str() == "unsigned"); - tok->isSigned(tok->next()->str() == "signed"); - tok->deleteNext(); - if (tok->strAt(1) == "int") - tok->deleteNext(); - } - } else if (tok->str() == "short") { - if (tok->strAt(1) == "int") - tok->deleteNext(); - if (Token::Match(tok->next(), "unsigned|signed")) { - tok->isUnsigned(tok->next()->str() == "unsigned"); - tok->isSigned(tok->next()->str() == "signed"); - tok->deleteNext(); - if (tok->strAt(1) == "int") - tok->deleteNext(); } } } @@ -5751,20 +5725,22 @@ void Tokenizer::simplifyStaticConst() Token* leftTok = tok; for (; leftTok; leftTok = leftTok->previous()) { if (!Token::Match(leftTok, "%type%|static|const|extern") || - (isCPP() && Token::Match(leftTok, "private:|protected:|public:"))) + (isCPP() && Token::Match(leftTok, "private:|protected:|public:|operator"))) break; } - // The token preceding the declaration should indicate the start of a statement - if (!leftTok || - leftTok == tok || - !Token::Match(leftTok, ";|{|}|private:|protected:|public:")) { + // The token preceding the declaration should indicate the start of a declaration + if (leftTok == tok || + (leftTok && !Token::Match(leftTok, ";|{|}|(|,|private:|protected:|public:"))) { continue; } // Move the qualifier to the left-most position in the declaration tok->deleteNext(); - if (leftTok->next()) + if (!leftTok) { + list.front()->insertToken(qualifier, false); + list.front()->swapWithNext(); + } else if (leftTok->next()) leftTok->next()->insertToken(qualifier, true); else leftTok->insertToken(qualifier); @@ -8645,23 +8621,6 @@ std::string Tokenizer::simplifyString(const std::string &source) return str; } -void Tokenizer::simplifyConst() -{ - for (Token *tok = list.front(); tok; tok = tok->next()) { - if (tok->isStandardType() && tok->strAt(1) == "const") { - tok->swapWithNext(); - } else if (Token::Match(tok, "struct %type% const")) { - tok->next()->swapWithNext(); - tok->swapWithNext(); - } else if (Token::Match(tok, "%type% const") && - (!tok->previous() || Token::Match(tok->previous(), "[;{}(,]")) && - tok->str().find(':') == std::string::npos && - tok->str() != "operator") { - tok->swapWithNext(); - } - } -} - void Tokenizer::getErrorMessages(ErrorLogger *errorLogger, const Settings *settings) { Tokenizer t(settings, errorLogger); diff --git a/lib/tokenize.h b/lib/tokenize.h index dbac16296..5425633c1 100644 --- a/lib/tokenize.h +++ b/lib/tokenize.h @@ -527,11 +527,6 @@ private: */ const Token * isFunctionHead(const Token *tok, const std::string &endsWith) const; - /** - * Change "int const x;" into "const int x;" - */ - void simplifyConst(); - /** * simplify "while (0)" */ diff --git a/test/testsimplifytypedef.cpp b/test/testsimplifytypedef.cpp index 352944b4d..f0211722a 100644 --- a/test/testsimplifytypedef.cpp +++ b/test/testsimplifytypedef.cpp @@ -684,7 +684,7 @@ private: "class Fred { " "" "const unsigned int * * get ( ) { return test ; } " - "const static unsigned int * test ( const unsigned int * p ) { return p ; } " + "static const unsigned int * test ( const unsigned int * p ) { return p ; } " "} ;"; ASSERT_EQUALS(expected, tok(code, false)); diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 82517d7f6..6d210afb2 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -3324,7 +3324,7 @@ private: static char exp[] = "struct S { " "char * a ; " "char & b ; " - "const static char * c ; " + "static const char * c ; " "} ;"; ASSERT_EQUALS(exp, tokenizeAndStringify(code)); } @@ -3464,7 +3464,7 @@ private: "a = SZ;\n" "}\n"; const char expected[] = - "const static char str [ 5 ] = \"abcd\" ;\n\nvoid f ( ) {\na = 5 ;\n}"; + "static const char str [ 5 ] = \"abcd\" ;\n\nvoid f ( ) {\na = 5 ;\n}"; ASSERT_EQUALS(expected, tokenizeAndStringify(code,true)); } @@ -3746,11 +3746,11 @@ private: " unsigned int *foo = &x;" "}"; ASSERT_EQUALS("unsigned int x ; " - "const static unsigned int A = 1 ; " - "const static unsigned int B = A ; " - "const static unsigned int C = 0 ; " - "const static unsigned int D = A ; " - "const static unsigned int E = 0 ; " + "static const unsigned int A = 1 ; " + "static const unsigned int B = A ; " + "static const unsigned int C = 0 ; " + "static const unsigned int D = A ; " + "static const unsigned int E = 0 ; " "void f ( ) { " "unsigned int * foo ; " "foo = & x ; " @@ -4590,9 +4590,34 @@ private: const char expected[] = "signed short x ;"; ASSERT_EQUALS(expected, tokenizeAndStringify(code)); } + { + const char code[] = "unsigned static short const int i;"; + const char expected[] = "static const unsigned short i ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + } { const char code[] = "float complex x;"; - const char expected[] = "float x ;"; + const char expected[] = "_Complex float x ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + } + { + const char code[] = "complex float x;"; + const char expected[] = "_Complex float x ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + } + { + const char code[] = "complex long double x;"; + const char expected[] = "_Complex long double x ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + } + { + const char code[] = "long double complex x;"; + const char expected[] = "_Complex long double x ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + } + { + const char code[] = "double complex;"; + const char expected[] = "double complex ;"; ASSERT_EQUALS(expected, tokenizeAndStringify(code)); } } @@ -8288,12 +8313,12 @@ private: "static const signed long long i4 ;\n" "static const signed long long i5 ;\n" "static const signed long long i6 ;\n" - "static const long long signed int i7 ;\n" - "static const long long signed int i8 ;\n" - "static const signed int long long i9 ;\n" - "static const signed int long long i10 ;\n" - "static const int signed long long i11 ;\n" - "static const int signed long long i12 ;\n" + "static const signed long long i7 ;\n" + "static const signed long long i8 ;\n" + "static const signed long long i9 ;\n" + "static const signed long long i10 ;\n" + "static const signed long long i11 ;\n" + "static const signed long long i12 ;\n" "static const signed long long i13 ;\n" "static const signed long long i14 ;\n" "static const signed long long i15 ;\n"