walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Add exprId to tokens (#2744)

This commit is contained in:
Paul Fultz II 2020-08-20 11:21:29 -05:00 committed by GitHub
parent 6446790d48
commit a332062385
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 172 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
lib/symboldatabase.cpp
View File

@ -21,6 +21,7 @@
#include "astutils.h"
#include "errorlogger.h"
#include "mathlib.h"
#include "platform.h"
#include "settings.h"
#include "token.h"
@ -34,6 +35,7 @@
#include <climits>
#include <iomanip>
#include <iostream>
#include <unordered_map>
//---------------------------------------------------------------------------
SymbolDatabase::SymbolDatabase(const Tokenizer *tokenizer, const Settings *settings, ErrorLogger *errorLogger)
@ -69,6 +71,7 @@ SymbolDatabase::SymbolDatabase(const Tokenizer *tokenizer, const Settings *setti
createSymbolDatabaseEnums();
createSymbolDatabaseEscapeFunctions();
createSymbolDatabaseIncompleteVars();
createSymbolDatabaseExprIds();
}
static const Token* skipScopeIdentifiers(const Token* tok)
@ -1425,6 +1428,49 @@ void SymbolDatabase::createSymbolDatabaseEscapeFunctions()
}
}
void SymbolDatabase::createSymbolDatabaseExprIds()
{
MathLib::bigint base = 0;
// Find highest varId
for (const Variable *var : mVariableList) {
if (!var)
continue;
base = std::max<MathLib::bigint>(base, var->declarationId());
}
MathLib::bigint id = base+1;
for (const Scope * scope : functionScopes) {
std::unordered_map<std::string, std::vector<Token*>> exprs;
// Assign IDs
for (Token* tok = const_cast<Token*>(scope->bodyStart); tok != scope->bodyEnd; tok = tok->next()) {
if (tok->varId() > 0) {
tok->exprId(tok->varId());
} else if (Token::Match(tok, "(|.|%cop%")) {
exprs[tok->str()].push_back(tok);
tok->exprId(id++);
}
}
// Apply CSE
for(const auto& p:exprs) {
const std::vector<Token*>& tokens = p.second;
for(Token* tok1:tokens) {
for(Token* tok2:tokens) {
if (tok1 == tok2)
continue;
if (tok1->exprId() == tok2->exprId())
continue;
if (!isSameExpression(isCPP(), true, tok1, tok2, mSettings->library, true, false))
continue;
MathLib::bigint cid = std::min(tok1->exprId(), tok2->exprId());
tok1->exprId(cid);
tok2->exprId(cid);
}
}
}
}
}
void SymbolDatabase::setArrayDimensionsUsingValueFlow()
{
// set all unknown array dimensions

1
lib/symboldatabase.h
View File

@ -1350,6 +1350,7 @@ private:
void createSymbolDatabaseEnums();
void createSymbolDatabaseEscapeFunctions();
void createSymbolDatabaseIncompleteVars();
void createSymbolDatabaseExprIds();
void addClassFunction(Scope **scope, const Token **tok, const Token *argStart);
Function *addGlobalFunctionDecl(Scope*& scope, const Token* tok, const Token *argStart, const Token* funcStart);

56
lib/token.cpp
View File

@ -1140,14 +1140,14 @@ void Token::printOut(const char *title) const
{
if (title && title[0])
std::cout << "\n### " << title << " ###\n";
std::cout << stringifyList(true, true, true, true, true, nullptr, nullptr) << std::endl;
std::cout << stringifyList(stringifyOptions::forDebugExprId(), nullptr, nullptr) << std::endl;
}
void Token::printOut(const char *title, const std::vector<std::string> &fileNames) const
{
if (title && title[0])
std::cout << "\n### " << title << " ###\n";
std::cout << stringifyList(true, true, true, true, true, &fileNames, nullptr) << std::endl;
std::cout << stringifyList(stringifyOptions::forDebugExprId(), &fileNames, nullptr) << std::endl;
}
void Token::printLines(int lines) const
@ -1155,12 +1155,12 @@ void Token::printLines(int lines) const
const Token *end = this;
while (end && end->linenr() < lines + linenr())
end = end->next();
std::cout << stringifyList(true, true, true, true, true, nullptr, end) << std::endl;
std::cout << stringifyList(stringifyOptions::forDebugExprId(), nullptr, end) << std::endl;
}
void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro) const
void Token::stringify(std::ostream& os, const stringifyOptions& options) const
{
if (attributes) {
if (options.attributes) {
if (isUnsigned())
os << "unsigned ";
else if (isSigned())
@ -1172,7 +1172,7 @@ void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro)
os << "long ";
}
}
if (macro && isExpandedMacro())
if (options.macro && isExpandedMacro())
os << "$";
if (isName() && mStr.find(' ') != std::string::npos) {
for (char i : mStr) {
@ -1189,19 +1189,30 @@ void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro)
os << i;
}
}
if (varid && mImpl->mVarId != 0)
if (options.varid && mImpl->mVarId != 0)
os << '@' << mImpl->mVarId;
if (options.exprid && mImpl->mExprId != 0)
os << '@' << mImpl->mExprId;
}
std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers, bool linebreaks, bool files, const std::vector<std::string>* fileNames, const Token* end) const
void Token::stringify(std::ostream& os, bool varid, bool attributes, bool macro) const
{
stringifyOptions options;
options.varid = varid;
options.attributes = attributes;
options.macro = macro;
stringify(os, options);
}
std::string Token::stringifyList(const stringifyOptions& options, const std::vector<std::string>* fileNames, const Token* end) const
{
if (this == end)
return "";
std::ostringstream ret;
unsigned int lineNumber = mImpl->mLineNumber - (linenumbers ? 1U : 0U);
unsigned int fileIndex = files ? ~0U : mImpl->mFileIndex;
unsigned int lineNumber = mImpl->mLineNumber - (options.linenumbers ? 1U : 0U);
unsigned int fileIndex = options.files ? ~0U : mImpl->mFileIndex;
std::map<int, unsigned int> lineNumbers;
for (const Token *tok = this; tok != end; tok = tok->next()) {
bool fileChange = false;
@ -1211,7 +1222,7 @@ std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers,
}
fileIndex = tok->mImpl->mFileIndex;
if (files) {
if (options.files) {
ret << "\n\n##file ";
if (fileNames && fileNames->size() > tok->mImpl->mFileIndex)
ret << fileNames->at(tok->mImpl->mFileIndex);
@ -1224,18 +1235,18 @@ std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers,
fileChange = true;
}
if (linebreaks && (lineNumber != tok->linenr() || fileChange)) {
if (options.linebreaks && (lineNumber != tok->linenr() || fileChange)) {
if (lineNumber+4 < tok->linenr() && fileIndex == tok->mImpl->mFileIndex) {
ret << '\n' << lineNumber+1 << ":\n|\n";
ret << tok->linenr()-1 << ":\n";
ret << tok->linenr() << ": ";
} else if (this == tok && linenumbers) {
} else if (this == tok && options.linenumbers) {
ret << tok->linenr() << ": ";
} else {
while (lineNumber < tok->linenr()) {
++lineNumber;
ret << '\n';
if (linenumbers) {
if (options.linenumbers) {
ret << lineNumber << ':';
if (lineNumber == tok->linenr())
ret << ' ';
@ -1245,14 +1256,25 @@ std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers,
lineNumber = tok->linenr();
}
tok->stringify(ret, varid, attributes, attributes); // print token
if (tok->next() != end && (!linebreaks || (tok->next()->linenr() <= tok->linenr() && tok->next()->fileIndex() == tok->fileIndex())))
tok->stringify(ret, options); // print token
if (tok->next() != end && (!options.linebreaks || (tok->next()->linenr() <= tok->linenr() && tok->next()->fileIndex() == tok->fileIndex())))
ret << ' ';
}
if (linebreaks && (files || linenumbers))
if (options.linebreaks && (options.files || options.linenumbers))
ret << '\n';
return ret.str();
}
std::string Token::stringifyList(bool varid, bool attributes, bool linenumbers, bool linebreaks, bool files, const std::vector<std::string>* fileNames, const Token* end) const
{
stringifyOptions options;
options.varid = varid;
options.attributes = attributes;
options.macro = attributes;
options.linenumbers = linenumbers;
options.linebreaks = linebreaks;
options.files = files;
return stringifyList(options, fileNames, end);
}
std::string Token::stringifyList(const Token* end, bool attributes) const
{

40
lib/token.h
View File

@ -65,6 +65,7 @@ struct TokenImpl {
nonneg int mFileIndex;
nonneg int mLineNumber;
nonneg int mColumn;
MathLib::bigint mExprId;
// AST..
Token *mAstOperand1;
@ -129,6 +130,7 @@ struct TokenImpl {
, mFileIndex(0)
, mLineNumber(0)
, mColumn(0)
, mExprId(0)
, mAstOperand1(nullptr)
, mAstOperand2(nullptr)
, mAstParent(nullptr)
@ -790,6 +792,13 @@ public:
}
}
MathLib::bigint exprId() const {
return mImpl->mExprId;
}
void exprId(MathLib::bigint id) {
mImpl->mExprId = id;
}
/**
* For debugging purposes, prints token and all tokens
* followed by it.
@ -822,6 +831,35 @@ public:
*/
static void replace(Token *replaceThis, Token *start, Token *end);
struct stringifyOptions {
bool varid = false;
bool exprid = false;
bool attributes = false;
bool macro = false;
bool linenumbers = false;
bool linebreaks = false;
bool files = false;
static stringifyOptions forDebug() {
stringifyOptions options;
options.attributes = true;
options.macro = true;
options.linenumbers = true;
options.linebreaks = true;
options.files = true;
return options;
}
static stringifyOptions forDebugVarId() {
stringifyOptions options = forDebug();
options.varid = true;
return options;
}
static stringifyOptions forDebugExprId() {
stringifyOptions options = forDebug();
options.exprid = true;
return options;
}
};
/**
* Stringify a token
* @param os The result is shifted into that output stream
@ -829,6 +867,7 @@ public:
* @param attributes Print attributes of tokens like "unsigned" in front of it.
* @param macro Prints $ in front of the token if it was expanded from a macro.
*/
void stringify(std::ostream& os, const stringifyOptions& options) const;
void stringify(std::ostream& os, bool varid, bool attributes, bool macro) const;
/**
@ -842,6 +881,7 @@ public:
* @param end Stringification ends before this token is reached. 0 to stringify until end of list.
* @return Stringified token list as a string
*/
std::string stringifyList(const stringifyOptions& options, const std::vector<std::string>* fileNames = nullptr, const Token* end = nullptr) const;
std::string stringifyList(bool varid, bool attributes, bool linenumbers, bool linebreaks, bool files, const std::vector<std::string>* fileNames = nullptr, const Token* end = nullptr) const;
std::string stringifyList(const Token* end, bool attributes = true) const;
std::string stringifyList(bool varid = false) const;

47
test/testvarid.cpp
View File

@ -202,6 +202,8 @@ private:
TEST_CASE(setVarIdStructMembers1);
TEST_CASE(decltype1);
TEST_CASE(exprid1);
}
std::string tokenize(const char code[], const char filename[] = "test.cpp") {
@ -217,7 +219,27 @@ private:
tokenizer.tokenize(istr, filename);
// result..
return tokenizer.tokens()->stringifyList(true,true,true,true,false);
Token::stringifyOptions options = Token::stringifyOptions::forDebugVarId();
options.files = false;
return tokenizer.tokens()->stringifyList(options);
}
std::string tokenizeExpr(const char code[], const char filename[] = "test.cpp") {
errout.str("");
Settings settings;
settings.platform(Settings::Unix64);
settings.standards.c = Standards::C89;
settings.standards.cpp = Standards::CPP11;
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.tokenize(istr, filename);
// result..
Token::stringifyOptions options = Token::stringifyOptions::forDebugExprId();
options.files = false;
return tokenizer.tokens()->stringifyList(options);
}
std::string compareVaridsForVariable(const char code[], const char varname[], const char filename[] = "test.cpp") {
@ -3179,6 +3201,29 @@ private:
const char expected[] = "1: void foo ( int x@1 , decltype ( A :: b ) * p@2 ) ;\n";
ASSERT_EQUALS(expected, tokenize(code));
}
void exprid1() {
const std::string actual = tokenizeExpr(
"struct A {\n"
" int x, y;\n"
"};\n"
"int f(A a, A b) {\n"
" int x = a.x + b.x;\n"
" int y = b.x + a.x;\n"
" return x + y + a.y + b.y;\n"
"}\n");
const char expected[] = "1: struct A {\n"
"2: int x ; int y ;\n"
"3: } ;\n"
"4: int f ( A a , A b ) {\n"
"5: int x@5 ; x@5 = a@3 .@9 x@6 +@10 b@4 .@11 x@7 ;\n"
"6: int y@8 ; y@8 = b@4 .@11 x@7 +@10 a@3 .@9 x@6 ;\n"
"7: return x@5 +@15 y@8 +@16 a@3 .@17 y@9 +@18 b@4 .@19 y@10 ;\n"
"8: }\n";
ASSERT_EQUALS(expected, actual);
}
};
REGISTER_TEST(TestVarID)