walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Merge pull request #721 from 3adev/optimize-tokenizing 

Separate dump from checks and optimize tokenizing
This commit is contained in:
Daniel Marjamäki 2015-12-13 12:31:48 +01:00
parent 073d5c1db4 58031147b2
commit dbc7e2f338
4 changed files with 174 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
lib/cppcheck.cpp
View File

@ -108,7 +108,7 @@ unsigned int CppCheck::processFile(const std::string& filename, std::istream& fi
return 0;
}
// Run rules on this code
// Run define rules on raw code
for (std::list<Settings::Rule>::const_iterator it = _settings.rules.begin(); it != _settings.rules.end(); ++it) {
if (it->tokenlist == "define") {
Tokenizer tokenizer2(&_settings, this);
@ -146,6 +146,10 @@ unsigned int CppCheck::processFile(const std::string& filename, std::istream& fi
std::set<unsigned long long> checksums;
unsigned int checkCount = 0;
for (std::list<std::string>::const_iterator it = configurations.begin(); it != configurations.end(); ++it) {
// bail out if terminated
if (_settings.terminated())
break;
// Check only a few configurations (default 12), after that bail out, unless --force
// was used.
if (!_settings._force && ++checkCount > _settings._maxConfigs)
@ -172,11 +176,95 @@ unsigned int CppCheck::processFile(const std::string& filename, std::istream& fi
codeWithoutCfg += _settings.append();
if (!checkFile(codeWithoutCfg, filename.c_str(), checksums, internalErrorFound)) {
Tokenizer _tokenizer(&_settings, this);
if (_settings._showtime != SHOWTIME_NONE)
_tokenizer.setTimerResults(&S_timerResults);
try {
// Create tokens, skip rest of iteration if failed
std::istringstream istr(codeWithoutCfg);
Timer timer("Tokenizer::createTokens", _settings._showtime, &S_timerResults);
bool result = _tokenizer.createTokens(istr, filename.c_str());
timer.Stop();
if (!result)
continue;
// skip rest of iteration if just checking configuration
if (_settings.checkConfiguration)
continue;
// Check raw tokens
checkRawTokens(_tokenizer);
// Simplify tokens into normal form, skip rest of iteration if failed
Timer timer2("Tokenizer::simplifyTokens1", _settings._showtime, &S_timerResults);
result = _tokenizer.simplifyTokens1(cfg);
timer2.Stop();
if (!result)
continue;
// dump xml
if (_settings.dump) {
std::ofstream fdump((filename + ".dump").c_str());
if (fdump.is_open()) {
fdump << "<?xml version=\"1.0\"?>" << std::endl;
fdump << "<dump cfg=\"" << cfg << "\">" << std::endl;
_tokenizer.dump(fdump);
fdump << "</dump>" << std::endl;
}
}
// Skip if we already met the same simplified token list
if (_settings._force || _settings._maxConfigs > 1) {
const unsigned long long checksum = _tokenizer.list.calculateChecksum();
if (checksums.find(checksum) != checksums.end())
continue;
checksums.insert(checksum);
}
// Check normal tokens
checkNormalTokens(_tokenizer);
// simplify more if required, skip rest of iteration if failed
if (_simplify) {
// if further simplification fails then skip rest of iteration
Timer timer3("Tokenizer::simplifyTokenList2", _settings._showtime, &S_timerResults);
result = _tokenizer.simplifyTokenList2();
timer3.Stop();
if (!result)
continue;
// Check simplified tokens
checkSimplifiedTokens(_tokenizer);
}
} catch (const InternalError &e) {
if (_settings.isEnabled("information") && (_settings.debug || _settings._verbose))
purgedConfigurationMessage(filename, cfg);
internalErrorFound=true;
std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
ErrorLogger::ErrorMessage::FileLocation loc;
if (e.token) {
loc.line = e.token->linenr();
const std::string fixedpath = Path::toNativeSeparators(_tokenizer.list.file(e.token));
loc.setfile(fixedpath);
} else {
ErrorLogger::ErrorMessage::FileLocation loc2;
loc2.setfile(Path::toNativeSeparators(filename.c_str()));
locationList.push_back(loc2);
loc.setfile(_tokenizer.list.getSourceFilePath());
}
locationList.push_back(loc);
const ErrorLogger::ErrorMessage errmsg(locationList,
Severity::error,
e.errorMessage,
e.id,
false);
reportErr(errmsg);
}
}
} catch (const std::runtime_error &e) {
internalError(filename, e.what());
} catch (const InternalError &e) {
@ -222,128 +310,57 @@ void CppCheck::internalError(const std::string &filename, const std::string &msg
}
//---------------------------------------------------------------------------
// CppCheck - A function that checks a specified file
// CppCheck - A function that checks a raw token list
//---------------------------------------------------------------------------
bool CppCheck::checkFile(const std::string &code, const char FileName[], std::set<unsigned long long>& checksums, bool& internalErrorFound)
void CppCheck::checkRawTokens(const Tokenizer &tokenizer)
{
internalErrorFound=false;
if (_settings.terminated() || _settings.checkConfiguration)
return true;
// Execute rules for "raw" code
executeRules("raw", tokenizer);
}
Tokenizer _tokenizer(&_settings, this);
if (_settings._showtime != SHOWTIME_NONE)
_tokenizer.setTimerResults(&S_timerResults);
try {
// Execute rules for "raw" code
for (std::list<Settings::Rule>::const_iterator it = _settings.rules.begin(); it != _settings.rules.end(); ++it) {
if (it->tokenlist == "raw") {
Tokenizer tokenizer2(&_settings, this);
std::istringstream istr(code);
tokenizer2.list.createTokens(istr, FileName);
executeRules("raw", tokenizer2);
break;
}
}
// Tokenize the file
std::istringstream istr(code);
Timer timer("Tokenizer::tokenize", _settings._showtime, &S_timerResults);
bool result = _tokenizer.tokenize(istr, FileName, cfg);
timer.Stop();
if (_settings._force || _settings._maxConfigs > 1) {
const unsigned long long checksum = _tokenizer.list.calculateChecksum();
if (checksums.find(checksum) != checksums.end())
return false;
checksums.insert(checksum);
}
if (!result) {
// File had syntax errors, abort
return true;
}
// dump
if (_settings.dump) {
std::string dumpfile = std::string(FileName) + ".dump";
std::ofstream fdump(dumpfile.c_str());
if (fdump.is_open()) {
fdump << "<?xml version=\"1.0\"?>" << std::endl;
fdump << "<dump cfg=\"" << cfg << "\">" << std::endl;
_tokenizer.dump(fdump);
fdump << "</dump>" << std::endl;
}
return true;
}
// call all "runChecks" in all registered Check classes
for (std::list<Check *>::const_iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
if (_settings.terminated())
return true;
Timer timerRunChecks((*it)->name() + "::runChecks", _settings._showtime, &S_timerResults);
(*it)->runChecks(&_tokenizer, &_settings, this);
}
// Analyse the tokens..
for (std::list<Check *>::const_iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
Check::FileInfo *fi = (*it)->getFileInfo(&_tokenizer, &_settings);
if (fi != nullptr)
fileInfo.push_back(fi);
}
executeRules("normal", _tokenizer);
if (!_simplify)
return true;
Timer timer3("Tokenizer::simplifyTokenList2", _settings._showtime, &S_timerResults);
result = _tokenizer.simplifyTokenList2();
timer3.Stop();
if (!result)
return true;
// call all "runSimplifiedChecks" in all registered Check classes
for (std::list<Check *>::const_iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
if (_settings.terminated())
return true;
Timer timerSimpleChecks((*it)->name() + "::runSimplifiedChecks", _settings._showtime, &S_timerResults);
(*it)->runSimplifiedChecks(&_tokenizer, &_settings, this);
}
//---------------------------------------------------------------------------
// CppCheck - A function that checks a normal token list
//---------------------------------------------------------------------------
void CppCheck::checkNormalTokens(const Tokenizer &tokenizer)
{
// call all "runChecks" in all registered Check classes
for (std::list<Check *>::const_iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
if (_settings.terminated())
return true;
return;
executeRules("simple", _tokenizer);
if (_settings.terminated())
return true;
} catch (const InternalError &e) {
internalErrorFound=true;
std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
ErrorLogger::ErrorMessage::FileLocation loc;
if (e.token) {
loc.line = e.token->linenr();
const std::string fixedpath = Path::toNativeSeparators(_tokenizer.list.file(e.token));
loc.setfile(fixedpath);
} else {
ErrorLogger::ErrorMessage::FileLocation loc2;
loc2.setfile(Path::toNativeSeparators(FileName));
locationList.push_back(loc2);
loc.setfile(_tokenizer.list.getSourceFilePath());
}
locationList.push_back(loc);
const ErrorLogger::ErrorMessage errmsg(locationList,
Severity::error,
e.errorMessage,
e.id,
false);
reportErr(errmsg);
Timer timerRunChecks((*it)->name() + "::runChecks", _settings._showtime, &S_timerResults);
(*it)->runChecks(&tokenizer, &_settings, this);
}
return true;
// Analyse the tokens..
for (std::list<Check *>::const_iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
Check::FileInfo *fi = (*it)->getFileInfo(&tokenizer, &_settings);
if (fi != nullptr)
fileInfo.push_back(fi);
}
executeRules("normal", tokenizer);
}
//---------------------------------------------------------------------------
// CppCheck - A function that checks a simplified token list
//---------------------------------------------------------------------------
void CppCheck::checkSimplifiedTokens(const Tokenizer &tokenizer)
{
// call all "runSimplifiedChecks" in all registered Check classes
for (std::list<Check *>::const_iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
if (_settings.terminated())
return;
Timer timerSimpleChecks((*it)->name() + "::runSimplifiedChecks", _settings._showtime, &S_timerResults);
(*it)->runSimplifiedChecks(&tokenizer, &_settings, this);
timerSimpleChecks.Stop();
}
if (!_settings.terminated())
executeRules("simple", tokenizer);
}
void CppCheck::executeRules(const std::string &tokenlist, const Tokenizer &tokenizer)

22
lib/cppcheck.h
View File

@ -145,14 +145,22 @@ private:
unsigned int processFile(const std::string& filename, std::istream& fileStream);
/**
* @brief Check file
* @param code
* @param FileName
* @param checksums
* @param[out] internalErrorFound will be set to true if an internal has been caught, false else
* @return false if file has been checked before, true else !?
* @brief Check raw tokens
* @param tokenizer
*/
bool checkFile(const std::string &code, const char FileName[], std::set<unsigned long long>& checksums, bool& internalErrorFound);
void checkRawTokens(const Tokenizer &tokenizer);
/**
* @brief Check normal tokens
* @param tokenizer
*/
void checkNormalTokens(const Tokenizer &tokenizer);
/**
* @brief Check simplified tokens
* @param tokenizer
*/
void checkSimplifiedTokens(const Tokenizer &tokenizer);
/**
* @brief Execute rules, if any

28
lib/tokenize.cpp
View File

@ -1710,23 +1710,24 @@ void Tokenizer::simplifyMulAndParens()
}
}
bool Tokenizer::tokenize(std::istream &code,
const char FileName[],
const std::string &configuration,
bool noSymbolDB_AST)
bool Tokenizer::createTokens(std::istream &code,
const char FileName[])
{
// make sure settings specified
assert(_settings);
return list.createTokens(code, Path::getRelativePath(Path::simplifyPath(FileName), _settings->_basePaths));
}
bool Tokenizer::simplifyTokens1(const std::string &configuration,
bool noSymbolDB_AST)
{
// Fill the map _typeSize..
fillTypeSizes();
_configuration = configuration;
if (!list.createTokens(code, Path::getRelativePath(Path::simplifyPath(FileName), _settings->_basePaths)))
cppcheckError(nullptr);
if (simplifyTokenList1(FileName)) {
if (simplifyTokenList1(list.getFiles()[0].c_str())) {
if (!noSymbolDB_AST) {
createSymbolDatabase();
@ -1754,6 +1755,17 @@ bool Tokenizer::tokenize(std::istream &code,
}
return false;
}
bool Tokenizer::tokenize(std::istream &code,
const char FileName[],
const std::string &configuration,
bool noSymbolDB_AST)
{
if (!createTokens(code, FileName))
return false;
return simplifyTokens1(configuration, noSymbolDB_AST);
}
//---------------------------------------------------------------------------
bool Tokenizer::tokenizeCondition(const std::string &code)

5
lib/tokenize.h
View File

@ -75,6 +75,11 @@ public:
*/
bool IsScopeNoReturn(const Token *endScopeToken, bool *unknown = nullptr) const;
bool createTokens(std::istream &code,
const char FileName[]);
bool simplifyTokens1(const std::string &configuration,
bool noSymbolDB_AST = false);
/**
* Tokenize code
* @param code input stream for code, e.g.