walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Splitted class TokenList from Tokenizer

This commit is contained in:
PKEuS 2012-05-05 09:33:26 -07:00
parent 5d088aa99c
commit 1a5fbd61d2
19 changed files with 724 additions and 626 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/check.h
View File

@ -156,13 +156,13 @@ private:
ErrorLogger::ErrorMessage::FileLocation loc;
loc.line = (*it)->linenr();
loc.setfile(_tokenizer->file(*it));
loc.setfile(_tokenizer->list.file(*it));
locationList.push_back(loc);
}
ErrorLogger::ErrorMessage errmsg(locationList, severity, msg, id, inconclusive);
if (_tokenizer && !_tokenizer->getFiles().empty())
errmsg.file0 = _tokenizer->getFiles()[0];
if (_tokenizer && !_tokenizer->list.getFiles().empty())
errmsg.file0 = _tokenizer->list.getFiles()[0];
if (_errorLogger)
_errorLogger->reportErr(errmsg);
else

2
lib/checkbufferoverrun.cpp
View File

@ -1293,7 +1293,7 @@ void CheckBufferOverrun::checkGlobalAndLocalVariable()
// nextTok : number of tokens used in variable declaration - used to skip to next statement.
int nextTok = 0;
_errorLogger->reportProgress(_tokenizer->getFiles().front(),
_errorLogger->reportProgress(_tokenizer->getSourceFilePath(),
"Check (BufferOverrun::checkGlobalAndLocalVariable)",
tok->progressValue());

16
lib/checkmemoryleak.cpp
View File

@ -366,7 +366,7 @@ void CheckMemoryLeak::reportErr(const std::list<const Token *> &callstack, Sever
ErrorLogger::ErrorMessage::FileLocation loc;
loc.line = tok->linenr();
loc.setfile(tokenizer->file(tok));
loc.setfile(tokenizer->list.file(tok));
locations.push_back(loc);
}
@ -669,7 +669,7 @@ const char * CheckMemoryLeakInFunction::call_func(const Token *tok, std::list<co
ret = "alloc";
else if (Token::simpleMatch(func, "; dealloc ; }"))
ret = "dealloc";
Tokenizer::deleteTokens(func);
TokenList::deleteTokens(func);
return ret;
}
@ -734,7 +734,7 @@ const char * CheckMemoryLeakInFunction::call_func(const Token *tok, std::list<co
else if (Token::findsimplematch(func_, "&use"))
ret = "&use";
Tokenizer::deleteTokens(func);
TokenList::deleteTokens(func);
return ret;
}
if (varid > 0 && Token::Match(tok, "& %varid% [,()]", varid)) {
@ -889,7 +889,7 @@ Token *CheckMemoryLeakInFunction::getcode(const Token *tok, std::list<const Toke
if (Token::Match(tok, "asprintf|vasprintf (")) {
// todo: check how the return value is used.
if (!Token::Match(tok->previous(), "[;{}]")) {
Tokenizer::deleteTokens(rethead);
TokenList::deleteTokens(rethead);
return 0;
}
alloc = Malloc;
@ -2071,7 +2071,7 @@ void CheckMemoryLeakInFunction::checkScope(const Token *Tok1, const std::string
// If the variable is not allocated at all => no memory leak
if (Token::findsimplematch(tok, "alloc") == 0) {
Tokenizer::deleteTokens(tok);
TokenList::deleteTokens(tok);
return;
}
@ -2083,13 +2083,13 @@ void CheckMemoryLeakInFunction::checkScope(const Token *Tok1, const std::string
// If the variable is not allocated at all => no memory leak
if (Token::findsimplematch(tok, "alloc") == 0) {
Tokenizer::deleteTokens(tok);
TokenList::deleteTokens(tok);
return;
}
/** @todo handle "goto" */
if (Token::findsimplematch(tok, "goto")) {
Tokenizer::deleteTokens(tok);
TokenList::deleteTokens(tok);
return;
}
@ -2129,7 +2129,7 @@ void CheckMemoryLeakInFunction::checkScope(const Token *Tok1, const std::string
}
}
Tokenizer::deleteTokens(tok);
TokenList::deleteTokens(tok);
}
//---------------------------------------------------------------------------

2
lib/checkmemoryleak.h
View File

@ -261,7 +261,7 @@ public:
* @param classmember should be set if the inspected function is a class member
* @param sz size of type, used to check for mismatching size of allocation. for example "int *a;" => the sz is "sizeof(int)"
* @return Newly allocated token array. Caller needs to release reserved
* memory by calling Tokenizer::deleteTokens(returnValue);
* memory by calling TokenList::deleteTokens(returnValue);
* Returned tokens:
* - "alloc" : the variable is allocated
* - "assign" : the variable is assigned a new value

4
lib/checkunusedfunctions.cpp
View File

@ -86,10 +86,10 @@ void CheckUnusedFunctions::parseTokens(const Tokenizer &tokenizer)
// No filename set yet..
if (func.filename.empty()) {
func.filename = tokenizer.getFiles().at(0);
func.filename = tokenizer.getSourceFilePath();
}
// Multiple files => filename = "+"
else if (func.filename != tokenizer.getFiles().at(0)) {
else if (func.filename != tokenizer.getSourceFilePath()) {
//func.filename = "+";
func.usedOtherFile |= func.usedSameFile;
}

6
lib/cppcheck.cpp
View File

@ -316,8 +316,8 @@ void CppCheck::checkFile(const std::string &code, const char FileName[])
}
// Update the _dependencies..
if (_tokenizer.getFiles().size() >= 2)
_dependencies.insert(_tokenizer.getFiles().begin()+1, _tokenizer.getFiles().end());
if (_tokenizer.list.getFiles().size() >= 2)
_dependencies.insert(_tokenizer.list.getFiles().begin()+1, _tokenizer.list.getFiles().end());
// call all "runChecks" in all registered Check classes
for (std::list<Check *>::iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
@ -423,7 +423,7 @@ void CppCheck::checkFile(const std::string &code, const char FileName[])
ErrorLogger::ErrorMessage::FileLocation loc;
if (e.token) {
loc.line = e.token->linenr();
const std::string fixedpath = Path::toNativeSeparators(_tokenizer.file(e.token));
const std::string fixedpath = Path::toNativeSeparators(_tokenizer.list.file(e.token));
loc.setfile(fixedpath);
} else {
loc.setfile(_tokenizer.getSourceFilePath());

2
lib/preprocessor.cpp
View File

@ -2275,7 +2275,7 @@ public:
// Tokenize the macro to make it easier to handle
std::istringstream istr(macro);
tokenizer.createTokens(istr);
tokenizer.list.createTokens(istr);
// macro name..
if (tokens() && tokens()->isName())

36
lib/symboldatabase.cpp
View File

@ -1225,7 +1225,7 @@ void SymbolDatabase::debugMessage(const Token *tok, const std::string &msg) cons
std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
ErrorLogger::ErrorMessage::FileLocation loc;
loc.line = tok->linenr();
loc.setfile(_tokenizer->file(tok));
loc.setfile(_tokenizer->list.file(tok));
locationList.push_back(loc);
const ErrorLogger::ErrorMessage errmsg(locationList,
@ -1288,14 +1288,14 @@ void SymbolDatabase::printVariable(const Variable *var, const char *indent) cons
{
std::cout << indent << "_name: " << var->nameToken();
if (var->nameToken()) {
std::cout << " " << var->name() << " " << _tokenizer->fileLine(var->nameToken()) << std::endl;
std::cout << " " << var->name() << " " << _tokenizer->list.fileLine(var->nameToken()) << std::endl;
std::cout << indent << " varId: " << var->varId() << std::endl;
} else
std::cout << std::endl;
std::cout << indent << "_start: " << var->typeStartToken() << " " << var->typeStartToken()->str()
<< " " << _tokenizer->fileLine(var->typeStartToken()) << std::endl;;
<< " " << _tokenizer->list.fileLine(var->typeStartToken()) << std::endl;;
std::cout << indent << "_end: " << var->typeEndToken() << " " << var->typeEndToken()->str()
<< " " << _tokenizer->fileLine(var->typeEndToken()) << std::endl;;
<< " " << _tokenizer->list.fileLine(var->typeEndToken()) << std::endl;;
std::cout << indent << "_index: " << var->index() << std::endl;
std::cout << indent << "_access: " <<
(var->isPublic() ? "Public" :
@ -1319,7 +1319,7 @@ void SymbolDatabase::printVariable(const Variable *var, const char *indent) cons
std::cout << indent << "_type: ";
if (var->type()) {
std::cout << var->type()->className << " " << var->type()->type << " "
<< _tokenizer->fileLine(var->type()->classDef) << std::endl;
<< _tokenizer->list.fileLine(var->type()->classDef) << std::endl;
} else
std::cout << "none" << std::endl;
@ -1327,7 +1327,7 @@ void SymbolDatabase::printVariable(const Variable *var, const char *indent) cons
if (var->scope()) {
std::cout << var->scope()->className << " " << var->scope()->type;
if (var->scope()->classDef)
std::cout << " " << _tokenizer->fileLine(var->scope()->classDef) << std::endl;
std::cout << " " << _tokenizer->list.fileLine(var->scope()->classDef) << std::endl;
else
std::cout << std::endl;
} else
@ -1353,19 +1353,19 @@ void SymbolDatabase::printOut(const char *title) const
std::cout << " className: " << scope->className << std::endl;
std::cout << " classDef: " << scope->classDef;
if (scope->classDef)
std::cout << " " << scope->classDef->str() << " " << _tokenizer->fileLine(scope->classDef) << std::endl;
std::cout << " " << scope->classDef->str() << " " << _tokenizer->list.fileLine(scope->classDef) << std::endl;
else
std::cout << std::endl;
std::cout << " classStart: " << scope->classStart;
if (scope->classStart)
std::cout << " " << scope->classStart->str() << " " << _tokenizer->fileLine(scope->classStart) << std::endl;
std::cout << " " << scope->classStart->str() << " " << _tokenizer->list.fileLine(scope->classStart) << std::endl;
else
std::cout << std::endl;
std::cout << " classEnd: " << scope->classEnd;
if (scope->classEnd)
std::cout << " " << scope->classEnd->str() << " " << _tokenizer->fileLine(scope->classEnd) << std::endl;
std::cout << " " << scope->classEnd->str() << " " << _tokenizer->list.fileLine(scope->classEnd) << std::endl;
else
std::cout << std::endl;
@ -1375,7 +1375,7 @@ void SymbolDatabase::printOut(const char *title) const
for (func = scope->functionList.begin(); func != scope->functionList.end(); ++func) {
std::cout << " Function: " << &*func << std::endl;
std::cout << " name: " << func->tokenDef->str() << " "
<< _tokenizer->fileLine(func->tokenDef) << std::endl;
<< _tokenizer->list.fileLine(func->tokenDef) << std::endl;
std::cout << " type: " << (func->type == Function::eConstructor? "Constructor" :
func->type == Function::eCopyConstructor ? "CopyConstructor" :
func->type == Function::eOperatorEqual ? "OperatorEqual" :
@ -1396,16 +1396,16 @@ void SymbolDatabase::printOut(const char *title) const
std::cout << " isExplicit: " << (func->isExplicit ? "true" : "false") << std::endl;
std::cout << " isOperator: " << (func->isOperator ? "true" : "false") << std::endl;
std::cout << " retFuncPtr: " << (func->retFuncPtr ? "true" : "false") << std::endl;
std::cout << " tokenDef: " << _tokenizer->fileLine(func->tokenDef) << std::endl;
std::cout << " argDef: " << _tokenizer->fileLine(func->argDef) << std::endl;
std::cout << " tokenDef: " << _tokenizer->list.fileLine(func->tokenDef) << std::endl;
std::cout << " argDef: " << _tokenizer->list.fileLine(func->argDef) << std::endl;
if (func->hasBody) {
std::cout << " token: " << _tokenizer->fileLine(func->token) << std::endl;
std::cout << " arg: " << _tokenizer->fileLine(func->arg) << std::endl;
std::cout << " token: " << _tokenizer->list.fileLine(func->token) << std::endl;
std::cout << " arg: " << _tokenizer->list.fileLine(func->arg) << std::endl;
}
std::cout << " functionScope: ";
if (func->functionScope) {
std::cout << func->functionScope->className << " "
<< _tokenizer->fileLine(func->functionScope->classDef) << std::endl;
<< _tokenizer->list.fileLine(func->functionScope->classDef) << std::endl;
} else
std::cout << "Unknown" << std::endl;
@ -1482,21 +1482,21 @@ void SymbolDatabase::printOut(const char *title) const
std::cout << "::" << tok1->strAt(1);
tok1 = tok1->tokAt(2);
}
std::cout << " " << _tokenizer->fileLine(*use) << std::endl;
std::cout << " " << _tokenizer->list.fileLine(*use) << std::endl;
}
std::cout << " functionOf: " << scope->functionOf;
if (scope->functionOf) {
std::cout << " " << scope->functionOf->type << " " << scope->functionOf->className;
if (scope->functionOf->classDef)
std::cout << " " << _tokenizer->fileLine(scope->functionOf->classDef);
std::cout << " " << _tokenizer->list.fileLine(scope->functionOf->classDef);
}
std::cout << std::endl;
std::cout << " function: " << scope->function;
if (scope->function) {
std::cout << " " << scope->function->tokenDef->str() << " "
<< _tokenizer->fileLine(scope->function->tokenDef);
<< _tokenizer->list.fileLine(scope->function->tokenDef);
}
std::cout << std::endl;
}

42
lib/templatesimplifier.cpp
View File

@ -589,7 +589,7 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
// member function implemented outside class definition
else if (TemplateSimplifier::simplifyTemplatesInstantiateMatch(tok3, name, typeParametersInDeclaration.size(), ":: ~| %var% (")) {
tokenizer.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
tokenizer.list.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
while (tok3->str() != "::")
tok3 = tok3->next();
}
@ -613,8 +613,8 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
// the "}" token should only be added if indentlevel is 1 but I add it always intentionally
// if indentlevel ever becomes 0, cppcheck will write:
// ### Error: Invalid number of character {
tokenizer.addtoken("}", tok3->linenr(), tok3->fileIndex());
Token::createMutualLinks(braces.top(), tokenizer._tokensBack);
tokenizer.list.addtoken("}", tok3->linenr(), tok3->fileIndex());
Token::createMutualLinks(braces.top(), tokenizer.list.back());
braces.pop();
break;
}
@ -633,7 +633,7 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
for (const Token *typetok = typesUsedInTemplateInstantion[itype];
typetok && !Token::Match(typetok, "[,>]");
typetok = typetok->next()) {
tokenizer.addtoken(typetok, tok3->linenr(), tok3->fileIndex());
tokenizer.list.addtoken(typetok, tok3->linenr(), tok3->fileIndex());
}
continue;
}
@ -641,36 +641,36 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
// replace name..
if (Token::Match(tok3, (name + " !!<").c_str())) {
tokenizer.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
tokenizer.list.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
continue;
}
// copy
tokenizer.addtoken(tok3, tok3->linenr(), tok3->fileIndex());
tokenizer.list.addtoken(tok3, tok3->linenr(), tok3->fileIndex());
if (Token::Match(tok3, "%type% <")) {
//if (!Token::simpleMatch(tok3, (name + " <").c_str()))
//done = false;
templateInstantiations.push_back(tokenizer._tokensBack);
templateInstantiations.push_back(tokenizer.list.back());
}
// link() newly tokens manually
if (tok3->str() == "{") {
braces.push(tokenizer._tokensBack);
braces.push(tokenizer.list.back());
} else if (tok3->str() == "}") {
assert(braces.empty() == false);
Token::createMutualLinks(braces.top(), tokenizer._tokensBack);
Token::createMutualLinks(braces.top(), tokenizer.list.back());
braces.pop();
} else if (tok3->str() == "(") {
brackets.push(tokenizer._tokensBack);
brackets.push(tokenizer.list.back());
} else if (tok3->str() == "[") {
brackets2.push(tokenizer._tokensBack);
brackets2.push(tokenizer.list.back());
} else if (tok3->str() == ")") {
assert(brackets.empty() == false);
Token::createMutualLinks(brackets.top(), tokenizer._tokensBack);
Token::createMutualLinks(brackets.top(), tokenizer.list.back());
brackets.pop();
} else if (tok3->str() == "]") {
assert(brackets2.empty() == false);
Token::createMutualLinks(brackets2.top(), tokenizer._tokensBack);
Token::createMutualLinks(brackets2.top(), tokenizer.list.back());
brackets2.pop();
}
@ -983,7 +983,7 @@ void TemplateSimplifier::simplifyTemplateInstantions(
for (std::list<Token *>::const_iterator iter2 = templateInstantiations.begin(); iter2 != templateInstantiations.end(); ++iter2) {
if (amountOftemplateInstantiations != templateInstantiations.size()) {
amountOftemplateInstantiations = templateInstantiations.size();
simplifyCalculations(tokenizer._tokens);
simplifyCalculations(tokenizer.list.front());
++recursiveCount;
if (recursiveCount > 100) {
// bail out..
@ -1103,20 +1103,20 @@ void TemplateSimplifier::simplifyTemplates(
)
{
std::set<std::string> expandedtemplates(TemplateSimplifier::simplifyTemplatesExpandSpecialized(tokenizer._tokens));
std::set<std::string> expandedtemplates(TemplateSimplifier::simplifyTemplatesExpandSpecialized(tokenizer.list.front()));
// Locate templates and set member variable _codeWithTemplates if the code has templates.
// this info is used by checks
std::list<Token *> templates(TemplateSimplifier::simplifyTemplatesGetTemplateDeclarations(tokenizer._tokens, _codeWithTemplates));
std::list<Token *> templates(TemplateSimplifier::simplifyTemplatesGetTemplateDeclarations(tokenizer.list.front(), _codeWithTemplates));
if (templates.empty()) {
TemplateSimplifier::removeTemplates(tokenizer._tokens);
TemplateSimplifier::removeTemplates(tokenizer.list.front());
return;
}
// There are templates..
// Remove "typename" unless used in template arguments..
for (Token *tok = tokenizer._tokens; tok; tok = tok->next()) {
for (Token *tok = tokenizer.list.front(); tok; tok = tok->next()) {
if (tok->str() == "typename")
tok->deleteThis();
@ -1129,11 +1129,11 @@ void TemplateSimplifier::simplifyTemplates(
}
// Locate possible instantiations of templates..
std::list<Token *> templateInstantiations(TemplateSimplifier::simplifyTemplatesGetTemplateInstantiations(tokenizer._tokens));
std::list<Token *> templateInstantiations(TemplateSimplifier::simplifyTemplatesGetTemplateInstantiations(tokenizer.list.front()));
// No template instantiations? Then remove all templates.
if (templateInstantiations.empty()) {
TemplateSimplifier::removeTemplates(tokenizer._tokens);
TemplateSimplifier::removeTemplates(tokenizer.list.front());
return;
}
@ -1153,5 +1153,5 @@ void TemplateSimplifier::simplifyTemplates(
}
}
TemplateSimplifier::removeTemplates(tokenizer._tokens);
TemplateSimplifier::removeTemplates(tokenizer.list.front());
}

2
lib/token.h
View File

@ -27,7 +27,7 @@
/// @{
/**
* @brief The token list that the Tokenizer generates is a linked-list of this class.
* @brief The token list that the TokenList generates is a linked-list of this class.
*
* Tokens are stored as strings. The "if", "while", etc are stored in plain text.
* The reason the Token class is needed (instead of using the string class) is that some extra functionality is also needed for tokens:

649
lib/tokenize.cpp
View File

File diff suppressed because it is too large Load Diff

85
lib/tokenize.h
View File

@ -23,13 +23,12 @@
//---------------------------------------------------------------------------
#include "errorlogger.h"
#include "tokenlist.h"
#include <string>
#include <map>
#include <list>
#include <vector>
class Token;
class Settings;
class SymbolDatabase;
class TimerResults;
@ -39,11 +38,6 @@ class TimerResults;
/** @brief The main purpose is to tokenize the source code. It also has functions that simplify the token list */
class Tokenizer {
friend class TemplateSimplifier; // TODO: Remove this. Cleanup interface between Tokenizer and TemplateSimplifier.
private:
/** Deallocate lists */
void deallocateTokens();
public:
Tokenizer();
Tokenizer(const Settings * settings, ErrorLogger *errorLogger);
@ -106,16 +100,6 @@ public:
const std::string &configuration = "",
const bool preprocessorCondition = false);
/**
* Create tokens from code.
* The code must be preprocessed first:
* - multiline strings are not handled.
* - UTF in the code are not handled.
* - comments are not handled.
* @param code input stream for code
*/
void createTokens(std::istream &code);
/** Set variable id */
void setVarId();
@ -127,12 +111,6 @@ public:
*/
bool simplifyTokenList();
/**
* Delete all tokens in given token list
* @param tok token list to delete
*/
static void deleteTokens(Token *tok);
/**
* Deletes dead code between 'begin' and 'end'.
* In general not everything can be erased, such as:
@ -166,13 +144,6 @@ public:
*/
static const char *getParameterName(const Token *ftok, unsigned int par);
/**
* Get file:line for a given token
* @param tok given token
* @return location for given token
*/
std::string fileLine(const Token *tok) const;
/**
* Calculates sizeof value for given type.
* @param type Token which will contain e.g. "int", "*", or string.
@ -180,13 +151,6 @@ public:
*/
unsigned int sizeOfType(const Token *type) const;
/**
* Get filenames (the sourcefile + the files it include).
* The first filename is the filename for the sourcefile
* @return vector with filenames
*/
const std::vector<std::string>& getFiles() const;
/**
* Get function token by function name
* @todo better handling of overloaded functions
@ -195,16 +159,6 @@ public:
*/
const Token *getFunctionTokenByName(const char funcname[]) const;
/** get tokens */
const Token *tokens() const;
/**
* get filename for given token
* @param tok The given token
* @return filename for the given token
*/
const std::string& file(const Token *tok) const;
/**
* get error messages that the tokenizer generate
*/
@ -397,9 +351,6 @@ public:
/** Simplify "if else" */
void elseif();
void addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split = false);
void addtoken(const Token *tok, const unsigned int lineno, const unsigned int fileno);
/**
* Simplify the operator "?:"
*/
@ -564,16 +515,6 @@ public:
*/
void removeExceptionSpecifications(Token *tok) const;
void insertTokens(Token *dest, const Token *src, unsigned int n);
/**
* Copy tokens.
* @param dest destination token where copied tokens will be inserted after
* @param first first token to copy
* @param last last token to copy
* @return new location of last token copied
*/
Token *copyTokens(Token *dest, const Token *first, const Token *last, bool one_line = true);
/**
* Send error message to error logger about internal bug.
@ -731,6 +672,7 @@ public:
void setSettings(const Settings *settings) {
_settings = settings;
list.setSettings(settings);
}
const SymbolDatabase *getSymbolDatabase() const;
@ -756,6 +698,16 @@ public:
*/
void printUnknownTypes();
/**
* Token list: stores all tokens.
*/
TokenList list;
// Implement tokens() as a wrapper for convinience when using the TokenList
const Token* tokens() const {
return list.front();
}
private:
/** Disable copy constructor, no implementation */
Tokenizer(const Tokenizer &);
@ -763,8 +715,14 @@ private:
/** Disable assignment operator, no implementation */
Tokenizer &operator=(const Tokenizer &);
/** Token list */
Token *_tokens, *_tokensBack;
/**
* Copy tokens.
* @param dest destination token where copied tokens will be inserted after
* @param first first token to copy
* @param last last token to copy
* @return new location of last token copied
*/
static Token *copyTokens(Token *dest, const Token *first, const Token *last, bool one_line = true);
/** settings */
const Settings * _settings;
@ -782,9 +740,6 @@ private:
/** sizeof information for known types */
std::map<std::string, unsigned int> _typeSize;
/** filenames for the tokenized source code (source + included) */
std::vector<std::string> _files;
/** variable count */
unsigned int _varId;

351
lib/tokenlist.cpp Normal file
View File

@ -0,0 +1,351 @@
/*
* Cppcheck - A tool for static C/C++ code analysis
* Copyright (C) 2007-2012 Daniel Marjamäki and Cppcheck team.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//---------------------------------------------------------------------------
#include "tokenlist.h"
#include "token.h"
#include "mathlib.h"
#include "path.h"
#include "preprocessor.h"
#include "settings.h"
#include <cstring>
#include <sstream>
#include <cctype>
#include <stack>
TokenList::TokenList(const Settings* settings) :
_front(0),
_back(0),
_settings(settings)
{
}
TokenList::~TokenList()
{
deallocateTokens();
}
//---------------------------------------------------------------------------
// Deallocate lists..
void TokenList::deallocateTokens()
{
deleteTokens(_front);
_front = 0;
_back = 0;
_files.clear();
}
void TokenList::deleteTokens(Token *tok)
{
while (tok) {
Token *next = tok->next();
delete tok;
tok = next;
}
}
//---------------------------------------------------------------------------
// add a token.
//---------------------------------------------------------------------------
void TokenList::addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split)
{
if (str[0] == 0)
return;
// If token contains # characters, split it up
if (split && strstr(str, "##")) {
std::string temp;
for (unsigned int i = 0; str[i]; ++i) {
if (strncmp(&str[i], "##", 2) == 0) {
addtoken(temp.c_str(), lineno, fileno, false);
temp.clear();
addtoken("##", lineno, fileno, false);
++i;
} else
temp += str[i];
}
addtoken(temp.c_str(), lineno, fileno, false);
return;
}
// Replace hexadecimal value with decimal
std::ostringstream str2;
if (MathLib::isHex(str) || MathLib::isOct(str)) {
str2 << MathLib::toLongNumber(str);
} else if (strncmp(str, "_Bool", 5) == 0) {
str2 << "bool";
} else {
str2 << str;
}
if (_back) {
_back->insertToken(str2.str());
} else {
_front = new Token(&_back);
_back = _front;
_back->str(str2.str());
}
_back->linenr(lineno);
_back->fileIndex(fileno);
}
void TokenList::addtoken(const Token * tok, const unsigned int lineno, const unsigned int fileno)
{
if (tok == 0)
return;
if (_back) {
_back->insertToken(tok->str());
} else {
_front = new Token(&_back);
_back = _front;
_back->str(tok->str());
}
_back->linenr(lineno);
_back->fileIndex(fileno);
_back->isUnsigned(tok->isUnsigned());
_back->isSigned(tok->isSigned());
_back->isLong(tok->isLong());
_back->isUnused(tok->isUnused());
}
//---------------------------------------------------------------------------
// InsertTokens - Copy and insert tokens
//---------------------------------------------------------------------------
void TokenList::insertTokens(Token *dest, const Token *src, unsigned int n)
{
std::stack<Token *> link;
while (n > 0) {
dest->insertToken(src->str());
dest = dest->next();
// Set links
if (Token::Match(dest, "(|[|{"))
link.push(dest);
else if (!link.empty() && Token::Match(dest, ")|]|}")) {
Token::createMutualLinks(dest, link.top());
link.pop();
}
dest->fileIndex(src->fileIndex());
dest->linenr(src->linenr());
dest->varId(src->varId());
dest->type(src->type());
dest->isUnsigned(src->isUnsigned());
dest->isSigned(src->isSigned());
dest->isPointerCompare(src->isPointerCompare());
dest->isLong(src->isLong());
dest->isUnused(src->isUnused());
src = src->next();
--n;
}
}
//---------------------------------------------------------------------------
// Tokenize - tokenizes a given file.
//---------------------------------------------------------------------------
bool TokenList::createTokens(std::istream &code, const std::string& file0)
{
_files.push_back(file0);
// line number in parsed code
unsigned int lineno = 1;
// The current token being parsed
std::string CurrentToken;
// lineNumbers holds line numbers for files in fileIndexes
// every time an include file is completely parsed, last item in the vector
// is removed and lineno is set to point to that value.
std::stack<unsigned int> lineNumbers;
// fileIndexes holds index for _files vector about currently parsed files
// every time an include file is completely parsed, last item in the vector
// is removed and FileIndex is set to point to that value.
std::stack<unsigned int> fileIndexes;
// FileIndex. What file in the _files vector is read now?
unsigned int FileIndex = 0;
bool expandedMacro = false;
// Read one byte at a time from code and create tokens
for (char ch = (char)code.get(); code.good(); ch = (char)code.get()) {
if (ch == Preprocessor::macroChar) {
while (code.peek() == Preprocessor::macroChar)
code.get();
ch = ' ';
expandedMacro = true;
} else if (ch == '\n') {
expandedMacro = false;
}
// char/string..
// multiline strings are not handled. The preprocessor should handle that for us.
else if (ch == '\'' || ch == '\"') {
std::string line;
// read char
bool special = false;
char c = ch;
do {
// Append token..
line += c;
// Special sequence '\.'
if (special)
special = false;
else
special = (c == '\\');
// Get next character
c = (char)code.get();
} while (code.good() && (special || c != ch));
line += ch;
// Handle #file "file.h"
if (CurrentToken == "#file") {
// Extract the filename
line = line.substr(1, line.length() - 2);
// Has this file been tokenized already?
++lineno;
bool foundOurfile = false;
fileIndexes.push(FileIndex);
for (unsigned int i = 0; i < _files.size(); ++i) {
if (Path::sameFileName(_files[i], line)) {
// Use this index
foundOurfile = true;
FileIndex = i;
}
}
if (!foundOurfile) {
// The "_files" vector remembers what files have been tokenized..
_files.push_back(Path::simplifyPath(line.c_str()));
FileIndex = static_cast<unsigned int>(_files.size() - 1);
}
lineNumbers.push(lineno);
lineno = 0;
} else {
// Add previous token
addtoken(CurrentToken.c_str(), lineno, FileIndex);
if (!CurrentToken.empty())
_back->setExpandedMacro(expandedMacro);
// Add content of the string
addtoken(line.c_str(), lineno, FileIndex);
if (!line.empty())
_back->setExpandedMacro(expandedMacro);
}
CurrentToken.clear();
continue;
}
if (ch == '.' &&
CurrentToken.length() > 0 &&
std::isdigit(CurrentToken[0])) {
// Don't separate doubles "5.4"
} else if (strchr("+-", ch) &&
CurrentToken.length() > 0 &&
std::isdigit(CurrentToken[0]) &&
(CurrentToken[CurrentToken.length()-1] == 'e' ||
CurrentToken[CurrentToken.length()-1] == 'E') &&
!MathLib::isHex(CurrentToken)) {
// Don't separate doubles "4.2e+10"
} else if (CurrentToken.empty() && ch == '.' && std::isdigit(code.peek())) {
// tokenize .125 into 0.125
CurrentToken = "0";
} else if (strchr("+-*/%&|^?!=<>[](){};:,.~\n ", ch)) {
if (CurrentToken == "#file") {
// Handle this where strings are handled
continue;
} else if (CurrentToken == "#endfile") {
if (lineNumbers.empty() || fileIndexes.empty()) { // error
deallocateTokens();
return false;
}
lineno = lineNumbers.top();
lineNumbers.pop();
FileIndex = fileIndexes.top();
fileIndexes.pop();
CurrentToken.clear();
continue;
}
addtoken(CurrentToken.c_str(), lineno, FileIndex, true);
if (!CurrentToken.empty())
_back->setExpandedMacro(expandedMacro);
CurrentToken.clear();
if (ch == '\n') {
++lineno;
continue;
} else if (ch == ' ') {
continue;
}
CurrentToken += ch;
// Add "++", "--", ">>" or ... token
if (strchr("+-<>=:&|", ch) && (code.peek() == ch))
CurrentToken += (char)code.get();
addtoken(CurrentToken.c_str(), lineno, FileIndex);
_back->setExpandedMacro(expandedMacro);
CurrentToken.clear();
continue;
}
CurrentToken += ch;
}
addtoken(CurrentToken.c_str(), lineno, FileIndex, true);
if (!CurrentToken.empty())
_back->setExpandedMacro(expandedMacro);
_front->assignProgressValues();
for (unsigned int i = 1; i < _files.size(); i++)
_files[i] = Path::getRelativePath(_files[i], _settings->_basePaths);
return true;
}
//---------------------------------------------------------------------------
const std::string& TokenList::file(const Token *tok) const
{
return _files.at(tok->fileIndex());
}
std::string TokenList::fileLine(const Token *tok) const
{
std::ostringstream ostr;
ostr << "[" << _files.at(tok->fileIndex()) << ":" << tok->linenr() << "]";
return ostr.str();
}

130
lib/tokenlist.h Normal file
View File

@ -0,0 +1,130 @@
/*
* Cppcheck - A tool for static C/C++ code analysis
* Copyright (C) 2007-2012 Daniel Marjamäki and Cppcheck team.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//---------------------------------------------------------------------------
#ifndef tokenlistH
#define tokenlistH
//---------------------------------------------------------------------------
#include <string>
#include <vector>
class Token;
class Settings;
/// @addtogroup Core
/// @{
class TokenList {
public:
TokenList(const Settings* settings);
~TokenList();
void setSettings(const Settings *settings) {
_settings = settings;
}
/**
* Delete all tokens in given token list
* @param tok token list to delete
*/
static void deleteTokens(Token *tok);
void addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split = false);
void addtoken(const Token *tok, const unsigned int lineno, const unsigned int fileno);
void insertTokens(Token *dest, const Token *src, unsigned int n);
/**
* Create tokens from code.
* The code must be preprocessed first:
* - multiline strings are not handled.
* - UTF in the code are not handled.
* - comments are not handled.
* @param code input stream for code
* @param file0 source file name
*/
bool createTokens(std::istream &code, const std::string& file0 = "");
/** Deallocate list */
void deallocateTokens();
/** get first token of list */
const Token *front() const {
return _front;
}
Token *front() {
return _front;
}
/** get last token of list */
const Token *back() const {
return _back;
}
Token *back() {
return _back;
}
/**
* Get filenames (the sourcefile + the files it include).
* The first filename is the filename for the sourcefile
* @return vector with filenames
*/
const std::vector<std::string>& getFiles() const {
return _files;
}
/**
* get filename for given token
* @param tok The given token
* @return filename for the given token
*/
const std::string& file(const Token *tok) const;
/**
* Get file:line for a given token
* @param tok given token
* @return location for given token
*/
std::string fileLine(const Token *tok) const;
private:
/** Disable copy constructor, no implementation */
TokenList(const TokenList &);
/** Disable assignment operator, no implementation */
TokenList &operator=(const TokenList &);
public:
private: /// private
/** Token list */
Token *_front, *_back;
/** filenames for the tokenized source code (source + included) */
std::vector<std::string> _files;
/** settings */
const Settings* _settings;
};
/// @}
//---------------------------------------------------------------------------
#endif

2
test/testmemleak.cpp
View File

@ -380,7 +380,7 @@ private:
for (const Token *tok = tokens; tok; tok = tok->next())
ret << tok->str();
Tokenizer::deleteTokens(tokens);
TokenList::deleteTokens(tokens);
return ret.str();
}

6
test/testpreprocessor.cpp
View File

@ -869,12 +869,6 @@ private:
Preprocessor preprocessor(&settings, this);
preprocessor.preprocess(istr, actual, "file.c");
Tokenizer tok(&settings, this);
std::istringstream codeStream(actual[""]);
tok.tokenize(codeStream, "main.cpp");
ASSERT_EQUALS("\n\n##file 0\n1:\n2:\n3:\n4: int main ( ) { }\n", tok.tokens()->stringifyList());
// Expected configurations: "" and "ABC"
ASSERT_EQUALS(2, static_cast<unsigned int>(actual.size()));
ASSERT_EQUALS("\n#file \"abc.h\"\n\n\n\n\n\n\n\n\n#endfile\n\nint main() {}\n", actual[""]);

3
test/testsimplifytokens.cpp
View File

@ -859,7 +859,6 @@ private:
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.createTokens(istr);
tokenizer.tokenize(istr, "test.cpp");
tokenizer.elseif();
return tokenizer.tokens()->stringifyList(false);
@ -3471,7 +3470,7 @@ private:
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.createTokens(istr);
tokenizer.list.createTokens(istr);
tokenizer.createLinks();
tokenizer.simplifyTypedef();

4
test/testtoken.cpp
View File

@ -104,11 +104,11 @@ private:
if (token->previous())
ASSERT_EQUALS("Null was expected", "");
Tokenizer::deleteTokens(token);
TokenList::deleteTokens(token);
}
bool Match(const std::string &code, const std::string &pattern, unsigned int varid=0) {
const Settings settings;
static const Settings settings;
Tokenizer tokenizer(&settings, this);
std::istringstream istr(code);
tokenizer.tokenize(istr, "test.cpp");

2
test/testtokenize.cpp
View File

@ -4108,7 +4108,7 @@ private:
std::istringstream istr(code);
tokenizer.tokenize(istr, "a.cpp");
ASSERT_EQUALS("[c:\\a.h:1]", tokenizer.fileLine(tokenizer.tokens()));
ASSERT_EQUALS("[c:\\a.h:1]", tokenizer.list.fileLine(tokenizer.tokens()));
}