Splitted class TokenList from Tokenizer

lib/check.h

@@ -156,13 +156,13 @@ private:
 
             ErrorLogger::ErrorMessage::FileLocation loc;
             loc.line = (*it)->linenr();
-            loc.setfile(_tokenizer->file(*it));
+            loc.setfile(_tokenizer->list.file(*it));
             locationList.push_back(loc);
         }
 
         ErrorLogger::ErrorMessage errmsg(locationList, severity, msg, id, inconclusive);
-        if (_tokenizer && !_tokenizer->getFiles().empty())
+        if (_tokenizer && !_tokenizer->list.getFiles().empty())
-            errmsg.file0 = _tokenizer->getFiles()[0];
+            errmsg.file0 = _tokenizer->list.getFiles()[0];
         if (_errorLogger)
             _errorLogger->reportErr(errmsg);
         else

lib/checkbufferoverrun.cpp

@@ -1293,7 +1293,7 @@ void CheckBufferOverrun::checkGlobalAndLocalVariable()
             // nextTok : number of tokens used in variable declaration - used to skip to next statement.
             int nextTok = 0;
 
-            _errorLogger->reportProgress(_tokenizer->getFiles().front(),
+            _errorLogger->reportProgress(_tokenizer->getSourceFilePath(),
                                          "Check (BufferOverrun::checkGlobalAndLocalVariable)",
                                          tok->progressValue());
 

lib/checkmemoryleak.cpp

@@ -366,7 +366,7 @@ void CheckMemoryLeak::reportErr(const std::list<const Token *> &callstack, Sever
 
         ErrorLogger::ErrorMessage::FileLocation loc;
         loc.line = tok->linenr();
-        loc.setfile(tokenizer->file(tok));
+        loc.setfile(tokenizer->list.file(tok));
 
         locations.push_back(loc);
     }
@@ -669,7 +669,7 @@ const char * CheckMemoryLeakInFunction::call_func(const Token *tok, std::list<co
             ret = "alloc";
         else if (Token::simpleMatch(func, "; dealloc ; }"))
             ret = "dealloc";
-        Tokenizer::deleteTokens(func);
+        TokenList::deleteTokens(func);
         return ret;
     }
 
@@ -734,7 +734,7 @@ const char * CheckMemoryLeakInFunction::call_func(const Token *tok, std::list<co
             else if (Token::findsimplematch(func_, "&use"))
                 ret = "&use";
 
-            Tokenizer::deleteTokens(func);
+            TokenList::deleteTokens(func);
             return ret;
         }
         if (varid > 0 && Token::Match(tok, "& %varid% [,()]", varid)) {
@@ -889,7 +889,7 @@ Token *CheckMemoryLeakInFunction::getcode(const Token *tok, std::list<const Toke
                 if (Token::Match(tok, "asprintf|vasprintf (")) {
                     // todo: check how the return value is used.
                     if (!Token::Match(tok->previous(), "[;{}]")) {
-                        Tokenizer::deleteTokens(rethead);
+                        TokenList::deleteTokens(rethead);
                         return 0;
                     }
                     alloc = Malloc;
@@ -2071,7 +2071,7 @@ void CheckMemoryLeakInFunction::checkScope(const Token *Tok1, const std::string
 
     // If the variable is not allocated at all => no memory leak
     if (Token::findsimplematch(tok, "alloc") == 0) {
-        Tokenizer::deleteTokens(tok);
+        TokenList::deleteTokens(tok);
         return;
     }
 
@@ -2083,13 +2083,13 @@ void CheckMemoryLeakInFunction::checkScope(const Token *Tok1, const std::string
 
     // If the variable is not allocated at all => no memory leak
     if (Token::findsimplematch(tok, "alloc") == 0) {
-        Tokenizer::deleteTokens(tok);
+        TokenList::deleteTokens(tok);
         return;
     }
 
     /** @todo handle "goto" */
     if (Token::findsimplematch(tok, "goto")) {
-        Tokenizer::deleteTokens(tok);
+        TokenList::deleteTokens(tok);
         return;
     }
 
@@ -2129,7 +2129,7 @@ void CheckMemoryLeakInFunction::checkScope(const Token *Tok1, const std::string
         }
     }
 
-    Tokenizer::deleteTokens(tok);
+    TokenList::deleteTokens(tok);
 }
 //---------------------------------------------------------------------------
 

lib/checkmemoryleak.h

@@ -261,7 +261,7 @@ public:
      * @param classmember should be set if the inspected function is a class member
      * @param sz size of type, used to check for mismatching size of allocation. for example "int *a;" => the sz is "sizeof(int)"
      * @return Newly allocated token array. Caller needs to release reserved
-     * memory by calling Tokenizer::deleteTokens(returnValue);
+     * memory by calling TokenList::deleteTokens(returnValue);
      * Returned tokens:
      * - "alloc" : the variable is allocated
      * - "assign" : the variable is assigned a new value

lib/checkunusedfunctions.cpp

@@ -86,10 +86,10 @@ void CheckUnusedFunctions::parseTokens(const Tokenizer &tokenizer)
 
             // No filename set yet..
             if (func.filename.empty()) {
-                func.filename = tokenizer.getFiles().at(0);
+                func.filename = tokenizer.getSourceFilePath();
             }
             // Multiple files => filename = "+"
-            else if (func.filename != tokenizer.getFiles().at(0)) {
+            else if (func.filename != tokenizer.getSourceFilePath()) {
                 //func.filename = "+";
                 func.usedOtherFile |= func.usedSameFile;
             }

lib/cppcheck.cpp

@@ -316,8 +316,8 @@ void CppCheck::checkFile(const std::string &code, const char FileName[])
         }
 
         // Update the _dependencies..
-        if (_tokenizer.getFiles().size() >= 2)
+        if (_tokenizer.list.getFiles().size() >= 2)
-            _dependencies.insert(_tokenizer.getFiles().begin()+1, _tokenizer.getFiles().end());
+            _dependencies.insert(_tokenizer.list.getFiles().begin()+1, _tokenizer.list.getFiles().end());
 
         // call all "runChecks" in all registered Check classes
         for (std::list<Check *>::iterator it = Check::instances().begin(); it != Check::instances().end(); ++it) {
@@ -423,7 +423,7 @@ void CppCheck::checkFile(const std::string &code, const char FileName[])
         ErrorLogger::ErrorMessage::FileLocation loc;
         if (e.token) {
             loc.line = e.token->linenr();
-            const std::string fixedpath = Path::toNativeSeparators(_tokenizer.file(e.token));
+            const std::string fixedpath = Path::toNativeSeparators(_tokenizer.list.file(e.token));
             loc.setfile(fixedpath);
         } else {
             loc.setfile(_tokenizer.getSourceFilePath());

lib/preprocessor.cpp

@@ -2275,7 +2275,7 @@ public:
 
         // Tokenize the macro to make it easier to handle
         std::istringstream istr(macro);
-        tokenizer.createTokens(istr);
+        tokenizer.list.createTokens(istr);
 
         // macro name..
         if (tokens() && tokens()->isName())

lib/symboldatabase.cpp

@@ -1225,7 +1225,7 @@ void SymbolDatabase::debugMessage(const Token *tok, const std::string &msg) cons
         std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
         ErrorLogger::ErrorMessage::FileLocation loc;
         loc.line = tok->linenr();
-        loc.setfile(_tokenizer->file(tok));
+        loc.setfile(_tokenizer->list.file(tok));
         locationList.push_back(loc);
 
         const ErrorLogger::ErrorMessage errmsg(locationList,
@@ -1288,14 +1288,14 @@ void SymbolDatabase::printVariable(const Variable *var, const char *indent) cons
 {
     std::cout << indent << "_name: " << var->nameToken();
     if (var->nameToken()) {
-        std::cout << " " << var->name() << " " << _tokenizer->fileLine(var->nameToken()) << std::endl;
+        std::cout << " " << var->name() << " " << _tokenizer->list.fileLine(var->nameToken()) << std::endl;
         std::cout << indent << "    varId: " << var->varId() << std::endl;
     } else
         std::cout << std::endl;
     std::cout << indent << "_start: " << var->typeStartToken() << " " << var->typeStartToken()->str()
-              << " " << _tokenizer->fileLine(var->typeStartToken()) << std::endl;;
+              << " " << _tokenizer->list.fileLine(var->typeStartToken()) << std::endl;;
     std::cout << indent << "_end: " << var->typeEndToken() << " " << var->typeEndToken()->str()
-              << " " << _tokenizer->fileLine(var->typeEndToken()) << std::endl;;
+              << " " << _tokenizer->list.fileLine(var->typeEndToken()) << std::endl;;
     std::cout << indent << "_index: " << var->index() << std::endl;
     std::cout << indent << "_access: " <<
               (var->isPublic() ? "Public" :
@@ -1319,7 +1319,7 @@ void SymbolDatabase::printVariable(const Variable *var, const char *indent) cons
     std::cout << indent << "_type: ";
     if (var->type()) {
         std::cout << var->type()->className << " " << var->type()->type << " "
-                  << _tokenizer->fileLine(var->type()->classDef) << std::endl;
+                  << _tokenizer->list.fileLine(var->type()->classDef) << std::endl;
     } else
         std::cout << "none" << std::endl;
 
@@ -1327,7 +1327,7 @@ void SymbolDatabase::printVariable(const Variable *var, const char *indent) cons
     if (var->scope()) {
         std::cout << var->scope()->className << " " << var->scope()->type;
         if (var->scope()->classDef)
-            std::cout << " " << _tokenizer->fileLine(var->scope()->classDef) << std::endl;
+            std::cout << " " << _tokenizer->list.fileLine(var->scope()->classDef) << std::endl;
         else
             std::cout << std::endl;
     } else
@@ -1353,19 +1353,19 @@ void SymbolDatabase::printOut(const char *title) const
         std::cout << "    className: " << scope->className << std::endl;
         std::cout << "    classDef: " << scope->classDef;
         if (scope->classDef)
-            std::cout << " " << scope->classDef->str() << " " << _tokenizer->fileLine(scope->classDef) << std::endl;
+            std::cout << " " << scope->classDef->str() << " " << _tokenizer->list.fileLine(scope->classDef) << std::endl;
         else
             std::cout << std::endl;
 
         std::cout << "    classStart: " << scope->classStart;
         if (scope->classStart)
-            std::cout << " " << scope->classStart->str() << " " << _tokenizer->fileLine(scope->classStart) << std::endl;
+            std::cout << " " << scope->classStart->str() << " " << _tokenizer->list.fileLine(scope->classStart) << std::endl;
         else
             std::cout << std::endl;
 
         std::cout << "    classEnd: " << scope->classEnd;
         if (scope->classEnd)
-            std::cout << " " << scope->classEnd->str() << " " <<  _tokenizer->fileLine(scope->classEnd) << std::endl;
+            std::cout << " " << scope->classEnd->str() << " " <<  _tokenizer->list.fileLine(scope->classEnd) << std::endl;
         else
             std::cout << std::endl;
 
@@ -1375,7 +1375,7 @@ void SymbolDatabase::printOut(const char *title) const
         for (func = scope->functionList.begin(); func != scope->functionList.end(); ++func) {
             std::cout << "    Function: " << &*func << std::endl;
             std::cout << "        name: " << func->tokenDef->str() << " "
-                      << _tokenizer->fileLine(func->tokenDef) << std::endl;
+                      << _tokenizer->list.fileLine(func->tokenDef) << std::endl;
             std::cout << "        type: " << (func->type == Function::eConstructor? "Constructor" :
                                               func->type == Function::eCopyConstructor ? "CopyConstructor" :
                                               func->type == Function::eOperatorEqual ? "OperatorEqual" :
@@ -1396,16 +1396,16 @@ void SymbolDatabase::printOut(const char *title) const
             std::cout << "        isExplicit: " << (func->isExplicit ? "true" : "false") << std::endl;
             std::cout << "        isOperator: " << (func->isOperator ? "true" : "false") << std::endl;
             std::cout << "        retFuncPtr: " << (func->retFuncPtr ? "true" : "false") << std::endl;
-            std::cout << "        tokenDef: " << _tokenizer->fileLine(func->tokenDef) << std::endl;
+            std::cout << "        tokenDef: " << _tokenizer->list.fileLine(func->tokenDef) << std::endl;
-            std::cout << "        argDef: " << _tokenizer->fileLine(func->argDef) << std::endl;
+            std::cout << "        argDef: " << _tokenizer->list.fileLine(func->argDef) << std::endl;
             if (func->hasBody) {
-                std::cout << "        token: " << _tokenizer->fileLine(func->token) << std::endl;
+                std::cout << "        token: " << _tokenizer->list.fileLine(func->token) << std::endl;
-                std::cout << "        arg: " << _tokenizer->fileLine(func->arg) << std::endl;
+                std::cout << "        arg: " << _tokenizer->list.fileLine(func->arg) << std::endl;
             }
             std::cout << "        functionScope: ";
             if (func->functionScope) {
                 std::cout << func->functionScope->className << " "
-                          <<  _tokenizer->fileLine(func->functionScope->classDef) << std::endl;
+                          <<  _tokenizer->list.fileLine(func->functionScope->classDef) << std::endl;
             } else
                 std::cout << "Unknown" << std::endl;
 
@@ -1482,21 +1482,21 @@ void SymbolDatabase::printOut(const char *title) const
                 std::cout << "::" << tok1->strAt(1);
                 tok1 = tok1->tokAt(2);
             }
-            std::cout << " " << _tokenizer->fileLine(*use) << std::endl;
+            std::cout << " " << _tokenizer->list.fileLine(*use) << std::endl;
         }
 
         std::cout << "    functionOf: " << scope->functionOf;
         if (scope->functionOf) {
             std::cout << " " << scope->functionOf->type << " " << scope->functionOf->className;
             if (scope->functionOf->classDef)
-                std::cout << " " << _tokenizer->fileLine(scope->functionOf->classDef);
+                std::cout << " " << _tokenizer->list.fileLine(scope->functionOf->classDef);
         }
         std::cout << std::endl;
 
         std::cout << "    function: " << scope->function;
         if (scope->function) {
             std::cout << " " << scope->function->tokenDef->str() << " "
-                      << _tokenizer->fileLine(scope->function->tokenDef);
+                      << _tokenizer->list.fileLine(scope->function->tokenDef);
         }
         std::cout << std::endl;
     }

lib/templatesimplifier.cpp

@@ -589,7 +589,7 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
 
         // member function implemented outside class definition
         else if (TemplateSimplifier::simplifyTemplatesInstantiateMatch(tok3, name, typeParametersInDeclaration.size(), ":: ~| %var% (")) {
-            tokenizer.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
+            tokenizer.list.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
             while (tok3->str() != "::")
                 tok3 = tok3->next();
         }
@@ -613,8 +613,8 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
                     // the "}" token should only be added if indentlevel is 1 but I add it always intentionally
                     // if indentlevel ever becomes 0, cppcheck will write:
                     // ### Error: Invalid number of character {
-                    tokenizer.addtoken("}", tok3->linenr(), tok3->fileIndex());
+                    tokenizer.list.addtoken("}", tok3->linenr(), tok3->fileIndex());
-                    Token::createMutualLinks(braces.top(), tokenizer._tokensBack);
+                    Token::createMutualLinks(braces.top(), tokenizer.list.back());
                     braces.pop();
                     break;
                 }
@@ -633,7 +633,7 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
                     for (const Token *typetok = typesUsedInTemplateInstantion[itype];
                          typetok && !Token::Match(typetok, "[,>]");
                          typetok = typetok->next()) {
-                        tokenizer.addtoken(typetok, tok3->linenr(), tok3->fileIndex());
+                        tokenizer.list.addtoken(typetok, tok3->linenr(), tok3->fileIndex());
                     }
                     continue;
                 }
@@ -641,36 +641,36 @@ void TemplateSimplifier::simplifyTemplatesExpandTemplate(
 
             // replace name..
             if (Token::Match(tok3, (name + " !!<").c_str())) {
-                tokenizer.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
+                tokenizer.list.addtoken(newName.c_str(), tok3->linenr(), tok3->fileIndex());
                 continue;
             }
 
             // copy
-            tokenizer.addtoken(tok3, tok3->linenr(), tok3->fileIndex());
+            tokenizer.list.addtoken(tok3, tok3->linenr(), tok3->fileIndex());
             if (Token::Match(tok3, "%type% <")) {
                 //if (!Token::simpleMatch(tok3, (name + " <").c_str()))
                 //done = false;
-                templateInstantiations.push_back(tokenizer._tokensBack);
+                templateInstantiations.push_back(tokenizer.list.back());
             }
 
             // link() newly tokens manually
             if (tok3->str() == "{") {
-                braces.push(tokenizer._tokensBack);
+                braces.push(tokenizer.list.back());
             } else if (tok3->str() == "}") {
                 assert(braces.empty() == false);
-                Token::createMutualLinks(braces.top(), tokenizer._tokensBack);
+                Token::createMutualLinks(braces.top(), tokenizer.list.back());
                 braces.pop();
             } else if (tok3->str() == "(") {
-                brackets.push(tokenizer._tokensBack);
+                brackets.push(tokenizer.list.back());
             } else if (tok3->str() == "[") {
-                brackets2.push(tokenizer._tokensBack);
+                brackets2.push(tokenizer.list.back());
             } else if (tok3->str() == ")") {
                 assert(brackets.empty() == false);
-                Token::createMutualLinks(brackets.top(), tokenizer._tokensBack);
+                Token::createMutualLinks(brackets.top(), tokenizer.list.back());
                 brackets.pop();
             } else if (tok3->str() == "]") {
                 assert(brackets2.empty() == false);
-                Token::createMutualLinks(brackets2.top(), tokenizer._tokensBack);
+                Token::createMutualLinks(brackets2.top(), tokenizer.list.back());
                 brackets2.pop();
             }
 
@@ -983,7 +983,7 @@ void TemplateSimplifier::simplifyTemplateInstantions(
     for (std::list<Token *>::const_iterator iter2 = templateInstantiations.begin(); iter2 != templateInstantiations.end(); ++iter2) {
         if (amountOftemplateInstantiations != templateInstantiations.size()) {
             amountOftemplateInstantiations = templateInstantiations.size();
-            simplifyCalculations(tokenizer._tokens);
+            simplifyCalculations(tokenizer.list.front());
             ++recursiveCount;
             if (recursiveCount > 100) {
                 // bail out..
@@ -1103,20 +1103,20 @@ void TemplateSimplifier::simplifyTemplates(
 )
 {
 
-    std::set<std::string> expandedtemplates(TemplateSimplifier::simplifyTemplatesExpandSpecialized(tokenizer._tokens));
+    std::set<std::string> expandedtemplates(TemplateSimplifier::simplifyTemplatesExpandSpecialized(tokenizer.list.front()));
 
     // Locate templates and set member variable _codeWithTemplates if the code has templates.
     // this info is used by checks
-    std::list<Token *> templates(TemplateSimplifier::simplifyTemplatesGetTemplateDeclarations(tokenizer._tokens, _codeWithTemplates));
+    std::list<Token *> templates(TemplateSimplifier::simplifyTemplatesGetTemplateDeclarations(tokenizer.list.front(), _codeWithTemplates));
 
     if (templates.empty()) {
-        TemplateSimplifier::removeTemplates(tokenizer._tokens);
+        TemplateSimplifier::removeTemplates(tokenizer.list.front());
         return;
     }
 
     // There are templates..
     // Remove "typename" unless used in template arguments..
-    for (Token *tok = tokenizer._tokens; tok; tok = tok->next()) {
+    for (Token *tok = tokenizer.list.front(); tok; tok = tok->next()) {
         if (tok->str() == "typename")
             tok->deleteThis();
 
@@ -1129,11 +1129,11 @@ void TemplateSimplifier::simplifyTemplates(
     }
 
     // Locate possible instantiations of templates..
-    std::list<Token *> templateInstantiations(TemplateSimplifier::simplifyTemplatesGetTemplateInstantiations(tokenizer._tokens));
+    std::list<Token *> templateInstantiations(TemplateSimplifier::simplifyTemplatesGetTemplateInstantiations(tokenizer.list.front()));
 
     // No template instantiations? Then remove all templates.
     if (templateInstantiations.empty()) {
-        TemplateSimplifier::removeTemplates(tokenizer._tokens);
+        TemplateSimplifier::removeTemplates(tokenizer.list.front());
         return;
     }
 
@@ -1153,5 +1153,5 @@ void TemplateSimplifier::simplifyTemplates(
         }
     }
 
-    TemplateSimplifier::removeTemplates(tokenizer._tokens);
+    TemplateSimplifier::removeTemplates(tokenizer.list.front());
 }

lib/token.h

@@ -27,7 +27,7 @@
 /// @{
 
 /**
- * @brief The token list that the Tokenizer generates is a linked-list of this class.
+ * @brief The token list that the TokenList generates is a linked-list of this class.
  *
  * Tokens are stored as strings. The "if", "while", etc are stored in plain text.
  * The reason the Token class is needed (instead of using the string class) is that some extra functionality is also needed for tokens:

lib/tokenize.h

@@ -23,13 +23,12 @@
 //---------------------------------------------------------------------------
 
 #include "errorlogger.h"
+#include "tokenlist.h"
 
 #include <string>
 #include <map>
 #include <list>
-#include <vector>
 
-class Token;
 class Settings;
 class SymbolDatabase;
 class TimerResults;
@@ -39,11 +38,6 @@ class TimerResults;
 
 /** @brief The main purpose is to tokenize the source code. It also has functions that simplify the token list */
 class Tokenizer {
-    friend class TemplateSimplifier; // TODO: Remove this. Cleanup interface between Tokenizer and TemplateSimplifier.
-private:
-    /** Deallocate lists */
-    void deallocateTokens();
-
 public:
     Tokenizer();
     Tokenizer(const Settings * settings, ErrorLogger *errorLogger);
@@ -106,16 +100,6 @@ public:
                   const std::string &configuration = "",
                   const bool preprocessorCondition = false);
 
-    /**
-     * Create tokens from code.
-     * The code must be preprocessed first:
-     * - multiline strings are not handled.
-     * - UTF in the code are not handled.
-     * - comments are not handled.
-     * @param code input stream for code
-     */
-    void createTokens(std::istream &code);
-
     /** Set variable id */
     void setVarId();
 
@@ -127,12 +111,6 @@ public:
      */
     bool simplifyTokenList();
 
-    /**
-     * Delete all tokens in given token list
-     * @param tok token list to delete
-     */
-    static void deleteTokens(Token *tok);
-
     /**
      * Deletes dead code between 'begin' and 'end'.
      * In general not everything can be erased, such as:
@@ -166,13 +144,6 @@ public:
      */
     static const char *getParameterName(const Token *ftok, unsigned int par);
 
-    /**
-     * Get file:line for a given token
-     * @param tok given token
-     * @return location for given token
-     */
-    std::string fileLine(const Token *tok) const;
-
     /**
      * Calculates sizeof value for given type.
      * @param type Token which will contain e.g. "int", "*", or string.
@@ -180,13 +151,6 @@ public:
      */
     unsigned int sizeOfType(const Token *type) const;
 
-    /**
-     * Get filenames (the sourcefile + the files it include).
-     * The first filename is the filename for the sourcefile
-     * @return vector with filenames
-     */
-    const std::vector<std::string>& getFiles() const;
-
     /**
      * Get function token by function name
      * @todo better handling of overloaded functions
@@ -195,16 +159,6 @@ public:
      */
     const Token *getFunctionTokenByName(const char funcname[]) const;
 
-    /** get tokens */
-    const Token *tokens() const;
-
-    /**
-     * get filename for given token
-     * @param tok The given token
-     * @return filename for the given token
-     */
-    const std::string& file(const Token *tok) const;
-
     /**
      * get error messages that the tokenizer generate
      */
@@ -397,9 +351,6 @@ public:
     /** Simplify "if else" */
     void elseif();
 
-    void addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split = false);
-    void addtoken(const Token *tok, const unsigned int lineno, const unsigned int fileno);
-
     /**
      * Simplify the operator "?:"
      */
@@ -564,16 +515,6 @@ public:
      */
     void removeExceptionSpecifications(Token *tok) const;
 
-    void insertTokens(Token *dest, const Token *src, unsigned int n);
-
-    /**
-     * Copy tokens.
-     * @param dest destination token where copied tokens will be inserted after
-     * @param first first token to copy
-     * @param last last token to copy
-     * @return new location of last token copied
-     */
-    Token *copyTokens(Token *dest, const Token *first, const Token *last, bool one_line = true);
 
     /**
      * Send error message to error logger about internal bug.
@@ -731,6 +672,7 @@ public:
 
     void setSettings(const Settings *settings) {
         _settings = settings;
+        list.setSettings(settings);
     }
 
     const SymbolDatabase *getSymbolDatabase() const;
@@ -756,6 +698,16 @@ public:
      */
     void printUnknownTypes();
 
+
+    /**
+     * Token list: stores all tokens.
+     */
+    TokenList list;
+    // Implement tokens() as a wrapper for convinience when using the TokenList
+    const Token* tokens() const {
+        return list.front();
+    }
+
 private:
     /** Disable copy constructor, no implementation */
     Tokenizer(const Tokenizer &);
@@ -763,8 +715,14 @@ private:
     /** Disable assignment operator, no implementation */
     Tokenizer &operator=(const Tokenizer &);
 
-    /** Token list */
+    /**
-    Token *_tokens, *_tokensBack;
+     * Copy tokens.
+     * @param dest destination token where copied tokens will be inserted after
+     * @param first first token to copy
+     * @param last last token to copy
+     * @return new location of last token copied
+     */
+    static Token *copyTokens(Token *dest, const Token *first, const Token *last, bool one_line = true);
 
     /** settings */
     const Settings * _settings;
@@ -782,9 +740,6 @@ private:
     /** sizeof information for known types */
     std::map<std::string, unsigned int> _typeSize;
 
-    /** filenames for the tokenized source code (source + included) */
-    std::vector<std::string> _files;
-
     /** variable count */
     unsigned int _varId;
 

lib/tokenlist.cpp

@@ -0,0 +1,351 @@
+/*
+ * Cppcheck - A tool for static C/C++ code analysis
+ * Copyright (C) 2007-2012 Daniel Marjamäki and Cppcheck team.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//---------------------------------------------------------------------------
+#include "tokenlist.h"
+#include "token.h"
+#include "mathlib.h"
+#include "path.h"
+#include "preprocessor.h"
+#include "settings.h"
+
+#include <cstring>
+#include <sstream>
+#include <cctype>
+#include <stack>
+
+
+TokenList::TokenList(const Settings* settings) :
+    _front(0),
+    _back(0),
+    _settings(settings)
+{
+}
+
+TokenList::~TokenList()
+{
+    deallocateTokens();
+}
+
+//---------------------------------------------------------------------------
+
+// Deallocate lists..
+void TokenList::deallocateTokens()
+{
+    deleteTokens(_front);
+    _front = 0;
+    _back = 0;
+    _files.clear();
+}
+
+void TokenList::deleteTokens(Token *tok)
+{
+    while (tok) {
+        Token *next = tok->next();
+        delete tok;
+        tok = next;
+    }
+}
+
+//---------------------------------------------------------------------------
+// add a token.
+//---------------------------------------------------------------------------
+
+void TokenList::addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split)
+{
+    if (str[0] == 0)
+        return;
+
+    // If token contains # characters, split it up
+    if (split && strstr(str, "##")) {
+        std::string temp;
+        for (unsigned int i = 0; str[i]; ++i) {
+            if (strncmp(&str[i], "##", 2) == 0) {
+                addtoken(temp.c_str(), lineno, fileno, false);
+                temp.clear();
+                addtoken("##", lineno, fileno, false);
+                ++i;
+            } else
+                temp += str[i];
+        }
+        addtoken(temp.c_str(), lineno, fileno, false);
+        return;
+    }
+
+    // Replace hexadecimal value with decimal
+    std::ostringstream str2;
+    if (MathLib::isHex(str) || MathLib::isOct(str)) {
+        str2 << MathLib::toLongNumber(str);
+    } else if (strncmp(str, "_Bool", 5) == 0) {
+        str2 << "bool";
+    } else {
+        str2 << str;
+    }
+
+    if (_back) {
+        _back->insertToken(str2.str());
+    } else {
+        _front = new Token(&_back);
+        _back = _front;
+        _back->str(str2.str());
+    }
+
+    _back->linenr(lineno);
+    _back->fileIndex(fileno);
+}
+
+void TokenList::addtoken(const Token * tok, const unsigned int lineno, const unsigned int fileno)
+{
+    if (tok == 0)
+        return;
+
+    if (_back) {
+        _back->insertToken(tok->str());
+    } else {
+        _front = new Token(&_back);
+        _back = _front;
+        _back->str(tok->str());
+    }
+
+    _back->linenr(lineno);
+    _back->fileIndex(fileno);
+    _back->isUnsigned(tok->isUnsigned());
+    _back->isSigned(tok->isSigned());
+    _back->isLong(tok->isLong());
+    _back->isUnused(tok->isUnused());
+}
+//---------------------------------------------------------------------------
+// InsertTokens - Copy and insert tokens
+//---------------------------------------------------------------------------
+
+void TokenList::insertTokens(Token *dest, const Token *src, unsigned int n)
+{
+    std::stack<Token *> link;
+
+    while (n > 0) {
+        dest->insertToken(src->str());
+        dest = dest->next();
+
+        // Set links
+        if (Token::Match(dest, "(|[|{"))
+            link.push(dest);
+        else if (!link.empty() && Token::Match(dest, ")|]|}")) {
+            Token::createMutualLinks(dest, link.top());
+            link.pop();
+        }
+
+        dest->fileIndex(src->fileIndex());
+        dest->linenr(src->linenr());
+        dest->varId(src->varId());
+        dest->type(src->type());
+        dest->isUnsigned(src->isUnsigned());
+        dest->isSigned(src->isSigned());
+        dest->isPointerCompare(src->isPointerCompare());
+        dest->isLong(src->isLong());
+        dest->isUnused(src->isUnused());
+        src  = src->next();
+        --n;
+    }
+}
+
+//---------------------------------------------------------------------------
+// Tokenize - tokenizes a given file.
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokens(std::istream &code, const std::string& file0)
+{
+    _files.push_back(file0);
+
+    // line number in parsed code
+    unsigned int lineno = 1;
+
+    // The current token being parsed
+    std::string CurrentToken;
+
+    // lineNumbers holds line numbers for files in fileIndexes
+    // every time an include file is completely parsed, last item in the vector
+    // is removed and lineno is set to point to that value.
+    std::stack<unsigned int> lineNumbers;
+
+    // fileIndexes holds index for _files vector about currently parsed files
+    // every time an include file is completely parsed, last item in the vector
+    // is removed and FileIndex is set to point to that value.
+    std::stack<unsigned int> fileIndexes;
+
+    // FileIndex. What file in the _files vector is read now?
+    unsigned int FileIndex = 0;
+
+    bool expandedMacro = false;
+
+    // Read one byte at a time from code and create tokens
+    for (char ch = (char)code.get(); code.good(); ch = (char)code.get()) {
+        if (ch == Preprocessor::macroChar) {
+            while (code.peek() == Preprocessor::macroChar)
+                code.get();
+            ch = ' ';
+            expandedMacro = true;
+        } else if (ch == '\n') {
+            expandedMacro = false;
+        }
+
+        // char/string..
+        // multiline strings are not handled. The preprocessor should handle that for us.
+        else if (ch == '\'' || ch == '\"') {
+            std::string line;
+
+            // read char
+            bool special = false;
+            char c = ch;
+            do {
+                // Append token..
+                line += c;
+
+                // Special sequence '\.'
+                if (special)
+                    special = false;
+                else
+                    special = (c == '\\');
+
+                // Get next character
+                c = (char)code.get();
+            } while (code.good() && (special || c != ch));
+            line += ch;
+
+            // Handle #file "file.h"
+            if (CurrentToken == "#file") {
+                // Extract the filename
+                line = line.substr(1, line.length() - 2);
+
+                // Has this file been tokenized already?
+                ++lineno;
+                bool foundOurfile = false;
+                fileIndexes.push(FileIndex);
+                for (unsigned int i = 0; i < _files.size(); ++i) {
+                    if (Path::sameFileName(_files[i], line)) {
+                        // Use this index
+                        foundOurfile = true;
+                        FileIndex = i;
+                    }
+                }
+
+                if (!foundOurfile) {
+                    // The "_files" vector remembers what files have been tokenized..
+                    _files.push_back(Path::simplifyPath(line.c_str()));
+                    FileIndex = static_cast<unsigned int>(_files.size() - 1);
+                }
+
+                lineNumbers.push(lineno);
+                lineno = 0;
+            } else {
+                // Add previous token
+                addtoken(CurrentToken.c_str(), lineno, FileIndex);
+                if (!CurrentToken.empty())
+                    _back->setExpandedMacro(expandedMacro);
+
+                // Add content of the string
+                addtoken(line.c_str(), lineno, FileIndex);
+                if (!line.empty())
+                    _back->setExpandedMacro(expandedMacro);
+            }
+
+            CurrentToken.clear();
+
+            continue;
+        }
+
+        if (ch == '.' &&
+            CurrentToken.length() > 0 &&
+            std::isdigit(CurrentToken[0])) {
+            // Don't separate doubles "5.4"
+        } else if (strchr("+-", ch) &&
+                   CurrentToken.length() > 0 &&
+                   std::isdigit(CurrentToken[0]) &&
+                   (CurrentToken[CurrentToken.length()-1] == 'e' ||
+                    CurrentToken[CurrentToken.length()-1] == 'E') &&
+                   !MathLib::isHex(CurrentToken)) {
+            // Don't separate doubles "4.2e+10"
+        } else if (CurrentToken.empty() && ch == '.' && std::isdigit(code.peek())) {
+            // tokenize .125 into 0.125
+            CurrentToken = "0";
+        } else if (strchr("+-*/%&|^?!=<>[](){};:,.~\n ", ch)) {
+            if (CurrentToken == "#file") {
+                // Handle this where strings are handled
+                continue;
+            } else if (CurrentToken == "#endfile") {
+                if (lineNumbers.empty() || fileIndexes.empty()) { // error
+                    deallocateTokens();
+                    return false;
+                }
+
+                lineno = lineNumbers.top();
+                lineNumbers.pop();
+                FileIndex = fileIndexes.top();
+                fileIndexes.pop();
+                CurrentToken.clear();
+                continue;
+            }
+
+            addtoken(CurrentToken.c_str(), lineno, FileIndex, true);
+            if (!CurrentToken.empty())
+                _back->setExpandedMacro(expandedMacro);
+
+            CurrentToken.clear();
+
+            if (ch == '\n') {
+                ++lineno;
+                continue;
+            } else if (ch == ' ') {
+                continue;
+            }
+
+            CurrentToken += ch;
+            // Add "++", "--", ">>" or ... token
+            if (strchr("+-<>=:&|", ch) && (code.peek() == ch))
+                CurrentToken += (char)code.get();
+            addtoken(CurrentToken.c_str(), lineno, FileIndex);
+            _back->setExpandedMacro(expandedMacro);
+            CurrentToken.clear();
+            continue;
+        }
+
+        CurrentToken += ch;
+    }
+    addtoken(CurrentToken.c_str(), lineno, FileIndex, true);
+    if (!CurrentToken.empty())
+        _back->setExpandedMacro(expandedMacro);
+    _front->assignProgressValues();
+
+    for (unsigned int i = 1; i < _files.size(); i++)
+        _files[i] = Path::getRelativePath(_files[i], _settings->_basePaths);
+
+    return true;
+}
+
+//---------------------------------------------------------------------------
+
+const std::string& TokenList::file(const Token *tok) const
+{
+    return _files.at(tok->fileIndex());
+}
+
+std::string TokenList::fileLine(const Token *tok) const
+{
+    std::ostringstream ostr;
+    ostr << "[" << _files.at(tok->fileIndex()) << ":" << tok->linenr() << "]";
+    return ostr.str();
+}

lib/tokenlist.h

@@ -0,0 +1,130 @@
+/*
+ * Cppcheck - A tool for static C/C++ code analysis
+ * Copyright (C) 2007-2012 Daniel Marjamäki and Cppcheck team.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+//---------------------------------------------------------------------------
+#ifndef tokenlistH
+#define tokenlistH
+//---------------------------------------------------------------------------
+
+#include <string>
+#include <vector>
+
+class Token;
+class Settings;
+
+/// @addtogroup Core
+/// @{
+
+class TokenList {
+public:
+    TokenList(const Settings* settings);
+    ~TokenList();
+
+    void setSettings(const Settings *settings) {
+        _settings = settings;
+    }
+
+    /**
+     * Delete all tokens in given token list
+     * @param tok token list to delete
+     */
+    static void deleteTokens(Token *tok);
+
+    void addtoken(const char str[], const unsigned int lineno, const unsigned int fileno, bool split = false);
+    void addtoken(const Token *tok, const unsigned int lineno, const unsigned int fileno);
+
+    void insertTokens(Token *dest, const Token *src, unsigned int n);
+
+    /**
+     * Create tokens from code.
+     * The code must be preprocessed first:
+     * - multiline strings are not handled.
+     * - UTF in the code are not handled.
+     * - comments are not handled.
+     * @param code input stream for code
+     * @param file0 source file name
+     */
+    bool createTokens(std::istream &code, const std::string& file0 = "");
+
+    /** Deallocate list */
+    void deallocateTokens();
+
+    /** get first token of list */
+    const Token *front() const {
+        return _front;
+    }
+    Token *front() {
+        return _front;
+    }
+
+    /** get last token of list */
+    const Token *back() const {
+        return _back;
+    }
+    Token *back() {
+        return _back;
+    }
+
+    /**
+     * Get filenames (the sourcefile + the files it include).
+     * The first filename is the filename for the sourcefile
+     * @return vector with filenames
+     */
+    const std::vector<std::string>& getFiles() const {
+        return _files;
+    }
+
+    /**
+     * get filename for given token
+     * @param tok The given token
+     * @return filename for the given token
+     */
+    const std::string& file(const Token *tok) const;
+
+    /**
+     * Get file:line for a given token
+     * @param tok given token
+     * @return location for given token
+     */
+    std::string fileLine(const Token *tok) const;
+
+private:
+    /** Disable copy constructor, no implementation */
+    TokenList(const TokenList &);
+
+    /** Disable assignment operator, no implementation */
+    TokenList &operator=(const TokenList &);
+
+public:
+
+private: /// private
+    /** Token list */
+    Token *_front, *_back;
+
+    /** filenames for the tokenized source code (source + included) */
+    std::vector<std::string> _files;
+
+    /** settings */
+    const Settings* _settings;
+};
+
+/// @}
+
+//---------------------------------------------------------------------------
+#endif

test/testmemleak.cpp

@@ -380,7 +380,7 @@ private:
         for (const Token *tok = tokens; tok; tok = tok->next())
             ret << tok->str();
 
-        Tokenizer::deleteTokens(tokens);
+        TokenList::deleteTokens(tokens);
 
         return ret.str();
     }

test/testpreprocessor.cpp

@@ -869,12 +869,6 @@ private:
         Preprocessor preprocessor(&settings, this);
         preprocessor.preprocess(istr, actual, "file.c");
 
-        Tokenizer tok(&settings, this);
-        std::istringstream codeStream(actual[""]);
-        tok.tokenize(codeStream, "main.cpp");
-
-        ASSERT_EQUALS("\n\n##file 0\n1:\n2:\n3:\n4: int main ( ) { }\n", tok.tokens()->stringifyList());
-
         // Expected configurations: "" and "ABC"
         ASSERT_EQUALS(2, static_cast<unsigned int>(actual.size()));
         ASSERT_EQUALS("\n#file \"abc.h\"\n\n\n\n\n\n\n\n\n#endfile\n\nint main() {}\n", actual[""]);

test/testsimplifytokens.cpp

@@ -859,7 +859,6 @@ private:
 
         Tokenizer tokenizer(&settings, this);
         std::istringstream istr(code);
-        tokenizer.createTokens(istr);
         tokenizer.tokenize(istr, "test.cpp");
         tokenizer.elseif();
         return tokenizer.tokens()->stringifyList(false);
@@ -3471,7 +3470,7 @@ private:
         Tokenizer tokenizer(&settings, this);
 
         std::istringstream istr(code);
-        tokenizer.createTokens(istr);
+        tokenizer.list.createTokens(istr);
         tokenizer.createLinks();
         tokenizer.simplifyTypedef();
 

test/testtoken.cpp

@@ -104,11 +104,11 @@ private:
         if (token->previous())
             ASSERT_EQUALS("Null was expected", "");
 
-        Tokenizer::deleteTokens(token);
+        TokenList::deleteTokens(token);
     }
 
     bool Match(const std::string &code, const std::string &pattern, unsigned int varid=0) {
-        const Settings settings;
+        static const Settings settings;
         Tokenizer tokenizer(&settings, this);
         std::istringstream istr(code);
         tokenizer.tokenize(istr, "test.cpp");

test/testtokenize.cpp

@@ -4108,7 +4108,7 @@ private:
         std::istringstream istr(code);
         tokenizer.tokenize(istr, "a.cpp");
 
-        ASSERT_EQUALS("[c:\\a.h:1]", tokenizer.fileLine(tokenizer.tokens()));
+        ASSERT_EQUALS("[c:\\a.h:1]", tokenizer.list.fileLine(tokenizer.tokens()));
     }