From aa579911ab5c8cae5ff4c17002f6fc404845b9e6 Mon Sep 17 00:00:00 2001
From: Reijo Tomperi <aggro80@users.sourceforge.net>
Date: Sun, 16 Nov 2008 15:58:52 +0000
Subject: [PATCH] Refactoring: Global variable tokenes is no more.

---
 CheckBufferOverrun.cpp |  8 ++--
 CheckClass.cpp         | 30 +++++++--------
 CheckHeaders.cpp       |  8 ++--
 CheckMemoryLeak.cpp    | 10 ++---
 CheckOther.cpp         | 36 +++++++++---------
 testsimplifytokens.cpp |  2 +-
 testtokenize.cpp       |  6 +--
 tokenize.cpp           | 85 +++++++++++++++++++++---------------------
 tokenize.h             |  4 +-
 9 files changed, 95 insertions(+), 94 deletions(-)

diff --git a/CheckBufferOverrun.cpp b/CheckBufferOverrun.cpp
index 9086bcf77..53076d0ea 100644
--- a/CheckBufferOverrun.cpp
+++ b/CheckBufferOverrun.cpp
@@ -308,7 +308,7 @@ void CheckBufferOverrunClass::CheckBufferOverrun_CheckScope( const TOKEN *tok, c
 void CheckBufferOverrunClass::CheckBufferOverrun_LocalVariable()
 {
     int indentlevel = 0;
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (Match(tok, "{"))
             indentlevel++;
@@ -359,7 +359,7 @@ void CheckBufferOverrunClass::CheckBufferOverrun_LocalVariable()
 void CheckBufferOverrunClass::CheckBufferOverrun_StructVariable()
 {
     const char *declstruct_pattern[] = {"","","{",0};
-    for ( const TOKEN * tok = Tokenizer::findtoken( tokens, declstruct_pattern );
+    for ( const TOKEN * tok = Tokenizer::findtoken( _tokenizer->tokens(), declstruct_pattern );
           tok;
           tok = Tokenizer::findtoken( tok->next, declstruct_pattern ) )
     {
@@ -399,7 +399,7 @@ void CheckBufferOverrunClass::CheckBufferOverrun_StructVariable()
             if (total_size == 0)
                 continue;
 
-            for ( const TOKEN *tok3 = tokens; tok3; tok3 = tok3->next )
+            for ( const TOKEN *tok3 = _tokenizer->tokens(); tok3; tok3 = tok3->next )
             {
                 if ( strcmp(tok3->str, structname) )
                     continue;
@@ -472,7 +472,7 @@ void CheckBufferOverrunClass::CheckBufferOverrun()
 
 void CheckBufferOverrunClass::WarningDangerousFunctions()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (Match(tok, "gets ("))
         {
diff --git a/CheckClass.cpp b/CheckClass.cpp
index ced8a18ea..55cdf1e31 100644
--- a/CheckClass.cpp
+++ b/CheckClass.cpp
@@ -311,7 +311,7 @@ void CheckClass::CheckConstructors()
 {
     // Locate class
     const char *pattern_classname[] = {"class","","{",NULL};
-    const TOKEN *tok1 = Tokenizer::findtoken(tokens, pattern_classname);
+    const TOKEN *tok1 = Tokenizer::findtoken(_tokenizer->tokens(), pattern_classname);
     while (tok1)
     {
         const char *classname = tok1->next->str;
@@ -324,7 +324,7 @@ void CheckClass::CheckConstructors()
         // Are there a class constructor?
         const char *constructor_pattern[] = {"","clKalle","(",NULL};
         constructor_pattern[1] = classname;
-        const TOKEN *constructor_token = Tokenizer::findtoken( tokens, constructor_pattern );
+        const TOKEN *constructor_token = Tokenizer::findtoken( _tokenizer->tokens(), constructor_pattern );
         while ( constructor_token && constructor_token->str[0] == '~' )
             constructor_token = Tokenizer::findtoken( constructor_token->next, constructor_pattern );
         if ( ! constructor_token )
@@ -370,7 +370,7 @@ void CheckClass::CheckConstructors()
                 const char *pattern[] = {"","::","","=",NULL};
                 pattern[0] = classname;
                 pattern[2] = var->name;
-                if (Tokenizer::findtoken(tokens, pattern))
+                if (Tokenizer::findtoken(_tokenizer->tokens(), pattern))
                     continue;
 
                 if (!var->init)
@@ -412,7 +412,7 @@ void CheckClass::CheckUnusedPrivateFunctions()
 {
     // Locate some class
     const char *pattern_class[] = {"class","","{",NULL};
-    for (const TOKEN *tok1 = Tokenizer::findtoken(tokens, pattern_class); tok1; tok1 = Tokenizer::findtoken(tok1->next, pattern_class))
+    for (const TOKEN *tok1 = Tokenizer::findtoken(_tokenizer->tokens(), pattern_class); tok1; tok1 = Tokenizer::findtoken(tok1->next, pattern_class))
     {
         const char *classname = tok1->next->str;
 
@@ -420,7 +420,7 @@ void CheckClass::CheckUnusedPrivateFunctions()
         const char *pattern_classconstructor[] = {"","::","",NULL};
         pattern_classconstructor[0] = classname;
         pattern_classconstructor[2] = classname;
-        if (!Tokenizer::findtoken(tokens,pattern_classconstructor))
+        if (!Tokenizer::findtoken(_tokenizer->tokens(),pattern_classconstructor))
             continue;
 
         // Get private functions..
@@ -470,7 +470,7 @@ void CheckClass::CheckUnusedPrivateFunctions()
         const char *pattern_function[] = {"","::",NULL};
         pattern_function[0] = classname;
         bool HasFuncImpl = false;
-        const TOKEN *ftok = tokens;
+        const TOKEN *ftok = _tokenizer->tokens();
         while (ftok)
         {
             ftok = Tokenizer::findtoken(ftok,pattern_function);
@@ -519,15 +519,15 @@ void CheckClass::CheckUnusedPrivateFunctions()
             // Final check; check if the function pointer is used somewhere..
             const char *_pattern[] = {"=","",NULL};
             _pattern[1] = FuncList.front().c_str();
-            fp |= (Tokenizer::findtoken(tokens, _pattern) != NULL);
+            fp |= (Tokenizer::findtoken(_tokenizer->tokens(), _pattern) != NULL);
             _pattern[0] = "return";
-            fp |= (Tokenizer::findtoken(tokens, _pattern) != NULL);
+            fp |= (Tokenizer::findtoken(_tokenizer->tokens(), _pattern) != NULL);
             _pattern[0] = "(";
-            fp |= (Tokenizer::findtoken(tokens, _pattern) != NULL);
+            fp |= (Tokenizer::findtoken(_tokenizer->tokens(), _pattern) != NULL);
             _pattern[0] = ")";
-            fp |= (Tokenizer::findtoken(tokens, _pattern) != NULL);
+            fp |= (Tokenizer::findtoken(_tokenizer->tokens(), _pattern) != NULL);
             _pattern[0] = ",";
-            fp |= (Tokenizer::findtoken(tokens, _pattern) != NULL);
+            fp |= (Tokenizer::findtoken(_tokenizer->tokens(), _pattern) != NULL);
 
             if (!fp)
             {
@@ -547,7 +547,7 @@ void CheckClass::CheckUnusedPrivateFunctions()
 void CheckClass::CheckMemset()
 {
     // Locate all 'memset' tokens..
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (!Match(tok,"memset") && !Match(tok,"memcpy") && !Match(tok,"memmove"))
             continue;
@@ -572,7 +572,7 @@ void CheckClass::CheckMemset()
         // Warn if type is a class..
         const char *pattern1[] = {"class","",NULL};
         pattern1[1] = type;
-        if (Tokenizer::findtoken(tokens,pattern1))
+        if (Tokenizer::findtoken(_tokenizer->tokens(),pattern1))
         {
             std::ostringstream ostr;
             ostr << FileLine(tok, _tokenizer) << ": Using '" << tok->str << "' on class.";
@@ -583,7 +583,7 @@ void CheckClass::CheckMemset()
         // Warn if type is a struct that contains any std::*
         const char *pattern2[] = {"struct","","{",NULL};
         pattern2[1] = type;
-        for (const TOKEN *tstruct = Tokenizer::findtoken(tokens, pattern2); tstruct; tstruct = tstruct->next)
+        for (const TOKEN *tstruct = Tokenizer::findtoken(_tokenizer->tokens(), pattern2); tstruct; tstruct = tstruct->next)
         {
             if (tstruct->str[0] == '}')
                 break;
@@ -609,7 +609,7 @@ void CheckClass::CheckMemset()
 void CheckClass::CheckOperatorEq1()
 {
     const char *pattern[] = {"void", "operator", "=", "(", NULL};
-    if (const TOKEN *tok = Tokenizer::findtoken(tokens,pattern))
+    if (const TOKEN *tok = Tokenizer::findtoken(_tokenizer->tokens(),pattern))
     {
         std::ostringstream ostr;
         ostr << FileLine(tok, _tokenizer) << ": 'operator=' should return something";
diff --git a/CheckHeaders.cpp b/CheckHeaders.cpp
index 5375c57b3..9d8eeb060 100644
--- a/CheckHeaders.cpp
+++ b/CheckHeaders.cpp
@@ -47,7 +47,7 @@ CheckHeaders::~CheckHeaders()
 
 void CheckHeaders::WarningHeaderWithImplementation()
 {
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         // Only interested in included file
         if (tok->FileIndex == 0)
@@ -82,7 +82,7 @@ void CheckHeaders::WarningHeaderWithImplementation()
 void CheckHeaders::WarningIncludeHeader()
 {
     // Including..
-    for (TOKEN *includetok = tokens; includetok; includetok = includetok->next)
+    for (TOKEN *includetok = _tokenizer->tokens(); includetok; includetok = includetok->next)
     {
         if (strcmp(includetok->str, "#include") != 0)
             continue;
@@ -111,7 +111,7 @@ void CheckHeaders::WarningIncludeHeader()
 
         // Extract classes and names in the header..
         int indentlevel = 0;
-        for ( TOKEN *tok1 = tokens; tok1; tok1 = tok1->next )
+        for ( TOKEN *tok1 = _tokenizer->tokens(); tok1; tok1 = tok1->next )
         {
             if ( tok1->FileIndex != hfile )
                 continue;
@@ -205,7 +205,7 @@ void CheckHeaders::WarningIncludeHeader()
         // Check if the extracted names are used...
         bool Needed = false;
         bool NeedDeclaration = false;
-        for (TOKEN *tok1 = tokens; tok1; tok1 = tok1->next)
+        for (TOKEN *tok1 = _tokenizer->tokens(); tok1; tok1 = tok1->next)
         {
             if (tok1->FileIndex != includetok->FileIndex)
                 continue;
diff --git a/CheckMemoryLeak.cpp b/CheckMemoryLeak.cpp
index 226253ec7..1d2dc1b62 100644
--- a/CheckMemoryLeak.cpp
+++ b/CheckMemoryLeak.cpp
@@ -64,7 +64,7 @@ bool CheckMemoryLeakClass::isclass( const std::string &typestr )
 
     std::ostringstream pattern;
     pattern << "struct " << typestr;
-    if ( findmatch( tokens, pattern.str().c_str() ) )
+    if ( findmatch( _tokenizer->tokens(), pattern.str().c_str() ) )
         return false;
 
     return true;
@@ -822,7 +822,7 @@ void CheckMemoryLeakClass::CheckMemoryLeak_InFunction()
 {
     bool infunc = false;
     int indentlevel = 0;
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (tok->str[0]=='{')
             indentlevel++;
@@ -865,7 +865,7 @@ void CheckMemoryLeakClass::CheckMemoryLeak_InFunction()
 void CheckMemoryLeakClass::CheckMemoryLeak_ClassMembers()
 {
     int indentlevel = 0;
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         if ( tok->str[0] == '{' )
             indentlevel++;
@@ -963,7 +963,7 @@ void CheckMemoryLeakClass::CheckMemoryLeak_ClassMembers_Variable( const std::vec
     // Loop through all tokens. Inspect member functions
     bool memberfunction = false;
     int indentlevel = 0;
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         if ( tok->str[0] == '{' )
             indentlevel++;
@@ -1016,7 +1016,7 @@ void CheckMemoryLeakClass::CheckMemoryLeak_ClassMembers_Variable( const std::vec
 
     if ( Alloc != No && Dealloc == No )
     {
-        MemoryLeak( tokens, FullVariableName.str().c_str() );
+        MemoryLeak( _tokenizer->tokens(), FullVariableName.str().c_str() );
     }
 }
 
diff --git a/CheckOther.cpp b/CheckOther.cpp
index 49be8509e..26e187ebe 100644
--- a/CheckOther.cpp
+++ b/CheckOther.cpp
@@ -47,7 +47,7 @@ CheckOther::~CheckOther()
 
 void CheckOther::WarningOldStylePointerCast()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         // Old style pointer casting..
         if (!Match(tok, "( %type% * ) %var%"))
@@ -56,7 +56,7 @@ void CheckOther::WarningOldStylePointerCast()
         // Is "type" a class?
         const char *pattern[] = {"class","",NULL};
         pattern[1] = Tokenizer::getstr(tok, 1);
-        if (!Tokenizer::findtoken(tokens, pattern))
+        if (!Tokenizer::findtoken(_tokenizer->tokens(), pattern))
             continue;
 
         std::ostringstream ostr;
@@ -74,7 +74,7 @@ void CheckOther::WarningOldStylePointerCast()
 
 void CheckOther::WarningIsDigit()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         bool err = false;
         err |= Match(tok, "%var% >= '0' && %var% <= '9'");
@@ -99,7 +99,7 @@ void CheckOther::WarningIsDigit()
 
 void CheckOther::WarningIsAlpha()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if ( ! Match(tok, "(") )
             continue;
@@ -159,7 +159,7 @@ void CheckOther::WarningRedundantCode()
 {
 
     // if (p) delete p
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (!Match(tok,"if"))
             continue;
@@ -221,7 +221,7 @@ void CheckOther::WarningIf()
 {
 
     // Search for 'if (condition);'
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (Match(tok,"if"))
         {
@@ -250,7 +250,7 @@ void CheckOther::WarningIf()
     }
 
     // Search for 'a=b; if (a==b)'
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         // Begin statement?
         if ( ! Match(tok, "[;{}]") )
@@ -310,7 +310,7 @@ void CheckOther::WarningIf()
 
 void CheckOther::InvalidFunctionUsage()
 {
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         if (!Match(tok, "strtol") && !Match(tok, "strtoul"))
             continue;
@@ -354,7 +354,7 @@ void CheckOther::InvalidFunctionUsage()
 
 void CheckOther::CheckIfAssignment()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if (Match(tok, "if ( %var% = %num% )") ||
             Match(tok, "if ( %var% = %str% )") ||
@@ -377,7 +377,7 @@ void CheckOther::CheckUnsignedDivision()
 {
     // Check for "ivar / uvar" and "uvar / ivar"
     std::map<std::string, char> varsign;
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         if ( Match(tok, "[{};(,] %type% %var% [;=,)]") )
         {
@@ -444,7 +444,7 @@ void CheckOther::CheckVariableScope()
     // Walk through all tokens..
     bool func = false;
     int indentlevel = 0;
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         // Skip class and struct declarations..
         if ( Match(tok, "class") || Match(tok, "struct") )
@@ -596,7 +596,7 @@ void CheckOther::CheckVariableScope_LookupVar( const TOKEN *tok1, const char var
 
 void CheckOther::CheckConstantFunctionParameter()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         if ( Match(tok,"[,(] const std :: %type% %var% [,)]") )
         {
@@ -610,14 +610,14 @@ void CheckOther::CheckConstantFunctionParameter()
             // Check if type is a struct or class.
             const char *pattern[3] = {"class","type",0};
             pattern[1] = Tokenizer::getstr(tok, 2);
-            if ( Tokenizer::findtoken(tokens, pattern) )
+            if ( Tokenizer::findtoken(_tokenizer->tokens(), pattern) )
             {
                 std::ostringstream errmsg;
                 errmsg << FileLine(tok, _tokenizer) << " " << Tokenizer::getstr(tok,3) << " is passed by value, it could be passed by reference/pointer instead";
                 ReportErr( errmsg.str() );
             }
             pattern[0] = "struct";
-            if ( Tokenizer::findtoken(tokens, pattern) )
+            if ( Tokenizer::findtoken(_tokenizer->tokens(), pattern) )
             {
                 std::ostringstream errmsg;
                 errmsg << FileLine(tok, _tokenizer) << " " << Tokenizer::getstr(tok,3) << " is passed by value, it could be passed by reference/pointer instead";
@@ -637,7 +637,7 @@ void CheckOther::CheckStructMemberUsage()
 {
     const char *structname = 0;
 
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         if ( tok->FileIndex != 0 )
             continue;
@@ -664,7 +664,7 @@ void CheckOther::CheckStructMemberUsage()
             varnames[0] = varname;
             varnames[1] = 0;
             bool used = false;
-            for ( const TOKEN *tok2 = tokens; tok2; tok2 = tok2->next )
+            for ( const TOKEN *tok2 = _tokenizer->tokens(); tok2; tok2 = tok2->next )
             {
                 if ( tok->FileIndex != 0 )
                     continue;
@@ -698,7 +698,7 @@ void CheckOther::CheckStructMemberUsage()
 
 void CheckOther::CheckCharVariable()
 {
-    for (const TOKEN *tok = tokens; tok; tok = tok->next)
+    for (const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next)
     {
         // Declaring the variable..
         if ( Match(tok, "[{};(,] char %var% [;=,)]") )
@@ -754,7 +754,7 @@ void CheckOther::CheckIncompleteStatement()
 {
     int parlevel = 0;
 
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokenizer->tokens(); tok; tok = tok->next )
     {
         if ( Match(tok, "(") )
             ++parlevel;
diff --git a/testsimplifytokens.cpp b/testsimplifytokens.cpp
index c81040a4f..76ca2f144 100644
--- a/testsimplifytokens.cpp
+++ b/testsimplifytokens.cpp
@@ -45,7 +45,7 @@ private:
         tokenizer.SimplifyTokenList();
 
         std::string ret;
-        for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+        for ( const TOKEN *tok = tokenizer.tokens(); tok; tok = tok->next )
         {
             ret += std::string(tok->str) + " ";
         }
diff --git a/testtokenize.cpp b/testtokenize.cpp
index 70cf1ef96..0444ddc8e 100644
--- a/testtokenize.cpp
+++ b/testtokenize.cpp
@@ -74,7 +74,7 @@ private:
         };
 
         // Compare..
-        ASSERT_EQUALS( true, cmptok(expected, tokens) );
+        ASSERT_EQUALS( true, cmptok(expected, tokenizer.tokens()) );
 
         tokenizer.DeallocateTokens();
     }
@@ -91,7 +91,7 @@ private:
         tokenizer.TokenizeCode(istr, 0);
 
         // Expected result..
-        ASSERT_EQUALS( std::string(10000,'a'), std::string(tokens->str) );
+        ASSERT_EQUALS( std::string(10000,'a'), std::string(tokenizer.tokens()->str) );
 
         tokenizer.DeallocateTokens();
     }
@@ -127,7 +127,7 @@ private:
         };
 
         // Compare..
-        ASSERT_EQUALS( true, cmptok(expected, tokens) );
+        ASSERT_EQUALS( true, cmptok(expected, tokenizer.tokens()) );
 
         tokenizer.DeallocateTokens();
     }
diff --git a/tokenize.cpp b/tokenize.cpp
index e67c9d075..02f0425e0 100644
--- a/tokenize.cpp
+++ b/tokenize.cpp
@@ -42,32 +42,13 @@
 #define _strdup(str) strdup(str)
 #endif
 
-//---------------------------------------------------------------------------
-
-// Helper functions..
-
-TOKEN *Tokenizer::_gettok(TOKEN *tok, int index)
-{
-    while (tok && index>0)
-    {
-        tok = tok->next;
-        index--;
-    }
-    return tok;
-}
-//---------------------------------------------------------------------------
-
-
-//---------------------------------------------------------------------------
-
-
-TOKEN *tokens;
+
 
 //---------------------------------------------------------------------------
 
 Tokenizer::Tokenizer()
 {
-    tokens = 0;
+    _tokens = 0;
     tokens_back = 0;
     dsymlist = 0;
 }
@@ -76,6 +57,26 @@ Tokenizer::~Tokenizer()
 {
 
 }
+
+//---------------------------------------------------------------------------
+
+// Helper functions..
+
+TOKEN *Tokenizer::_gettok(TOKEN *tok, int index)
+{
+    while (tok && index>0)
+    {
+        tok = tok->next;
+        index--;
+    }
+    return tok;
+}
+//---------------------------------------------------------------------------
+
+TOKEN *Tokenizer::tokens()
+{
+    return _tokens;
+}
 
 //---------------------------------------------------------------------------
 // Defined symbols.
@@ -172,7 +173,7 @@ void Tokenizer::addtoken(const char str[], const unsigned int lineno, const unsi
     }
     else
     {
-        tokens = tokens_back = newtoken;
+        _tokens = tokens_back = newtoken;
     }
 
     // Check if str is defined..
@@ -551,7 +552,7 @@ void Tokenizer::TokenizeCode(std::istream &code, const unsigned int FileIndex)
     addtoken( CurrentToken.c_str(), lineno, FileIndex );
 
     // Combine tokens..
-    for (TOKEN *tok = tokens; tok && tok->next; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok && tok->next; tok = tok->next)
     {
         combine_2tokens(tok, "<", "<");
         combine_2tokens(tok, ">", ">");
@@ -580,7 +581,7 @@ void Tokenizer::TokenizeCode(std::istream &code, const unsigned int FileIndex)
     }
 
     // Replace "->" with "."
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if ( strcmp(tok->str, "->") == 0 )
         {
@@ -589,7 +590,7 @@ void Tokenizer::TokenizeCode(std::istream &code, const unsigned int FileIndex)
     }
 
     // typedef..
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if (Match(tok, "typedef %type% %type% ;"))
         {
@@ -634,7 +635,7 @@ void Tokenizer::TokenizeCode(std::istream &code, const unsigned int FileIndex)
 
 
     // Remove __asm..
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if ( Match(tok->next, "__asm {") )
         {
@@ -672,7 +673,7 @@ void Tokenizer::SimplifyTokenList()
 {
 
     // Remove the keyword 'unsigned'
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if (tok->next && strcmp(tok->next->str,"unsigned")==0)
         {
@@ -681,7 +682,7 @@ void Tokenizer::SimplifyTokenList()
     }
 
     // Replace constants..
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok; tok = tok->next)
     {
         if (Match(tok,"const %type% %var% = %num% ;"))
         {
@@ -707,7 +708,7 @@ void Tokenizer::SimplifyTokenList()
     TypeSize["long"] = sizeof(long);
     TypeSize["float"] = sizeof(float);
     TypeSize["double"] = sizeof(double);
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok; tok = tok->next)
     {
         if (Match(tok,"class %var%"))
         {
@@ -722,7 +723,7 @@ void Tokenizer::SimplifyTokenList()
 
 
     // Replace 'sizeof(type)'..
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok; tok = tok->next)
     {
         if (strcmp(tok->str,"sizeof") != 0)
             continue;
@@ -765,7 +766,7 @@ void Tokenizer::SimplifyTokenList()
     }
 
     // Replace 'sizeof(var)'
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok; tok = tok->next)
     {
         // type array [ num ] ;
         if ( ! Match(tok, "%type% %var% [ %num% ] ;") )
@@ -822,7 +823,7 @@ void Tokenizer::SimplifyTokenList()
     {
         done = true;
 
-        for (TOKEN *tok = tokens; tok; tok = tok->next)
+        for (TOKEN *tok = _tokens; tok; tok = tok->next)
         {
             if (Match(tok->next, "* 1") || Match(tok->next, "1 *"))
             {
@@ -868,7 +869,7 @@ void Tokenizer::SimplifyTokenList()
 
 
     // Replace "*(str + num)" => "str[num]"
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok; tok = tok->next)
     {
         if ( ! strchr(";{}(=<>", tok->str[0]) )
             continue;
@@ -897,7 +898,7 @@ void Tokenizer::SimplifyTokenList()
 
 
     // Split up variable declarations if possible..
-    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    for (TOKEN *tok = _tokens; tok; tok = tok->next)
     {
         if ( ! strchr("{};", tok->str[0]) )
             continue;
@@ -1018,14 +1019,14 @@ void Tokenizer::SimplifyTokenList()
     }
 
     // Replace NULL with 0..
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if ( Match(tok, "NULL") )
             tok->setstr("0");
     }
 
     // Replace pointer casts of 0.. "(char *)0" => "0"
-    for ( TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if ( Match(tok->next, "( %type% * ) 0") || Match(tok->next,"( %type% %type% * ) 0") )
         {
@@ -1095,13 +1096,13 @@ const char *Tokenizer::getstr(const TOKEN *tok, int index)
 // Deallocate lists..
 void Tokenizer::DeallocateTokens()
 {
-    while (tokens)
+    while (_tokens)
     {
-        TOKEN *next = tokens->next;
-        delete tokens;
-        tokens = next;
+        TOKEN *next = _tokens->next;
+        delete _tokens;
+        _tokens = next;
     }
-    tokens_back = tokens;
+    tokens_back = _tokens;
 
     while (dsymlist)
     {
@@ -1145,7 +1146,7 @@ void Tokenizer::FillFunctionList(const unsigned int file_id)
     bool classfunc = false;
 
     int indentlevel = 0;
-    for ( const TOKEN *tok = tokens; tok; tok = tok->next )
+    for ( const TOKEN *tok = _tokens; tok; tok = tok->next )
     {
         if ( tok->str[0] == '{' )
             indentlevel++;
diff --git a/tokenize.h b/tokenize.h
index 65eaae88b..9310d5fe3 100644
--- a/tokenize.h
+++ b/tokenize.h
@@ -59,8 +59,6 @@ public:
     TOKEN *next;
 };
 
-extern TOKEN *tokens, *tokens_back;
-
 class Tokenizer
 {
 public:
@@ -96,6 +94,7 @@ public:
     const TOKEN *GetFunctionTokenByName( const char funcname[] ) const;
     void CheckGlobalFunctionUsage(const std::vector<std::string> &filenames);
     void settings( const Settings &settings );
+    TOKEN *tokens();
 private:
 
     struct DefineSymbol
@@ -144,6 +143,7 @@ private:
 
 
     struct DefineSymbol * dsymlist;
+    TOKEN *_tokens;
 };