From 5070bfd87d4eb0cd4e3a12d41909dd57660214a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= <hyd_danmar@users.sourceforge.net>
Date: Tue, 23 Oct 2007 06:36:29 +0000
Subject: [PATCH] tokenizer: removed some preprocessing => less false positives

---
 tokenize.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tokenize.cpp b/tokenize.cpp
index cdf75f6ca..d381b9ed2 100644
--- a/tokenize.cpp
+++ b/tokenize.cpp
@@ -258,19 +258,23 @@ void Tokenize(const char FileName[])
             return;
     }
 
+    // Open file..
     std::ifstream fin(FileName);
     if (!fin.is_open())
         return;
 
+    // The "Files" vector remembers what files have been tokenized..
     unsigned int CurrentFile = Files.size();
     Files.push_back(FileName);
 
+    // Tokenize the file.
     unsigned int lineno = 1;
     char CurrentToken[1000];
     memset(CurrentToken, 0, sizeof(CurrentToken));
     char *pToken = CurrentToken;
     for (char ch = (char)fin.get(); !fin.eof(); ch = (char)fin.get())
     {
+        // Preprocessor stuff?
         if (ch == '#' && !CurrentToken[0])
         {
             std::string line;
@@ -540,7 +544,7 @@ void SimplifyTokenList()
     }
 
 
-
+    /*
     // typedefs..
     TOKEN *prev = NULL;
     for (TOKEN *tok = tokens; tok; tok = tok->next)
@@ -639,6 +643,7 @@ void SimplifyTokenList()
         }
         prev = tok;
     }
+    */
 
 
     // Fill the map TypeSize..