tok: Added a program that tests the tokenizer.

TestTok.cpp

@@ -0,0 +1,45 @@
+
+// Internal testing
+// Tokenize a file and print tokens
+
+#include <iostream>
+#include "tokenize.h"
+
+//---------------------------------------------------------------------------
+
+int main(int argc, char* argv[])
+{
+    if (argc != 2)
+        return 0;
+
+    tokens = tokens_back = NULL;
+    Tokenize(argv[1]);
+
+    unsigned int linenr = 0;
+    for (TOKEN *tok = tokens; tok; tok = tok->next)
+    {
+        // Print either a "\n" or a " "
+        if (tok->linenr != linenr)
+            std::cout << "\n";
+        else
+            std::cout << " ";
+
+        std::cout << tok->str;
+
+        linenr = tok->linenr;
+    }
+
+
+    // Delete all tokens..
+    while (tokens)
+    {
+        TOKEN *next = tokens->next;
+        free(tokens->str);
+        delete tokens;
+        tokens = next;
+    }
+
+    return 0;
+}
+//---------------------------------------------------------------------------
+ 

tok.bpf

@@ -0,0 +1,3 @@
+This file is used by the project manager only and should be treated like the project file
+
+main 

tok.bpr

@@ -0,0 +1,183 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!-- C++Builder XML Project -->
+<PROJECT>
+  <MACROS>
+    <VERSION value="BCB.06.00"/>
+    <PROJECT value="tok.exe"/>
+    <OBJFILES value="TestTok.obj tokenize.obj CommonCheck.obj"/>
+    <RESFILES value="tok.res"/>
+    <IDLFILES value=""/>
+    <DEFFILE value=""/>
+    <RESDEPEN value="$(RESFILES)"/>
+    <LIBFILES value=""/>
+    <LIBRARIES value=""/>
+    <SPARELIBS value=""/>
+    <PACKAGES value=""/>
+    <PATHCPP value=".;"/>
+    <PATHPAS value=".;"/>
+    <PATHRC value=".;"/>
+    <PATHASM value=".;"/>
+    <DEBUGLIBPATH value="$(BCB)\lib\debug"/>
+    <RELEASELIBPATH value="$(BCB)\lib\release"/>
+    <LINKER value="ilink32"/>
+    <USERDEFINES value=""/>
+    <SYSDEFINES value="NO_STRICT;_NO_VCL;USEPACKAGES"/>
+    <MAINSOURCE value="tok.bpf"/>
+    <INCLUDEPATH value="S:\test\cppcheck;&quot;C:\Program Files\Borland\CBuilder6\Projects&quot;;S:\test\prjman;$(BCB)\include;$(BCB)\include\vcl"/>
+    <LIBPATH value="S:\test\cppcheck;$(BCB)\lib\obj;$(BCB)\lib"/>
+    <WARNINGS value=""/>
+    <WARNOPTSTR value=""/>
+    <OTHERFILES value=""/>
+  </MACROS>
+  <OPTIONS>
+    <CFLAG1 value="-vGc -vGt -vGd -Od -H=c:\PROGRA~1\borland\CBUILD~1\lib\vcl35.csm -Hc -w 
+      -Vx -Ve -Tkh30000 -X- -r- -a8 -b- -k -y -v -vi- -tWC -tWM- -c"/>
+    <CFLAG2 value=""/>
+    <CFLAG3 value=""/>
+    <PFLAGS value="-$YD -$W -$O- -$A8 -v -M -JPHNE"/>
+    <RFLAGS value=""/>
+    <AFLAGS value="/mx /w2 /zd"/>
+    <LFLAGS value="-D&quot;&quot; -ap -Tpe -x -Gn -v"/>
+    <IFLAGS value=""/>
+    <OTHERFILES value=""/>
+  </OPTIONS>
+  <LINKER>
+    <ALLOBJ value="c0x32.obj $(PACKAGES) $(OBJFILES)"/>
+    <ALLRES value="$(RESFILES)"/>
+    <ALLLIB value="cg32.lib $(LIBFILES) $(LIBRARIES) import32.lib cw32.lib"/>
+    <OTHERFILES value=""/>
+  </LINKER>
+  <FILELIST>
+      <FILE FILENAME="tok.res" FORMNAME="" UNITNAME="tok.res" CONTAINERID="ResTool" DESIGNCLASS="" LOCALCOMMAND=""/>
+      <FILE FILENAME="tok.bpf" FORMNAME="" UNITNAME="tok" CONTAINERID="BPF" DESIGNCLASS="" LOCALCOMMAND=""/>
+      <FILE FILENAME="TestTok.cpp" FORMNAME="" UNITNAME="TestTok" CONTAINERID="CCompiler" DESIGNCLASS="" LOCALCOMMAND=""/>
+      <FILE FILENAME="tokenize.cpp" FORMNAME="" UNITNAME="tokenize" CONTAINERID="CCompiler" DESIGNCLASS="" LOCALCOMMAND=""/>
+      <FILE FILENAME="CommonCheck.cpp" FORMNAME="" UNITNAME="CommonCheck" CONTAINERID="CCompiler" DESIGNCLASS="" LOCALCOMMAND=""/>
+  </FILELIST>
+  <BUILDTOOLS>
+  </BUILDTOOLS>
+
+  <IDEOPTIONS>
+[Version Info]
+IncludeVerInfo=0
+AutoIncBuild=0
+MajorVer=1
+MinorVer=0
+Release=0
+Build=0
+Debug=0
+PreRelease=0
+Special=0
+Private=0
+DLL=0
+Locale=1053
+CodePage=1252
+
+[Version Info Keys]
+CompanyName=
+FileDescription=
+FileVersion=1.0.0.0
+InternalName=
+LegalCopyright=
+LegalTrademarks=
+OriginalFilename=
+ProductName=
+ProductVersion=1.0.0.0
+Comments=
+
+[Excluded Packages]
+c:\program files\borland\cbuilder6\Bin\dclbcbsmp60.bpl=Borland Sample Controls Design Time Package
+c:\program files\borland\cbuilder6\Bin\applet60.bpl=Borland Control Panel Applet Package
+c:\program files\borland\cbuilder6\Bin\dclsmpedit60.bpl=Borland Editor Script Enhancements
+c:\program files\borland\cbuilder6\Bin\dclemacsedit60.bpl=Borland Editor Emacs Enhancements
+c:\program files\borland\cbuilder6\Bin\dclact60.bpl=Borland ActionBar Components
+c:\program files\borland\cbuilder6\Bin\idefilters60.bpl=IDE Filter for CPP32
+c:\program files\borland\cbuilder6\Bin\dclmlwiz60.bpl=Borland Markup Language Wizards
+c:\program files\borland\cbuilder6\Bin\dclnet60.bpl=Borland Internet Components
+c:\program files\borland\cbuilder6\Bin\DCLNMF60.bpl=NetMasters Fastnet Tools
+c:\program files\borland\cbuilder6\Bin\dclocx60.bpl=Borland Sample Imported ActiveX Controls
+c:\program files\borland\cbuilder6\Bin\dclindy60.bpl=Internet Direct (Indy) for D6 Property and Component Editors
+c:\program files\borland\cbuilder6\Projects\Bpl\B304vd61.bpl=TurboPower Abbrevia 3.04 Design-time package - VCL61
+c:\program files\borland\cbuilder6\Projects\Bpl\DreamEdit_C6.bpl=Dream Editor Package
+c:\program files\borland\cbuilder6\PROJECTS\BPL\Iocomp65.bpl=Iocomp Package VCL
+c:\program files\borland\cbuilder6\Projects\Bpl\tb2k_cb6.bpl=Toolbar2000 Components (Jordan Russell)
+c:\program files\borland\cbuilder6\Projects\Bpl\tb2kdsgn_cb6.bpl=Toolbar2000 Design Package (Jordan Russell)
+c:\program files\borland\cbuilder6\Projects\Bpl\A406_D61.bpl=TurboPower Async Professional 4.05 Designtime Package - VCL61
+c:\program files\borland\cbuilder6\Projects\Bpl\ThemeManagerC6D.bpl=Soft Gems Theme Manager
+c:\program files\borland\cbuilder6\Bin\dclado60.bpl=Borland ADO DB Components
+C:\Program Files\Borland\CBuilder6\Bin\dcldb60.bpl=Borland Database Components
+c:\program files\borland\cbuilder6\Bin\dclqrt60.bpl=QuickReport Components
+C:\WINNT\system32\ibevnt60.bpl=Borland Interbase Event Alerter Component
+c:\program files\borland\cbuilder6\Bin\dclbde60.bpl=Borland BDE DB Components
+C:\Program Files\Borland\CBuilder6\Bin\dbx60.bpl=Borland SQL Explorer UI Package
+c:\program files\borland\cbuilder6\Bin\dclcds60.bpl=Borland Base Cached ClientDataset Component
+C:\Program Files\Borland\CBuilder6\Bin\dclmid60.bpl=Borland MyBase DataAccess Components
+c:\program files\borland\cbuilder6\Bin\dclbdecds60.bpl=Borland Local BDE ClientDataset Components
+c:\program files\borland\cbuilder6\Bin\dcltee60.bpl=TeeChart Components
+c:\program files\borland\cbuilder6\Bin\dcltqr60.bpl=TeeChart for QuickReport Components
+c:\program files\borland\cbuilder6\Bin\dclib60.bpl=InterBase Data Access Components
+c:\program files\borland\cbuilder6\Bin\dclclxdb60.bpl=Borland CLX Database Components
+C:\Program Files\Borland\CBuilder6\Bin\dclclxstd60.bpl=Borland CLX Standard Components
+c:\program files\borland\cbuilder6\Bin\DBWEBXPRT.BPL=Borland Web Wizard Package
+c:\program files\borland\cbuilder6\Bin\bcbie60.bpl=Borland C++Builder Internet Explorer 5 Components Package
+c:\program files\borland\cbuilder6\Bin\dclsoap60.bpl=Borland SOAP Components
+c:\program files\borland\cbuilder6\Bin\dcldbx60.bpl=Borland dbExpress Components
+c:\program files\borland\cbuilder6\Bin\dcldbxcds60.bpl=Borland Local DBX ClientDataset Components
+c:\program files\borland\cbuilder6\Bin\bcb2kaxserver60.bpl=Borland C++Builder COM Server Components Sample Package
+c:\program files\borland\cbuilder6\PROJECTS\BPL\Iocomp65clx.bpl=Iocomp Package CLX
+c:\program files\borland\cbuilder6\Bin\dclite60.bpl=Borland Integrated Translation Environment
+
+[HistoryLists\hlIncludePath]
+Count=3
+Item0=C:\Program Files\Borland\CBuilder6\Projects;S:\test\prjman;$(BCB)\include;$(BCB)\include\vcl
+Item1=$(BCB)\include;$(BCB)\include\vcl
+Item2=e:\minadok\kod\bcb\components;e:\minadok\kod\bcb\clientserver;$(BCB)\include;$(BCB)\include\vcl
+
+[HistoryLists\hlLibraryPath]
+Count=6
+Item0=C:\Program Files\Borland\CBuilder6\Projects;S:\test\prjman;$(BCB)\lib\obj;$(BCB)\lib
+Item1=$(BCB)\lib\obj;$(BCB)\lib
+Item2=..\..\..\..\..\program\borland\cbuilder3\lib;$(BCB)\lib\obj;$(BCB)\lib;d:\minadok\kod\bcb\components
+Item3=..\..\..\..\..\program\borland\cbuilder3\lib;$(BCB)\lib\obj;$(BCB)\lib
+Item4=e:\minadok\kod\bcb\components;e:\minadok\kod\bcb\clientserver;..\..\bokf\components;..\bokf\components;$(BCB)\lib\obj;$(BCB)\lib
+Item5=..\..\bokf\components;..\bokf\components;$(BCB)\lib\obj;$(BCB)\lib
+
+[HistoryLists\hlDebugSourcePath]
+Count=1
+Item0=$(BCB)\source\vcl
+
+[HistoryLists\hlConditionals]
+Count=1
+Item0=_RTLDLL;USEPACKAGES
+
+[HistoryLists\hlRunParameters]
+Count=7
+Item0=&quot;D:\MINADOK\kod\920i\bjornbole45\bjornb.run&quot;
+Item1=&quot;d:\minadok\kod\920\bjornbole45\bjornb.run&quot;
+Item2=&quot;C:\WINDOWS\Skrivbord\bjornb\bjornbole45\bjornb.run&quot;
+Item3=&quot;C:\WINDOWS\Skrivbord\bjornb\bjornbole45&quot;
+Item4=&quot;c:\test.run&quot;
+Item5=&quot;C:\Documents and Settings\Daniel\Mina dokument\kod\bjornbole22\bjornb.run&quot;
+Item6=bjornb.cod bjornb.920
+
+[Debugging]
+DebugSourceDirs=$(BCB)\source\vcl
+
+[Parameters]
+RunParams=testbufferoverrun6\testbufferoverrun6.cpp
+Launcher=
+UseLauncher=0
+DebugCWD=
+HostApplication=
+RemoteHost=
+RemotePath=
+RemoteLauncher=
+RemoteCWD=
+RemoteDebug=0
+
+[Compiler]
+ShowInfoMsgs=0
+LinkDebugVcl=0
+LinkCGLIB=1
+  </IDEOPTIONS>
+</PROJECT>