diff --git a/src/token.cpp b/src/token.cpp index e78352095..3129aeadc 100644 --- a/src/token.cpp +++ b/src/token.cpp @@ -80,6 +80,17 @@ const Token *Token::tokAt(int index) const return tok; } +Token *Token::tokAt(int index) +{ + Token *tok = this; + while (index > 0 && tok) + { + tok = tok->next(); + --index; + } + return tok; +} + const char *Token::strAt(int index) const { const Token *tok = this->tokAt(index); diff --git a/src/token.h b/src/token.h index 040832d81..ae7e24350 100644 --- a/src/token.h +++ b/src/token.h @@ -59,6 +59,7 @@ public: * would return next from that one. */ const Token *tokAt(int index) const; + Token *tokAt(int index); const char *strAt(int index) const; diff --git a/src/tokenize.cpp b/src/tokenize.cpp index bb42bbc37..1ace705b4 100644 --- a/src/tokenize.cpp +++ b/src/tokenize.cpp @@ -49,7 +49,6 @@ Tokenizer::Tokenizer() { _tokens = 0; _tokensBack = 0; - _dsymlist = 0; } Tokenizer::~Tokenizer() @@ -61,15 +60,6 @@ Tokenizer::~Tokenizer() // Helper functions.. -Token *Tokenizer::_gettok(Token *tok, int index) -{ - while (tok && index > 0) - { - tok = tok->next(); - --index; - } - return tok; -} //--------------------------------------------------------------------------- @@ -78,61 +68,12 @@ const Token *Tokenizer::tokens() const return _tokens; } -//--------------------------------------------------------------------------- -// Defined symbols. -// "#define abc 123" will create a defined symbol "abc" with the value 123 -//--------------------------------------------------------------------------- - - const std::vector *Tokenizer::getFiles() const { return &_files; } -void Tokenizer::Define(const char Name[], const char Value[]) -{ - if (!(Name && Name[0])) - return; - - if (!(Value && Value[0])) - return; - - // Is 'Value' a decimal value.. - bool dec = true, hex = true; - for (int i = 0; Value[i]; i++) - { - if (! isdigit(Value[i])) - dec = false; - - if (! isxdigit(Value[i]) && (!(i == 1 && Value[i] == 'x'))) - hex = false; - } - - if (!dec && !hex) - return; - - char *strValue = strdup(Value); - - if (!dec && hex) - { - // Convert Value from hexadecimal to decimal - unsigned long value; - std::istringstream istr(Value + 2); - istr >> std::hex >> value; - std::ostringstream ostr; - ostr << value; - free(strValue); - strValue = strdup(ostr.str().c_str()); - } - - DefineSymbol *NewSym = new DefineSymbol; - memset(NewSym, 0, sizeof(DefineSymbol)); - NewSym->name = strdup(Name); - NewSym->value = strValue; - NewSym->next = _dsymlist; - _dsymlist = NewSym; -} //--------------------------------------------------------------------------- //--------------------------------------------------------------------------- @@ -170,16 +111,6 @@ void Tokenizer::addtoken(const char str[], const unsigned int lineno, const unsi _tokensBack->linenr(lineno); _tokensBack->fileIndex(fileno); - - // Check if str is defined.. - for (DefineSymbol *sym = _dsymlist; sym; sym = sym->next) - { - if (strcmp(str, sym->name) == 0) - { - _tokensBack->str(sym->value); - break; - } - } } //--------------------------------------------------------------------------- @@ -227,32 +158,23 @@ void Tokenizer::InsertTokens(Token *dest, Token *src, unsigned int n) void Tokenizer::tokenize(std::istream &code, const char FileName[]) { - // Has this file been tokenized already? - for (unsigned int i = 0; i < _files.size(); i++) - { - if (SameFileName(_files[i].c_str(), FileName)) - return; - } - // The "_files" vector remembers what files have been tokenized.. _files.push_back(FileLister::simplifyPath(FileName)); - // Tokenize the file.. - tokenizeCode(code, (unsigned int)(_files.size() - 1)); -} -//--------------------------------------------------------------------------- - -//--------------------------------------------------------------------------- -// Tokenize - tokenizes input stream -//--------------------------------------------------------------------------- - -void Tokenizer::tokenizeCode(std::istream &code, unsigned int FileIndex) -{ - // Tokenize the file. + // line number in parsed code unsigned int lineno = 1; + + // The current token being parsed std::string CurrentToken; + + // fileIndexes is a vector that holds linenumbers for file indexes std::vector fileIndexes; - fileIndexes.push_back(FileIndex); + fileIndexes.push_back(0); + + // FileIndex. What file in the _files vector is read now? + unsigned int FileIndex = 0; + + // Read one byte at a time from code and create tokens for (char ch = (char)code.get(); code.good(); ch = (char)code.get()) { // We are not handling UTF and stuff like that. Code is supposed to plain simple text. @@ -628,7 +550,7 @@ void Tokenizer::simplifyTokenList() const char *sym = tok->strAt(2); const char *num = tok->strAt(4); - for (Token *tok2 = _gettok(tok, 6); tok2; tok2 = tok2->next()) + for (Token *tok2 = tok->tokAt(6); tok2; tok2 = tok2->next()) { if (tok2->str() == sym) { @@ -720,7 +642,7 @@ void Tokenizer::simplifyTokenList() // Replace 'sizeof(var)' with number int indentlevel = 0; - for (Token *tok2 = _gettok(tok, 5); tok2; tok2 = tok2->next()) + for (Token *tok2 = tok->tokAt(5); tok2; tok2 = tok2->next()) { if (tok2->str() == "{") { @@ -855,7 +777,7 @@ void Tokenizer::simplifyTokenList() { if (type0->next()->str() != "operator") { - tok2 = _gettok(type0, 2); // The ',' or '=' token + tok2 = type0->tokAt(2); // The ',' or '=' token typelen = 1; } } @@ -864,32 +786,32 @@ void Tokenizer::simplifyTokenList() { if (type0->next()->next()->str() != "operator") { - tok2 = _gettok(type0, 3); // The ',' token + tok2 = type0->tokAt(3); // The ',' token typelen = 1; } } else if (Token::Match(type0, "%type% %var% [ %num% ] ,|=")) { - tok2 = _gettok(type0, 5); // The ',' token + tok2 = type0->tokAt(5); // The ',' token typelen = 1; } else if (Token::Match(type0, "%type% * %var% [ %num% ] ,|=")) { - tok2 = _gettok(type0, 6); // The ',' token + tok2 = type0->tokAt(6); // The ',' token typelen = 1; } else if (Token::Match(type0, "struct %type% %var% ,|=")) { - tok2 = _gettok(type0, 3); + tok2 = type0->tokAt(3); typelen = 2; } else if (Token::Match(type0, "struct %type% * %var% ,|=")) { - tok2 = _gettok(type0, 4); + tok2 = type0->tokAt(4); typelen = 2; } @@ -924,7 +846,7 @@ void Tokenizer::simplifyTokenList() else if (parlevel == 0 && strchr(";,", tok2->aaaa0())) { // "type var =" => "type var; var =" - Token *VarTok = _gettok(type0, typelen); + Token *VarTok = type0->tokAt(typelen); if (VarTok->aaaa0() == '*') VarTok = VarTok->next(); InsertTokens(eq, VarTok, 2); @@ -1544,16 +1466,6 @@ void Tokenizer::DeallocateTokens() deleteTokens(_tokens); _tokens = 0; _tokensBack = 0; - - while (_dsymlist) - { - struct DefineSymbol *next = _dsymlist->next; - free(_dsymlist->name); - free(_dsymlist->value); - delete _dsymlist; - _dsymlist = next; - } - _files.clear(); } diff --git a/src/tokenize.h b/src/tokenize.h index 3b813d05f..5bf1bde41 100644 --- a/src/tokenize.h +++ b/src/tokenize.h @@ -35,11 +35,6 @@ private: // Deallocate lists.. void DeallocateTokens(); - /** - * Helper function for "tokenize". This recursively parses into included header files. - */ - void tokenizeCode(std::istream &code, unsigned int FileIndex = 0); - public: Tokenizer(); ~Tokenizer(); @@ -84,12 +79,6 @@ public: private: #endif - struct DefineSymbol - { - char *name; - char *value; - struct DefineSymbol *next; - }; /** * Finds matching "end" for "start". @@ -100,8 +89,6 @@ private: */ static const Token *findClosing(const Token *tok, const char *start, const char *end); - void Define(const char Name[], const char Value[]); - void addtoken(const char str[], const unsigned int lineno, const unsigned int fileno); /** Add braces to an if-block @@ -145,15 +132,12 @@ private: */ bool simplifyKnownVariables(); - Token *_gettok(Token *tok, int index); - void InsertTokens(Token *dest, Token *src, unsigned int n); Token *_tokensBack; std::map _typeSize; std::vector _functionList; std::vector _files; - struct DefineSymbol * _dsymlist; Token *_tokens; };