Improved Tokens and Tokenizer:
- Better handling of deleteThis() as last element of token list - Code simplified
This commit is contained in:
PKEuS
parent
da8abeb63c
commit
bb319c1c96
|
|
@ -118,7 +118,7 @@ std::string Token::strValue() const
|
|||
{
|
||||
assert(_str.length() >= 2);
|
||||
assert(_str[0] == '"');
|
||||
assert(_str[_str.length()-1] == '"');
|
||||
assert(_str.back() == '"');
|
||||
return _str.substr(1, _str.length() - 2);
|
||||
}
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ void Token::deleteNext(unsigned long index)
|
|||
|
||||
void Token::deleteThis()
|
||||
{
|
||||
if (_next) {
|
||||
if (_next) { // Copy next to this and delete next
|
||||
_str = _next->_str;
|
||||
_isName = _next->_isName;
|
||||
_isNumber = _next->_isNumber;
|
||||
|
|
@ -158,14 +158,34 @@ void Token::deleteThis()
|
|||
_link->link(this);
|
||||
|
||||
deleteNext();
|
||||
} else if (_previous) {
|
||||
// This should never be used for tokens
|
||||
// at the end of the list
|
||||
str(";");
|
||||
} else if (_previous && _previous->_previous) { // Copy previous to this and delete previous
|
||||
_str = _previous->_str;
|
||||
_isName = _previous->_isName;
|
||||
_isNumber = _previous->_isNumber;
|
||||
_isBoolean = _previous->_isBoolean;
|
||||
_isUnsigned = _previous->_isUnsigned;
|
||||
_isSigned = _previous->_isSigned;
|
||||
_isPointerCompare = _previous->_isPointerCompare;
|
||||
_isLong = _previous->_isLong;
|
||||
_isUnused = _previous->_isUnused;
|
||||
_isStandardType = _previous->_isStandardType;
|
||||
_isExpandedMacro = _previous->_isExpandedMacro;
|
||||
_varId = _previous->_varId;
|
||||
_fileIndex = _previous->_fileIndex;
|
||||
_linenr = _previous->_linenr;
|
||||
_link = _previous->_link;
|
||||
if (_link)
|
||||
_link->link(this);
|
||||
|
||||
Token* toDelete = _previous;
|
||||
_previous = _previous->_previous;
|
||||
_previous->_next = this;
|
||||
|
||||
delete toDelete;
|
||||
} else {
|
||||
// We are the last token in the list, we can't delete
|
||||
// ourselves, so just make us ;
|
||||
str(";");
|
||||
// ourselves, so just make us empty
|
||||
str("");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -839,25 +859,31 @@ const Token *Token::findmatch(const Token *tok, const char pattern[], const Toke
|
|||
|
||||
void Token::insertToken(const std::string &tokenStr)
|
||||
{
|
||||
Token *newToken = new Token(tokensBack);
|
||||
Token *newToken;
|
||||
if (_str == "")
|
||||
newToken = this;
|
||||
else
|
||||
newToken = new Token(tokensBack);
|
||||
newToken->str(tokenStr);
|
||||
newToken->_linenr = _linenr;
|
||||
newToken->_fileIndex = _fileIndex;
|
||||
newToken->_progressValue = _progressValue;
|
||||
|
||||
if (newToken != this) {
|
||||
if (this->next()) {
|
||||
newToken->next(this->next());
|
||||
newToken->next()->previous(newToken);
|
||||
} else if (tokensBack) {
|
||||
*tokensBack = newToken;
|
||||
}
|
||||
|
||||
this->next(newToken);
|
||||
newToken->previous(this);
|
||||
}
|
||||
}
|
||||
|
||||
void Token::eraseTokens(Token *begin, const Token *end)
|
||||
{
|
||||
if (!begin)
|
||||
if (!begin || begin == end)
|
||||
return;
|
||||
|
||||
while (begin->next() && begin->next() != end) {
|
||||
|
|
@ -901,6 +927,8 @@ std::string Token::stringify(const Token* end) const
|
|||
ret << str();
|
||||
|
||||
for (const Token *tok = this->next(); tok && tok != end; tok = tok->next()) {
|
||||
if (tok->str() == "")
|
||||
continue;
|
||||
if (tok->isUnsigned())
|
||||
ret << " unsigned";
|
||||
else if (tok->isSigned())
|
||||
|
|
|
|||
|
|
@ -381,32 +381,6 @@ void Tokenizer::createTokens(std::istream &code)
|
|||
} else if (CurrentToken.empty() && ch == '.' && std::isdigit(code.peek())) {
|
||||
// tokenize .125 into 0.125
|
||||
CurrentToken = "0";
|
||||
} else if (ch=='&' && code.peek() == '&') {
|
||||
if (!CurrentToken.empty()) {
|
||||
addtoken(CurrentToken.c_str(), lineno, FileIndex, true);
|
||||
if (!CurrentToken.empty())
|
||||
_tokensBack->setExpandedMacro(expandedMacro);
|
||||
CurrentToken.clear();
|
||||
}
|
||||
|
||||
// &&
|
||||
ch = (char)code.get();
|
||||
addtoken("&&", lineno, FileIndex, true);
|
||||
_tokensBack->setExpandedMacro(expandedMacro);
|
||||
continue;
|
||||
} else if (ch==':' && CurrentToken.empty() && code.peek() == ' ') {
|
||||
// :
|
||||
addtoken(":", lineno, FileIndex, true);
|
||||
_tokensBack->setExpandedMacro(expandedMacro);
|
||||
CurrentToken.clear();
|
||||
continue;
|
||||
} else if (ch==':' && CurrentToken.empty() && code.peek() == ':') {
|
||||
// ::
|
||||
ch = (char)code.get();
|
||||
addtoken("::", lineno, FileIndex, true);
|
||||
_tokensBack->setExpandedMacro(expandedMacro);
|
||||
CurrentToken.clear();
|
||||
continue;
|
||||
} else if (strchr("+-*/%&|^?!=<>[](){};:,.~\n ", ch)) {
|
||||
if (CurrentToken == "#file") {
|
||||
// Handle this where strings are handled
|
||||
|
|
@ -440,8 +414,8 @@ void Tokenizer::createTokens(std::istream &code)
|
|||
}
|
||||
|
||||
CurrentToken += ch;
|
||||
// Add "++", "--" or ">>" token
|
||||
if ((ch == '+' || ch == '-' || ch == '>') && (code.peek() == ch))
|
||||
// Add "++", "--", ">>" or ... token
|
||||
if (strchr("+-<>=:&|", ch) && (code.peek() == ch))
|
||||
CurrentToken += (char)code.get();
|
||||
addtoken(CurrentToken.c_str(), lineno, FileIndex);
|
||||
_tokensBack->setExpandedMacro(expandedMacro);
|
||||
|
|
@ -2027,19 +2001,8 @@ bool Tokenizer::tokenize(std::istream &code,
|
|||
if (tok->str().length() == 1 && tok->next()->str().length() == 1) {
|
||||
const char c2 = tok->next()->str()[0];
|
||||
|
||||
// combine equal tokens..
|
||||
if (c1 == c2 && (c1 == '<' || c1 == '|' || c1 == ':')) {
|
||||
tok->str(tok->str() + c2);
|
||||
tok->deleteNext();
|
||||
if (c1 == '<' && Token::simpleMatch(tok->next(), "=")) {
|
||||
tok->str("<<=");
|
||||
tok->deleteNext();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// combine +-*/ and =
|
||||
else if (c2 == '=' && (strchr("+-*/%&|^=!<>", c1))) {
|
||||
if (c2 == '=' && (strchr("+-*/%&|^=!<>", c1))) {
|
||||
tok->str(tok->str() + c2);
|
||||
tok->deleteNext();
|
||||
continue;
|
||||
|
|
@ -2058,6 +2021,11 @@ bool Tokenizer::tokenize(std::istream &code,
|
|||
tok->deleteNext();
|
||||
}
|
||||
|
||||
else if (tok->str() == "<<" && tok->next()->str() == "=") {
|
||||
tok->str("<<=");
|
||||
tok->deleteNext();
|
||||
}
|
||||
|
||||
else if ((c1 == 'p' || c1 == '_') && tok->next()->str() == ":" && tok->strAt(2) != ":") {
|
||||
if (tok->str() == "private" || tok->str() == "protected" || tok->str() == "public" || tok->str() == "__published") {
|
||||
tok->str(tok->str() + ":");
|
||||
|
|
|
|||
|
|
@ -1763,7 +1763,7 @@ private:
|
|||
"\n"
|
||||
"template<typename T> inline B<T> h() { return B<T>(); }\n";
|
||||
|
||||
ASSERT_EQUALS(";", sizeof_(code));
|
||||
ASSERT_EQUALS("", sizeof_(code));
|
||||
|
||||
ASSERT_EQUALS("class A { } ;", sizeof_("class A{ template<typename T> int foo(T d);};"));
|
||||
}
|
||||
|
|
@ -2279,7 +2279,7 @@ private:
|
|||
"{ }";
|
||||
|
||||
// The expected result..
|
||||
const std::string expected(";");
|
||||
const std::string expected("");
|
||||
ASSERT_EQUALS(expected, sizeof_(code));
|
||||
}
|
||||
|
||||
|
|
@ -4487,7 +4487,7 @@ private:
|
|||
void simplifyTypedef39() {
|
||||
const char code[] = "typedef int A;\n"
|
||||
"template <const A, volatile A>::value;";
|
||||
const char expected[] = ";";
|
||||
const char expected[] = "";
|
||||
ASSERT_EQUALS(expected, tok(code, false));
|
||||
|
||||
checkSimplifyTypedef(code);
|
||||
|
|
@ -5253,7 +5253,7 @@ private:
|
|||
|
||||
void simplifyTypedef75() { // ticket #2426
|
||||
const char code[] = "typedef _Packed struct S { long l; }; \n";
|
||||
const std::string expected = ";";
|
||||
const std::string expected = "";
|
||||
ASSERT_EQUALS(expected, sizeof_(code));
|
||||
ASSERT_EQUALS("", errout.str());
|
||||
}
|
||||
|
|
@ -6950,7 +6950,7 @@ private:
|
|||
|
||||
void enum20() { // ticket #2600 segmentation fault
|
||||
const char code[] = "enum { const }\n";
|
||||
ASSERT_EQUALS(";", tok(code, false));
|
||||
ASSERT_EQUALS("", tok(code, false));
|
||||
}
|
||||
|
||||
void enum21() { // ticket #2720 syntax error
|
||||
|
|
|
|||
Loading…
Reference in New Issue