walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Refactorizations in class Token: changed handling of different types of tokens. 

- Replace _isNumber, _isName, _isBoolean attributes by a single _type attribute (enum Token::Type), because not two of the old booleans could be true at the same time.
-> Add support for lots of different other kinds of tokens. (More precise checking of token type possible)
-> Replaced instant checking of type for Operators, etc. by a value calculated at creation time. (Faster checking)
This commit is contained in:
PKEuS 2012-04-23 21:05:26 +02:00
parent 8cbed66089
commit e0a3ca0845
4 changed files with 162 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
lib/token.cpp
View File

@ -33,9 +33,7 @@ Token::Token(Token **t) :
_next(0), _next(0),
_previous(0), _previous(0),
_link(0), _link(0),
_isName(false), _type(eNone),
_isNumber(false),
_isBoolean(false),
_isUnsigned(false), _isUnsigned(false),
_isSigned(false), _isSigned(false),
_isPointerCompare(false), _isPointerCompare(false),
@ -59,20 +57,56 @@ Token::~Token()
void Token::update_property_info() void Token::update_property_info()
{ {
if (!_str.empty()) { if (!_str.empty()) {
_isName = bool(_str[0] == '_' || std::isalpha(_str[0])); if (_str == "true" || _str == "false")
_type = eBoolean;
if (std::isdigit(_str[0])) else if (_str[0] == '_' || std::isalpha(_str[0])) { // Name
_isNumber = true; if (_varId)
else if (_str.length() > 1 && _str[0] == '-' && std::isdigit(_str[1])) _type = eVariable;
_isNumber = true; _type = eName;
} else if (std::isdigit(_str[0]) || (_str.length() > 1 && _str[0] == '-' && std::isdigit(_str[1])))
_type = eNumber;
else if (_str.length() > 1 && _str[0] == '"' && _str[_str.length()-1] == '"')
_type = eString;
else if (_str.length() > 1 && _str[0] == '\'' && _str[_str.length()-1] == '\'')
_type = eChar;
else if (_str == "=" ||
_str == "+=" ||
_str == "-=" ||
_str == "*=" ||
_str == "/=" ||
_str == "%=" ||
_str == "&=" ||
_str == "^=" ||
_str == "|=" ||
_str == "<<=" ||
_str == ">>=")
_type = eAssignmentOp;
else if (_str.size() == 1 && _str.find_first_of(",[]()?:") != std::string::npos)
_type = eExtendedOp;
else if (_str=="<<" || _str==">>" || (_str.size()==1 && _str.find_first_of("+-*/%") != std::string::npos))
_type = eArithmeticalOp;
else if (_str.size() == 1 && _str.find_first_of("&|^~") != std::string::npos)
_type = eBitOp;
else if (_str == "&&" ||
_str == "||" ||
_str == "!")
_type = eLogicalOp;
else if (_str == "==" ||
_str == "!=" ||
_str == "<" ||
_str == "<=" ||
_str == ">" ||
_str == ">=")
_type = eComparisionOp;
else if (_str == "++" ||
_str == "--")
_type = eIncDecOp;
else if (_str.size() == 1 && (_str.find_first_of("{}") != std::string::npos || (_link && _str.find_first_of("<>") != std::string::npos)))
_type = eBracket;
else else
_isNumber = false; _type = eOther;
_isBoolean = (_str == "true" || _str == "false");
} else { } else {
_isName = false; _type = eNone;
_isNumber = false;
_isBoolean = false;
} }
update_property_isStandardType(); update_property_isStandardType();
@ -89,6 +123,7 @@ void Token::update_property_isStandardType()
for (int i = 0; type[i]; i++) { for (int i = 0; type[i]; i++) {
if (_str == type[i]) { if (_str == type[i]) {
_isStandardType = true; _isStandardType = true;
_type = eType;
break; break;
} }
} }
@ -137,9 +172,7 @@ void Token::deleteThis()
{ {
if (_next) { // Copy next to this and delete next if (_next) { // Copy next to this and delete next
_str = _next->_str; _str = _next->_str;
_isName = _next->_isName; _type = _next->_type;
_isNumber = _next->_isNumber;
_isBoolean = _next->_isBoolean;
_isUnsigned = _next->_isUnsigned; _isUnsigned = _next->_isUnsigned;
_isSigned = _next->_isSigned; _isSigned = _next->_isSigned;
_isPointerCompare = _next->_isPointerCompare; _isPointerCompare = _next->_isPointerCompare;
@ -157,9 +190,7 @@ void Token::deleteThis()
deleteNext(); deleteNext();
} else if (_previous && _previous->_previous) { // Copy previous to this and delete previous } else if (_previous && _previous->_previous) { // Copy previous to this and delete previous
_str = _previous->_str; _str = _previous->_str;
_isName = _previous->_isName; _type = _previous->_type;
_isNumber = _previous->_isNumber;
_isBoolean = _previous->_isBoolean;
_isUnsigned = _previous->_isUnsigned; _isUnsigned = _previous->_isUnsigned;
_isSigned = _previous->_isSigned; _isSigned = _previous->_isSigned;
_isPointerCompare = _previous->_isPointerCompare; _isPointerCompare = _previous->_isPointerCompare;

64
lib/token.h
View File

@ -44,6 +44,15 @@ private:
Token(); Token();
public: public:
enum Type {
eVariable, eType, eFunction, eName, // Names: Variable (varId), Type (typeId, later), Function (FuncId, later), Name (unknown identifier)
eNumber, eString, eChar, eBoolean, eLiteral, // Literals: Number, String, Character, User defined literal (C++11)
eArithmeticalOp, eComparisionOp, eAssignmentOp, eLogicalOp, eBitOp, eIncDecOp, eExtendedOp, // Operators: Arithmetical, Comparision, Assignment, Logical, Bitwise, ++/--, Extended
eBracket, // {, }, <, >: < and > only if link() is set. Otherwise they are comparision operators.
eOther,
eNone
};
explicit Token(Token **tokensBack); explicit Token(Token **tokensBack);
~Token(); ~Token();
@ -150,56 +159,39 @@ public:
**/ **/
static std::size_t getStrLength(const Token *tok); static std::size_t getStrLength(const Token *tok);
bool isName() const { Type type() const {
return _isName; return _type;
} }
void isName(bool name) { void type(Type t) {
_isName = name; _type = t;
}
bool isName() const {
return _type == eName || _type == eType || _type == eVariable || _type == eFunction ||
_type == eBoolean; // TODO: "true"/"false" aren't really a name...
} }
bool isNumber() const { bool isNumber() const {
return _isNumber; return _type == eNumber;
}
void isNumber(bool number) {
_isNumber = number;
} }
bool isArithmeticalOp() const { bool isArithmeticalOp() const {
return (_str=="<<" || _str==">>" || (_str.size()==1 && _str.find_first_of("+-*/%") != std::string::npos)); return _type == eArithmeticalOp;
} }
bool isOp() const { bool isOp() const {
return (isArithmeticalOp() || return (isArithmeticalOp() ||
_str == "&&" || _type == eLogicalOp ||
_str == "||" || _type == eComparisionOp ||
_str == "==" || _type == eBitOp);
_str == "!=" ||
_str == "<" ||
_str == "<=" ||
_str == ">" ||
_str == ">=" ||
(_str.size() == 1 && _str.find_first_of("&|^~!") != std::string::npos));
} }
bool isExtendedOp() const { bool isExtendedOp() const {
return isOp() || return isOp() ||
(_str.size() == 1 && _str.find_first_of(",[]()?:") != std::string::npos); _type == eExtendedOp;
} }
bool isAssignmentOp() const { bool isAssignmentOp() const {
return (_str == "=" || return _type == eAssignmentOp;
_str == "+=" ||
_str == "-=" ||
_str == "*=" ||
_str == "/=" ||
_str == "%=" ||
_str == "&=" ||
_str == "^=" ||
_str == "|=" ||
_str == "<<=" ||
_str == ">>=");
} }
bool isBoolean() const { bool isBoolean() const {
return _isBoolean; return _type == eBoolean;
}
void isBoolean(bool boolean) {
_isBoolean = boolean;
} }
bool isUnsigned() const { bool isUnsigned() const {
return _isUnsigned; return _isUnsigned;
} }
@ -481,9 +473,7 @@ private:
Token *_previous; Token *_previous;
Token *_link; Token *_link;
bool _isName; Type _type;
bool _isNumber;
bool _isBoolean;
bool _isUnsigned; bool _isUnsigned;
bool _isSigned; bool _isSigned;
bool _isPointerCompare; bool _isPointerCompare;

8
lib/tokenize.cpp
View File

@ -212,9 +212,7 @@ void Tokenizer::insertTokens(Token *dest, const Token *src, unsigned int n)
dest->fileIndex(src->fileIndex()); dest->fileIndex(src->fileIndex());
dest->linenr(src->linenr()); dest->linenr(src->linenr());
dest->varId(src->varId()); dest->varId(src->varId());
dest->isName(src->isName()); dest->type(src->type());
dest->isNumber(src->isNumber());
dest->isBoolean(src->isBoolean());
dest->isUnsigned(src->isUnsigned()); dest->isUnsigned(src->isUnsigned());
dest->isSigned(src->isSigned()); dest->isSigned(src->isSigned());
dest->isPointerCompare(src->isPointerCompare()); dest->isPointerCompare(src->isPointerCompare());
@ -238,9 +236,7 @@ Token *Tokenizer::copyTokens(Token *dest, const Token *first, const Token *last,
tok2 = tok2->next(); tok2 = tok2->next();
tok2->fileIndex(commonFileIndex); tok2->fileIndex(commonFileIndex);
tok2->linenr(linenrs); tok2->linenr(linenrs);
tok2->isName(tok->isName()); tok2->type(tok->type());
tok2->isNumber(tok->isNumber());
tok2->isBoolean(tok->isBoolean());
tok2->isUnsigned(tok->isUnsigned()); tok2->isUnsigned(tok->isUnsigned());
tok2->isSigned(tok->isSigned()); tok2->isSigned(tok->isSigned());
tok2->isPointerCompare(tok->isPointerCompare()); tok2->isPointerCompare(tok->isPointerCompare());

100
test/testtoken.cpp
View File

@ -32,7 +32,9 @@ public:
private: private:
std::vector<std::string> arithmeticalOps; std::vector<std::string> arithmeticalOps;
std::vector<std::string> normalOps; std::vector<std::string> logicalOps;
std::vector<std::string> bitOps;
std::vector<std::string> comparisionOps;
std::vector<std::string> extendedOps; std::vector<std::string> extendedOps;
std::vector<std::string> assignmentOps; std::vector<std::string> assignmentOps;
@ -66,6 +68,8 @@ private:
TEST_CASE(isExtendedOp); TEST_CASE(isExtendedOp);
TEST_CASE(isAssignmentOp); TEST_CASE(isAssignmentOp);
TEST_CASE(isStandardType); TEST_CASE(isStandardType);
TEST_CASE(literals);
TEST_CASE(operators);
TEST_CASE(updateProperties) TEST_CASE(updateProperties)
TEST_CASE(updatePropertiesConcatStr) TEST_CASE(updatePropertiesConcatStr)
@ -382,19 +386,19 @@ private:
arithmeticalOps.push_back("<<"); arithmeticalOps.push_back("<<");
arithmeticalOps.push_back(">>"); arithmeticalOps.push_back(">>");
normalOps.push_back("&&"); logicalOps.push_back("&&");
normalOps.push_back("||"); logicalOps.push_back("||");
normalOps.push_back("=="); logicalOps.push_back("!");
normalOps.push_back("!="); comparisionOps.push_back("==");
normalOps.push_back("<"); comparisionOps.push_back("!=");
normalOps.push_back("<="); comparisionOps.push_back("<");
normalOps.push_back(">"); comparisionOps.push_back("<=");
normalOps.push_back(">="); comparisionOps.push_back(">");
normalOps.push_back("&"); comparisionOps.push_back(">=");
normalOps.push_back("|"); bitOps.push_back("&");
normalOps.push_back("^"); bitOps.push_back("|");
normalOps.push_back("~"); bitOps.push_back("^");
normalOps.push_back("!"); bitOps.push_back("~");
extendedOps.push_back(","); extendedOps.push_back(",");
extendedOps.push_back("["); extendedOps.push_back("[");
@ -420,7 +424,9 @@ private:
void matchOp() { void matchOp() {
std::vector<std::string> test_ops; std::vector<std::string> test_ops;
append_vector(test_ops, arithmeticalOps); append_vector(test_ops, arithmeticalOps);
append_vector(test_ops, normalOps); append_vector(test_ops, bitOps);
append_vector(test_ops, comparisionOps);
append_vector(test_ops, logicalOps);
std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end(); std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end();
for (test_op = test_ops.begin(); test_op != test_ops_end; ++test_op) { for (test_op = test_ops.begin(); test_op != test_ops_end; ++test_op) {
@ -448,7 +454,9 @@ private:
// Negative test against other operators // Negative test against other operators
std::vector<std::string> other_ops; std::vector<std::string> other_ops;
append_vector(other_ops, normalOps); append_vector(other_ops, bitOps);
append_vector(other_ops, comparisionOps);
append_vector(other_ops, logicalOps);
append_vector(other_ops, extendedOps); append_vector(other_ops, extendedOps);
append_vector(other_ops, assignmentOps); append_vector(other_ops, assignmentOps);
@ -463,7 +471,9 @@ private:
void isOp() { void isOp() {
std::vector<std::string> test_ops; std::vector<std::string> test_ops;
append_vector(test_ops, arithmeticalOps); append_vector(test_ops, arithmeticalOps);
append_vector(test_ops, normalOps); append_vector(test_ops, bitOps);
append_vector(test_ops, comparisionOps);
append_vector(test_ops, logicalOps);
std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end(); std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end();
for (test_op = test_ops.begin(); test_op != test_ops_end; ++test_op) { for (test_op = test_ops.begin(); test_op != test_ops_end; ++test_op) {
@ -488,7 +498,9 @@ private:
void isExtendedOp() { void isExtendedOp() {
std::vector<std::string> test_ops; std::vector<std::string> test_ops;
append_vector(test_ops, arithmeticalOps); append_vector(test_ops, arithmeticalOps);
append_vector(test_ops, normalOps); append_vector(test_ops, bitOps);
append_vector(test_ops, comparisionOps);
append_vector(test_ops, logicalOps);
append_vector(test_ops, extendedOps); append_vector(test_ops, extendedOps);
std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end(); std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end();
@ -518,7 +530,9 @@ private:
// Negative test against other operators // Negative test against other operators
std::vector<std::string> other_ops; std::vector<std::string> other_ops;
append_vector(other_ops, arithmeticalOps); append_vector(other_ops, arithmeticalOps);
append_vector(other_ops, normalOps); append_vector(other_ops, bitOps);
append_vector(other_ops, comparisionOps);
append_vector(other_ops, logicalOps);
append_vector(other_ops, extendedOps); append_vector(other_ops, extendedOps);
std::vector<std::string>::const_iterator other_op, other_ops_end = other_ops.end(); std::vector<std::string>::const_iterator other_op, other_ops_end = other_ops.end();
@ -529,6 +543,54 @@ private:
} }
} }
void operators() {
std::vector<std::string>::const_iterator test_op;
for (test_op = extendedOps.begin(); test_op != extendedOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eExtendedOp, tok.type());
}
for (test_op = logicalOps.begin(); test_op != logicalOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eLogicalOp, tok.type());
}
for (test_op = bitOps.begin(); test_op != bitOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eBitOp, tok.type());
}
for (test_op = comparisionOps.begin(); test_op != comparisionOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eComparisionOp, tok.type());
}
Token tok(NULL);
tok.str("++");
ASSERT_EQUALS(Token::eIncDecOp, tok.type());
tok.str("--");
ASSERT_EQUALS(Token::eIncDecOp, tok.type());
}
void literals() {
Token tok(NULL);
tok.str("\"foo\"");
ASSERT(tok.type() == Token::eString);
tok.str("\"\"");
ASSERT(tok.type() == Token::eString);
tok.str("'f'");
ASSERT(tok.type() == Token::eChar);
tok.str("12345");
ASSERT(tok.type() == Token::eNumber);
tok.str("-55");
ASSERT(tok.type() == Token::eNumber);
tok.str("true");
ASSERT(tok.type() == Token::eBoolean);
tok.str("false");
ASSERT(tok.type() == Token::eBoolean);
}
void isStandardType() { void isStandardType() {
std::vector<std::string> standard_types; std::vector<std::string> standard_types;
standard_types.push_back("bool"); standard_types.push_back("bool");