walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Refactorizations in class Token: changed handling of different types of tokens. 

- Replace _isNumber, _isName, _isBoolean attributes by a single _type attribute (enum Token::Type), because not two of the old booleans could be true at the same time.
-> Add support for lots of different other kinds of tokens. (More precise checking of token type possible)
-> Replaced instant checking of type for Operators, etc. by a value calculated at creation time. (Faster checking)
This commit is contained in:
PKEuS 2012-04-23 21:05:26 +02:00
parent 8cbed66089
commit e0a3ca0845
4 changed files with 162 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
lib/token.cpp
View File

@ -33,9 +33,7 @@ Token::Token(Token **t) :
_next(0),
_previous(0),
_link(0),
_isName(false),
_isNumber(false),
_isBoolean(false),
_type(eNone),
_isUnsigned(false),
_isSigned(false),
_isPointerCompare(false),
@ -59,20 +57,56 @@ Token::~Token()
void Token::update_property_info()
{
if (!_str.empty()) {
_isName = bool(_str[0] == '_' || std::isalpha(_str[0]));
if (std::isdigit(_str[0]))
_isNumber = true;
else if (_str.length() > 1 && _str[0] == '-' && std::isdigit(_str[1]))
_isNumber = true;
if (_str == "true" || _str == "false")
_type = eBoolean;
else if (_str[0] == '_' || std::isalpha(_str[0])) { // Name
if (_varId)
_type = eVariable;
_type = eName;
} else if (std::isdigit(_str[0]) || (_str.length() > 1 && _str[0] == '-' && std::isdigit(_str[1])))
_type = eNumber;
else if (_str.length() > 1 && _str[0] == '"' && _str[_str.length()-1] == '"')
_type = eString;
else if (_str.length() > 1 && _str[0] == '\'' && _str[_str.length()-1] == '\'')
_type = eChar;
else if (_str == "=" ||
_str == "+=" ||
_str == "-=" ||
_str == "*=" ||
_str == "/=" ||
_str == "%=" ||
_str == "&=" ||
_str == "^=" ||
_str == "|=" ||
_str == "<<=" ||
_str == ">>=")
_type = eAssignmentOp;
else if (_str.size() == 1 && _str.find_first_of(",[]()?:") != std::string::npos)
_type = eExtendedOp;
else if (_str=="<<" || _str==">>" || (_str.size()==1 && _str.find_first_of("+-*/%") != std::string::npos))
_type = eArithmeticalOp;
else if (_str.size() == 1 && _str.find_first_of("&|^~") != std::string::npos)
_type = eBitOp;
else if (_str == "&&" ||
_str == "||" ||
_str == "!")
_type = eLogicalOp;
else if (_str == "==" ||
_str == "!=" ||
_str == "<" ||
_str == "<=" ||
_str == ">" ||
_str == ">=")
_type = eComparisionOp;
else if (_str == "++" ||
_str == "--")
_type = eIncDecOp;
else if (_str.size() == 1 && (_str.find_first_of("{}") != std::string::npos || (_link && _str.find_first_of("<>") != std::string::npos)))
_type = eBracket;
else
_isNumber = false;
_isBoolean = (_str == "true" || _str == "false");
_type = eOther;
} else {
_isName = false;
_isNumber = false;
_isBoolean = false;
_type = eNone;
}
update_property_isStandardType();
@ -89,6 +123,7 @@ void Token::update_property_isStandardType()
for (int i = 0; type[i]; i++) {
if (_str == type[i]) {
_isStandardType = true;
_type = eType;
break;
}
}
@ -137,9 +172,7 @@ void Token::deleteThis()
{
if (_next) { // Copy next to this and delete next
_str = _next->_str;
_isName = _next->_isName;
_isNumber = _next->_isNumber;
_isBoolean = _next->_isBoolean;
_type = _next->_type;
_isUnsigned = _next->_isUnsigned;
_isSigned = _next->_isSigned;
_isPointerCompare = _next->_isPointerCompare;
@ -157,9 +190,7 @@ void Token::deleteThis()
deleteNext();
} else if (_previous && _previous->_previous) { // Copy previous to this and delete previous
_str = _previous->_str;
_isName = _previous->_isName;
_isNumber = _previous->_isNumber;
_isBoolean = _previous->_isBoolean;
_type = _previous->_type;
_isUnsigned = _previous->_isUnsigned;
_isSigned = _previous->_isSigned;
_isPointerCompare = _previous->_isPointerCompare;

64
lib/token.h
View File

@ -44,6 +44,15 @@ private:
Token();
public:
enum Type {
eVariable, eType, eFunction, eName, // Names: Variable (varId), Type (typeId, later), Function (FuncId, later), Name (unknown identifier)
eNumber, eString, eChar, eBoolean, eLiteral, // Literals: Number, String, Character, User defined literal (C++11)
eArithmeticalOp, eComparisionOp, eAssignmentOp, eLogicalOp, eBitOp, eIncDecOp, eExtendedOp, // Operators: Arithmetical, Comparision, Assignment, Logical, Bitwise, ++/--, Extended
eBracket, // {, }, <, >: < and > only if link() is set. Otherwise they are comparision operators.
eOther,
eNone
};
explicit Token(Token **tokensBack);
~Token();
@ -150,56 +159,39 @@ public:
**/
static std::size_t getStrLength(const Token *tok);
bool isName() const {
return _isName;
Type type() const {
return _type;
}
void isName(bool name) {
_isName = name;
void type(Type t) {
_type = t;
}
bool isName() const {
return _type == eName || _type == eType || _type == eVariable || _type == eFunction ||
_type == eBoolean; // TODO: "true"/"false" aren't really a name...
}
bool isNumber() const {
return _isNumber;
}
void isNumber(bool number) {
_isNumber = number;
return _type == eNumber;
}
bool isArithmeticalOp() const {
return (_str=="<<" || _str==">>" || (_str.size()==1 && _str.find_first_of("+-*/%") != std::string::npos));
return _type == eArithmeticalOp;
}
bool isOp() const {
return (isArithmeticalOp() ||
_str == "&&" ||
_str == "||" ||
_str == "==" ||
_str == "!=" ||
_str == "<" ||
_str == "<=" ||
_str == ">" ||
_str == ">=" ||
(_str.size() == 1 && _str.find_first_of("&|^~!") != std::string::npos));
_type == eLogicalOp ||
_type == eComparisionOp ||
_type == eBitOp);
}
bool isExtendedOp() const {
return isOp() ||
(_str.size() == 1 && _str.find_first_of(",[]()?:") != std::string::npos);
_type == eExtendedOp;
}
bool isAssignmentOp() const {
return (_str == "=" ||
_str == "+=" ||
_str == "-=" ||
_str == "*=" ||
_str == "/=" ||
_str == "%=" ||
_str == "&=" ||
_str == "^=" ||
_str == "|=" ||
_str == "<<=" ||
_str == ">>=");
return _type == eAssignmentOp;
}
bool isBoolean() const {
return _isBoolean;
}
void isBoolean(bool boolean) {
_isBoolean = boolean;
return _type == eBoolean;
}
bool isUnsigned() const {
return _isUnsigned;
}
@ -481,9 +473,7 @@ private:
Token *_previous;
Token *_link;
bool _isName;
bool _isNumber;
bool _isBoolean;
Type _type;
bool _isUnsigned;
bool _isSigned;
bool _isPointerCompare;

8
lib/tokenize.cpp
View File

@ -212,9 +212,7 @@ void Tokenizer::insertTokens(Token *dest, const Token *src, unsigned int n)
dest->fileIndex(src->fileIndex());
dest->linenr(src->linenr());
dest->varId(src->varId());
dest->isName(src->isName());
dest->isNumber(src->isNumber());
dest->isBoolean(src->isBoolean());
dest->type(src->type());
dest->isUnsigned(src->isUnsigned());
dest->isSigned(src->isSigned());
dest->isPointerCompare(src->isPointerCompare());
@ -238,9 +236,7 @@ Token *Tokenizer::copyTokens(Token *dest, const Token *first, const Token *last,
tok2 = tok2->next();
tok2->fileIndex(commonFileIndex);
tok2->linenr(linenrs);
tok2->isName(tok->isName());
tok2->isNumber(tok->isNumber());
tok2->isBoolean(tok->isBoolean());
tok2->type(tok->type());
tok2->isUnsigned(tok->isUnsigned());
tok2->isSigned(tok->isSigned());
tok2->isPointerCompare(tok->isPointerCompare());

100
test/testtoken.cpp
View File

@ -32,7 +32,9 @@ public:
private:
std::vector<std::string> arithmeticalOps;
std::vector<std::string> normalOps;
std::vector<std::string> logicalOps;
std::vector<std::string> bitOps;
std::vector<std::string> comparisionOps;
std::vector<std::string> extendedOps;
std::vector<std::string> assignmentOps;
@ -66,6 +68,8 @@ private:
TEST_CASE(isExtendedOp);
TEST_CASE(isAssignmentOp);
TEST_CASE(isStandardType);
TEST_CASE(literals);
TEST_CASE(operators);
TEST_CASE(updateProperties)
TEST_CASE(updatePropertiesConcatStr)
@ -382,19 +386,19 @@ private:
arithmeticalOps.push_back("<<");
arithmeticalOps.push_back(">>");
normalOps.push_back("&&");
normalOps.push_back("||");
normalOps.push_back("==");
normalOps.push_back("!=");
normalOps.push_back("<");
normalOps.push_back("<=");
normalOps.push_back(">");
normalOps.push_back(">=");
normalOps.push_back("&");
normalOps.push_back("|");
normalOps.push_back("^");
normalOps.push_back("~");
normalOps.push_back("!");
logicalOps.push_back("&&");
logicalOps.push_back("||");
logicalOps.push_back("!");
comparisionOps.push_back("==");
comparisionOps.push_back("!=");
comparisionOps.push_back("<");
comparisionOps.push_back("<=");
comparisionOps.push_back(">");
comparisionOps.push_back(">=");
bitOps.push_back("&");
bitOps.push_back("|");
bitOps.push_back("^");
bitOps.push_back("~");
extendedOps.push_back(",");
extendedOps.push_back("[");
@ -420,7 +424,9 @@ private:
void matchOp() {
std::vector<std::string> test_ops;
append_vector(test_ops, arithmeticalOps);
append_vector(test_ops, normalOps);
append_vector(test_ops, bitOps);
append_vector(test_ops, comparisionOps);
append_vector(test_ops, logicalOps);
std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end();
for (test_op = test_ops.begin(); test_op != test_ops_end; ++test_op) {
@ -448,7 +454,9 @@ private:
// Negative test against other operators
std::vector<std::string> other_ops;
append_vector(other_ops, normalOps);
append_vector(other_ops, bitOps);
append_vector(other_ops, comparisionOps);
append_vector(other_ops, logicalOps);
append_vector(other_ops, extendedOps);
append_vector(other_ops, assignmentOps);
@ -463,7 +471,9 @@ private:
void isOp() {
std::vector<std::string> test_ops;
append_vector(test_ops, arithmeticalOps);
append_vector(test_ops, normalOps);
append_vector(test_ops, bitOps);
append_vector(test_ops, comparisionOps);
append_vector(test_ops, logicalOps);
std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end();
for (test_op = test_ops.begin(); test_op != test_ops_end; ++test_op) {
@ -488,7 +498,9 @@ private:
void isExtendedOp() {
std::vector<std::string> test_ops;
append_vector(test_ops, arithmeticalOps);
append_vector(test_ops, normalOps);
append_vector(test_ops, bitOps);
append_vector(test_ops, comparisionOps);
append_vector(test_ops, logicalOps);
append_vector(test_ops, extendedOps);
std::vector<std::string>::const_iterator test_op, test_ops_end = test_ops.end();
@ -518,7 +530,9 @@ private:
// Negative test against other operators
std::vector<std::string> other_ops;
append_vector(other_ops, arithmeticalOps);
append_vector(other_ops, normalOps);
append_vector(other_ops, bitOps);
append_vector(other_ops, comparisionOps);
append_vector(other_ops, logicalOps);
append_vector(other_ops, extendedOps);
std::vector<std::string>::const_iterator other_op, other_ops_end = other_ops.end();
@ -529,6 +543,54 @@ private:
}
}
void operators() {
std::vector<std::string>::const_iterator test_op;
for (test_op = extendedOps.begin(); test_op != extendedOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eExtendedOp, tok.type());
}
for (test_op = logicalOps.begin(); test_op != logicalOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eLogicalOp, tok.type());
}
for (test_op = bitOps.begin(); test_op != bitOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eBitOp, tok.type());
}
for (test_op = comparisionOps.begin(); test_op != comparisionOps.end(); ++test_op) {
Token tok(NULL);
tok.str(*test_op);
ASSERT_EQUALS(Token::eComparisionOp, tok.type());
}
Token tok(NULL);
tok.str("++");
ASSERT_EQUALS(Token::eIncDecOp, tok.type());
tok.str("--");
ASSERT_EQUALS(Token::eIncDecOp, tok.type());
}
void literals() {
Token tok(NULL);
tok.str("\"foo\"");
ASSERT(tok.type() == Token::eString);
tok.str("\"\"");
ASSERT(tok.type() == Token::eString);
tok.str("'f'");
ASSERT(tok.type() == Token::eChar);
tok.str("12345");
ASSERT(tok.type() == Token::eNumber);
tok.str("-55");
ASSERT(tok.type() == Token::eNumber);
tok.str("true");
ASSERT(tok.type() == Token::eBoolean);
tok.str("false");
ASSERT(tok.type() == Token::eBoolean);
}
void isStandardType() {
std::vector<std::string> standard_types;
standard_types.push_back("bool");