walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Handle prefixed strings and characters in Token (#1742) 

This makes it possible to call getStrLength() and similar functions
before the tokenizer is called.
This commit is contained in:
Rikard Falkeborn 2019-03-18 06:18:25 +01:00 committed by Daniel Marjamäki
parent b53a2e5dc4
commit 794f65bac1
4 changed files with 39 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lib/token.cpp
View File

@ -34,14 +34,17 @@
#include <stack>
#include <utility>
static const std::string literal_prefix[4] = {"u8", "u", "U", "L"};
static bool isStringCharLiteral(const std::string &str, char q)
{
if (!endsWith(str, q))
return false;
if (str[0] == q && str.length() > 1)
return true;
const std::string prefix[5] = { "", "u8", "L", "U", "u" };
for (const std::string & p: prefix) {
for (const std::string & p: literal_prefix) {
if ((str.length() + 1) > p.length() && (str.compare(0, p.size() + 1, (p + q)) == 0))
return true;
}
@ -131,6 +134,7 @@ void Token::update_property_info()
tokType(eNone);
}
update_property_char_string_literal();
update_property_isStandardType();
}
@ -160,6 +164,20 @@ void Token::update_property_isStandardType()
}
}
void Token::update_property_char_string_literal()
{
if (!(mTokType == Token::eString || mTokType == Token::eChar)) // Token has already been updated
return;
for (const std::string & p : literal_prefix) {
if (((mTokType == Token::eString) && mStr.compare(0, p.size() + 1, p + "\"") == 0) ||
((mTokType == Token::eChar) && (mStr.compare(0, p.size() + 1, p + "\'") == 0))) {
mStr = mStr.substr(p.size());
isLong(p != "u8");
break;
}
}
}
bool Token::isUpperCaseName() const
{

3
lib/token.h
View File

@ -1069,6 +1069,9 @@ private:
/** Update internal property cache about isStandardType() */
void update_property_isStandardType();
/** Update internal property cache about string and char literals */
void update_property_char_string_literal();
public:
void astOperand1(Token *tok);
void astOperand2(Token *tok);

12
lib/tokenize.cpp
View File

@ -1928,18 +1928,6 @@ void Tokenizer::combineOperators()
void Tokenizer::combineStringAndCharLiterals()
{
for (Token *tok = list.front(); tok; tok = tok->next()) {
const std::string prefix[4] = {"u8", "L", "U", "u"};
for (const std::string & p : prefix) {
if (((tok->tokType() == Token::eString) && (tok->str().find(p + "\"") == 0)) ||
((tok->tokType() == Token::eChar) && (tok->str().find(p + "\'") == 0))) {
tok->str(tok->str().substr(p.size()));
tok->isLong(p != "u8");
break;
}
}
}
// Combine strings
for (Token *tok = list.front();
tok;

20
test/testtoken.cpp
View File

@ -267,16 +267,28 @@ private:
Token tok;
tok.str("\"\"");
ASSERT_EQUALS(0, (int)Token::getStrLength(&tok));
ASSERT_EQUALS(0, Token::getStrLength(&tok));
tok.str("\"test\"");
ASSERT_EQUALS(4, (int)Token::getStrLength(&tok));
ASSERT_EQUALS(4, Token::getStrLength(&tok));
tok.str("\"test \\\\test\"");
ASSERT_EQUALS(10, (int)Token::getStrLength(&tok));
ASSERT_EQUALS(10, Token::getStrLength(&tok));
tok.str("\"a\\0\"");
ASSERT_EQUALS(1, (int)Token::getStrLength(&tok));
ASSERT_EQUALS(1, Token::getStrLength(&tok));
tok.str("L\"\"");
ASSERT_EQUALS(0, Token::getStrLength(&tok));
tok.str("u8\"test\"");
ASSERT_EQUALS(4, Token::getStrLength(&tok));
tok.str("U\"test \\\\test\"");
ASSERT_EQUALS(10, Token::getStrLength(&tok));
tok.str("u\"a\\0\"");
ASSERT_EQUALS(1, Token::getStrLength(&tok));
}
void getStrSize() const {