Modify template simplifier to add forward declarations of some templa… (#1489)

* Modify template simplifier to add forward declarations of some template functions so symbol database can make sense of the expanded templates.

* Fix travis.
This commit is contained in:
IOBYTE 2018-11-23 05:36:09 -05:00 committed by Daniel Marjamäki
parent ef05be2600
commit 358f0c473d
4 changed files with 272 additions and 33 deletions

View File

@ -53,8 +53,8 @@ namespace {
};
}
TemplateSimplifier::TokenAndName::TokenAndName(Token *tok, const std::string &s, const std::string &n) :
token(tok), scope(s), name(n)
TemplateSimplifier::TokenAndName::TokenAndName(Token *tok, const std::string &s, const std::string &n, const Token *nt) :
token(tok), scope(s), name(n), nameToken(nt)
{
token->hasTemplateSimplifierPointer(true);
}
@ -513,14 +513,14 @@ bool TemplateSimplifier::getTemplateDeclarations()
else if (tok2->str() == ";") {
const int namepos = getTemplateNamePosition(parmEnd, true);
if (namepos > 0)
mTemplateForwardDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos));
mTemplateForwardDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos), parmEnd->tokAt(namepos));
break;
}
// Implementation => add to mTemplateDeclarations
else if (tok2->str() == "{") {
const int namepos = getTemplateNamePosition(parmEnd, false);
if (namepos > 0)
mTemplateDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos));
mTemplateDeclarations.emplace_back(tok, getScopeName(scopeInfo), parmEnd->strAt(namepos), parmEnd->tokAt(namepos));
break;
}
}
@ -549,7 +549,7 @@ void TemplateSimplifier::getTemplateInstantiations()
const Token *tok2 = Token::findmatch(tok, "{|;");
if (tok2 && tok2->str() == "{")
tok = tok2->link();
} else if (Token::Match(tok->previous(), "[({};=] %name% ::|<") ||
} else if (Token::Match(tok->previous(), "(|{|}|;|=|>|<<|:|. %name% ::|<") ||
Token::Match(tok->previous(), "%type% %name% ::|<") ||
Token::Match(tok->tokAt(-2), "[,:] private|protected|public %name% ::|<")) {
@ -570,7 +570,7 @@ void TemplateSimplifier::getTemplateInstantiations()
for (; tok2 && tok2 != tok; tok2 = tok2->previous()) {
if (Token::Match(tok2, ", %name% <") &&
templateParameters(tok2->tokAt(2))) {
mTemplateInstantiations.emplace_back(tok2->next(), getScopeName(scopeList), tok2->strAt(1));
mTemplateInstantiations.emplace_back(tok2->next(), getScopeName(scopeList), tok2->strAt(1), tok2->tokAt(1));
} else if (Token::Match(tok2->next(), "class|struct"))
const_cast<Token *>(tok2)->deleteNext();
@ -583,11 +583,11 @@ void TemplateSimplifier::getTemplateInstantiations()
const std::string fullName = scopeName + (scopeName.empty()?"":" :: ") + tok->str();
const std::list<TokenAndName>::const_iterator it = std::find_if(mTemplateDeclarations.begin(), mTemplateDeclarations.end(), FindName(fullName));
if (it != mTemplateDeclarations.end()) {
mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), fullName);
mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), fullName, tok);
break;
} else {
if (scopeName.empty()) {
mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), scopeName1 + (scopeName1.empty()?"":" :: ") + tok->str());
mTemplateInstantiations.emplace_back(tok, getScopeName(scopeList), scopeName1 + (scopeName1.empty()?"":" :: ") + tok->str(), tok);
break;
}
const std::string::size_type pos = scopeName.rfind(" :: ");
@ -831,7 +831,7 @@ void TemplateSimplifier::simplifyTemplateAliases()
mTemplateInstantiations.end(),
FindToken(tok1));
if (it != mTemplateInstantiations.end())
mTemplateInstantiations.emplace_back(tok2, it->scope, it->name);
mTemplateInstantiations.emplace_back(tok2, it->scope, it->name, it->nameToken);
}
continue;
}
@ -1016,12 +1016,66 @@ void TemplateSimplifier::expandTemplate(
const Token *endOfTemplateDefinition = nullptr;
const Token * const templateDeclarationNameToken = templateDeclarationToken->tokAt(getTemplateNamePosition(templateDeclarationToken));
const bool isClass = Token::Match(templateDeclarationToken->next(), "class|struct|union %name% <|{|:");
const bool isFunction = templateDeclarationNameToken->strAt(1) == "(";
// add forward declaration for classes
// add forward declarations
if (copy && isClass) {
templateDeclaration.token->insertToken(templateDeclarationToken->strAt(1), "", true);
templateDeclaration.token->insertToken(newName, "", true);
templateDeclaration.token->insertToken(";", "", true);
} else if (copy && isFunction) {
// check if this is an explicit instantiation
Token * temp = templateInstantiation.token;
while (temp && !Token::Match(temp->previous(), "}|;"))
temp = temp->previous();
if (Token::Match(temp, "template !!<")) {
// just delete "template"
deleteToken(temp);
} else {
// add forward declaration
Token * dst = templateDeclaration.token;
Token * start;
Token * end;
auto it = mTemplateForwardDeclarationsMap.find(dst);
if (it != mTemplateForwardDeclarationsMap.end()) {
dst = it->second;
const Token * temp1 = dst->tokAt(1)->findClosingBracket();
const Token * temp2 = temp1->tokAt(getTemplateNamePosition(temp1));
start = temp1->next();
end = temp2->linkAt(1)->next();
} else {
start = templateDeclarationToken->next();
end = templateDeclarationNameToken->linkAt(1)->next();
}
std::map<const Token *, Token *> links;
while (start && start != end) {
unsigned int itype = 0;
while (itype < typeParametersInDeclaration.size() && typeParametersInDeclaration[itype]->str() != start->str())
++itype;
if (itype < typeParametersInDeclaration.size()) {
dst->insertToken(mTypesUsedInTemplateInstantiation[itype]->str(), "", true);
dst->previous()->isTemplateArg(true);
} else {
if (start->str() == templateDeclarationNameToken->str())
dst->insertToken(newName, "", true);
else
dst->insertToken(start->str(), "", true);
if (start->link()) {
if (Token::Match(start, "[|{|(")) {
links[start->link()] = dst->previous();
} else if (Token::Match(start, "]|}|)")) {
Token::createMutualLinks(links[start], dst->previous());
links.erase(start);
}
}
}
start = start->next();
}
dst->insertToken(";", "", true);
}
}
for (Token *tok3 = mTokenList.front(); tok3; tok3 = tok3 ? tok3->next() : nullptr) {
@ -1220,7 +1274,7 @@ void TemplateSimplifier::expandTemplate(
std::string name = tok3->str();
for (const Token *prev = tok3->tokAt(-2); Token::Match(prev, "%name% ::"); prev = prev->tokAt(-2))
name = prev->str() + " :: " + name;
mTemplateInstantiations.emplace_back(mTokenList.back(), getScopeName(scopeInfo), name);
mTemplateInstantiations.emplace_back(mTokenList.back(), getScopeName(scopeInfo), name, tok3);
}
// link() newly tokens manually
@ -1757,8 +1811,8 @@ bool TemplateSimplifier::simplifyTemplateInstantiations(
startToken = startToken->tokAt(-2);
}
if (Token::Match(startToken->previous(), "[;{}=]") &&
(!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*| %name%")))
if (Token::Match(startToken->previous(), ";|{|}|=|const") &&
(!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*|&| %name%")))
continue;
// New type..
@ -1823,8 +1877,8 @@ bool TemplateSimplifier::simplifyTemplateInstantiations(
startToken = startToken->tokAt(-2);
}
if (Token::Match(startToken->previous(), "[;{}=]") &&
(!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*| %name%")))
if (Token::Match(startToken->previous(), ";|{|}|=|const") &&
(!specialized && !instantiateMatch(tok2, typeParametersInDeclaration.size(), isfunc ? "(" : "*|&| %name%")))
return false;
// already simplified
@ -1988,8 +2042,35 @@ void TemplateSimplifier::fixForwardDeclaredDefaultArgumentValues()
// make sure the number of arguments match
if (params1.size() == params2.size()) {
std::string declName = decl.scope;
if (!declName.empty())
declName += " :: ";
if (decl.nameToken->strAt(-1) == "::") {
const Token * start = decl.nameToken;
while (Token::Match(start->tokAt(-2), "%name% ::"))
start = start->tokAt(-2);
while (start != decl.nameToken) {
declName += (start->str() + " ");
start = start->next();
}
}
declName += decl.name;
std::string forwardDeclName = forwardDecl.scope;
if (!forwardDeclName.empty())
forwardDeclName += " :: ";
forwardDeclName += forwardDecl.name;
// make sure the scopes and names match
if (forwardDecl.scope == decl.scope && forwardDecl.name == decl.name) {
if (forwardDeclName == declName) {
// save forward declaration for lookup later
if ((decl.nameToken->strAt(1) == "(" && forwardDecl.nameToken->strAt(1) == "(") ||
(decl.nameToken->strAt(1) == "{" && forwardDecl.nameToken->strAt(1) == ";")) {
mTemplateForwardDeclarationsMap[decl.token] = forwardDecl.token;
}
for (size_t k = 0; k < params1.size(); k++) {
// copy default value to declaration if not present
if (params1[k]->strAt(1) == "=" && params2[k]->strAt(1) != "=") {
@ -2018,6 +2099,7 @@ void TemplateSimplifier::simplifyTemplates(
if (i) {
mTemplateDeclarations.clear();
mTemplateForwardDeclarations.clear();
mTemplateForwardDeclarationsMap.clear();
mTemplateInstantiations.clear();
mInstantiatedTemplates.clear();
}

View File

@ -26,6 +26,7 @@
#include <ctime>
#include <list>
#include <map>
#include <set>
#include <string>
#include <vector>
@ -68,13 +69,14 @@ public:
* Token and its full scopename
*/
struct TokenAndName {
TokenAndName(Token *tok, const std::string &s, const std::string &n);
TokenAndName(Token *tok, const std::string &s, const std::string &n, const Token *nt);
bool operator == (const TokenAndName & rhs) const {
return token == rhs.token && scope == rhs.scope && name == rhs.name;
return token == rhs.token && scope == rhs.scope && name == rhs.name && nameToken == rhs.nameToken;
}
Token *token;
std::string scope;
std::string name;
const Token *nameToken;
};
/**
@ -263,6 +265,7 @@ private:
std::list<TokenAndName> mTemplateDeclarations;
std::list<TokenAndName> mTemplateForwardDeclarations;
std::map<Token *, Token *> mTemplateForwardDeclarationsMap;
std::list<TokenAndName> mTemplateInstantiations;
std::list<TokenAndName> mInstantiatedTemplates;
std::list<TokenAndName> mMemberFunctionsToDelete;

View File

@ -109,6 +109,14 @@ private:
TEST_CASE(template69); // #8791
TEST_CASE(template70); // #5289
TEST_CASE(template71); // #8821
TEST_CASE(template72);
TEST_CASE(template73);
TEST_CASE(template74);
TEST_CASE(template75);
TEST_CASE(template76);
TEST_CASE(template77);
TEST_CASE(template78);
TEST_CASE(template79); // #5133
TEST_CASE(template_specialization_1); // #7868 - template specialization template <typename T> struct S<C<T>> {..};
TEST_CASE(template_specialization_2); // #7868 - template specialization template <typename T> struct S<C<T>> {..};
TEST_CASE(template_enum); // #6299 Syntax error in complex enum declaration (including template)
@ -184,7 +192,8 @@ private:
const char code[] = "template <class T> void f(T val) { T a; }\n"
"f<int>(10);";
const char expected[] = "f<int> ( 10 ) ; "
const char expected[] = "void f<int> ( int val ) ; "
"f<int> ( 10 ) ; "
"void f<int> ( int val ) { }";
ASSERT_EQUALS(expected, tok(code));
@ -374,7 +383,8 @@ private:
"}\n";
// The expected result..
const char expected[] = "void f ( ) "
const char expected[] = "int * foo<3,int> ( ) ; "
"void f ( ) "
"{"
" foo<3,int> ( ) ; "
"} "
@ -392,7 +402,8 @@ private:
"}\n";
// The expected result..
const char expected[] = "void f ( ) "
const char expected[] = "char * foo<3,char> ( ) ; "
"void f ( ) "
"{"
" char * p ; p = foo<3,char> ( ) ; "
"} "
@ -484,7 +495,9 @@ private:
"}\n";
// The expected result..
const char expected[] = "void a<0> ( ) { } "
const char expected[] = "void a<2> ( ) ; "
"void a<1> ( ) ; "
"void a<0> ( ) { } "
"int main ( ) "
"{ a<2> ( ) ; return 0 ; } "
"void a<2> ( ) { a<1> ( ) ; } "
@ -522,7 +535,9 @@ private:
" return 0;\n"
"}\n";
const char expected[] = "int main ( ) { b<2> ( ) ; return 0 ; } "
const char expected[] = "void a<2> ( ) ; "
"void b<2> ( ) ; "
"int main ( ) { b<2> ( ) ; return 0 ; } "
"void b<2> ( ) { a<2> ( ) ; } "
"void a<2> ( ) { }";
@ -566,7 +581,8 @@ private:
"}\n";
// The expected result..
const char expected[] = "void f ( ) "
const char expected[] = "char & foo<char> ( ) ; "
"void f ( ) "
"{"
" char p ; p = foo<char> ( ) ; "
"} "
@ -653,7 +669,8 @@ private:
" std::cout << (foo<double>());\n"
"}";
const char expected[] = "void bar ( ) {"
const char expected[] = "void foo<double> ( ) ; "
"void bar ( ) {"
" std :: cout << ( foo<double> ( ) ) ; "
"} "
"void foo<double> ( ) { }";
@ -906,7 +923,9 @@ private:
const char code2[] = "template<class T> T f(T t) { return t; }\n"
"int x() { return f<int>(123); }";
ASSERT_EQUALS("int x ( ) { return f<int> ( 123 ) ; } int f<int> ( int t ) { return t ; }", tok(code2));
ASSERT_EQUALS("int f<int> ( int t ) ; "
"int x ( ) { return f<int> ( 123 ) ; } "
"int f<int> ( int t ) { return t ; }", tok(code2));
}
void template42() { // #4878 cpcheck aborts in ext-blocks.cpp (clang testcode)
@ -1123,7 +1142,8 @@ private:
"void foo() {\n"
" TestArithmetic<int>();\n"
"}";
const char exp[] = "void foo ( ) {"
const char exp[] = "void TestArithmetic<int> ( ) ; "
"void foo ( ) {"
" TestArithmetic<int> ( ) ; "
"} "
"void TestArithmetic<int> ( ) {"
@ -1153,6 +1173,7 @@ private:
"struct Factorial<2> ; "
"struct Factorial<1> ; "
"struct Factorial<0> { enum FacHelper { value = 1 } ; } ; "
"int diagonalGroupTest<4> ( ) ; "
"int main ( ) { return diagonalGroupTest<4> ( ) ; } "
"int diagonalGroupTest<4> ( ) { return Factorial<4> :: value ; } "
"struct Factorial<4> { enum FacHelper { value = 4 * Factorial<3> :: value } ; } ; "
@ -1170,6 +1191,7 @@ private:
"void j() { h<int>(); }";
const char exp[] = "struct S<int> ; "
"template < typename T > void f ( ) { } " // <- TODO: This template is not expanded
"void h<int> ( ) ; "
"void j ( ) { h<int> ( ) ; } "
"void h<int> ( ) { f < S<int> :: type ( 0 ) > ( ) ; } "
"struct S<int> { } ;";
@ -1327,6 +1349,7 @@ private:
"};";
const char exp [] = "class Test { "
"int test ; "
"int lookup<int> ( ) ; "
"int Fun ( ) { return lookup<int> ( ) ; } "
"} ; "
"int Test :: lookup<int> ( ) { return test ; }";
@ -1373,6 +1396,127 @@ private:
ASSERT_EQUALS(exp, tok(code));
}
void template72() {
const char code[] = "template <typename N, typename P> class Tokenizer;\n"
"const Tokenizer<Node, Path> *tokenizer() const;\n"
"template <typename N, typename P>\n"
"Tokenizer<N, P>::Tokenizer() { }";
const char exp [] = "template < typename N , typename P > class Tokenizer ; "
"const Tokenizer < Node , Path > * tokenizer ( ) const ; "
"template < typename N , typename P > "
"Tokenizer < N , P > :: Tokenizer ( ) { }";
ASSERT_EQUALS(exp, tok(code));
}
void template73() {
const char code[] = "template<typename T>\n"
"void keep_range(T& value, const T mini, const T maxi){}\n"
"template void keep_range<float>(float& v, const float l, const float u);\n"
"template void keep_range<int>(int& v, const int l, const int u);";
const char exp[] = "void keep_range<float> ( float & v , const float l , const float u ) ; "
"void keep_range<int> ( int & v , const int l , const int u ) ; "
"void keep_range<float> ( float & value , const float mini , const float maxi ) { } "
"void keep_range<int> ( int & value , const int mini , const int maxi ) { }";
ASSERT_EQUALS(exp, tok(code));
}
void template74() {
const char code[] = "template <class T> class BTlist { };\n"
"class PushBackStreamBuf {\n"
"public:\n"
" void pushBack(const BTlist<int> &vec);\n"
"};";
const char exp[] = "class BTlist<int> ; "
"class PushBackStreamBuf { "
"public: "
"void pushBack ( const BTlist<int> & vec ) ; "
"} ; "
"class BTlist<int> { } ;";
ASSERT_EQUALS(exp, tok(code));
}
void template75() {
const char code[] = "template<typename T>\n"
"T foo(T& value){ return value; }\n"
"template std::vector<std::vector<int>> foo<std::vector<std::vector<int>>>(std::vector<std::vector<int>>& v);";
const char exp[] = "std :: vector < std :: vector < int > > foo<std::vector<std::vector<int>>> ( std :: vector < std :: vector < int > > & v ) ; "
"std :: vector < std :: vector < int > > foo<std::vector<std::vector<int>>> ( std :: vector < std :: vector < int > > & value ) { return value ; }";
ASSERT_EQUALS(exp, tok(code));
}
void template76() {
const char code[] = "namespace NS {\n"
" template<typename T> T foo(T& value) { return value; }\n"
" template std::vector<std::vector<int>> foo<std::vector<std::vector<int>>>(std::vector<std::vector<int>>& v);\n"
"}\n"
"std::vector<std::vector<int>> v;\n"
"v = foo<std::vector<std::vector<int>>>(v);\n";
const char exp[] = "namespace NS { "
"std :: vector < std :: vector < int > > foo<std::vector<std::vector<int>>> ( std :: vector < std :: vector < int > > & v ) ; "
"} "
"std :: vector < std :: vector < int > > v ; "
"v = foo<std::vector<std::vector<int>>> ( v ) ; "
"std :: vector < std :: vector < int > > NS :: foo<std::vector<std::vector<int>>> ( std :: vector < std :: vector < int > > & value ) { return value ; }";
ASSERT_EQUALS(exp, tok(code));
}
void template77() {
const char code[] = "template<typename T>\n"
"struct is_void : std::false_type { };\n"
"template<>\n"
"struct is_void<void> : std::true_type { };\n"
"int main() {\n"
" std::cout << is_void<char>::value << std::endl;\n"
" std::cout << is_void<void>::value << std::endl;\n"
"}";
const char exp[] = "struct is_void<char> ; "
"struct is_void<void> : std :: true_type { } ; "
"int main ( ) { "
"std :: cout << is_void<char> :: value << std :: endl ; "
"std :: cout << is_void<void> :: value << std :: endl ; "
"} "
"struct is_void<char> : std :: false_type { } ;";
ASSERT_EQUALS(exp, tok(code));
}
void template78() {
const char code[] = "template <typename>\n"
"struct Base { };\n"
"struct S : Base <void>::Type { };";
const char exp[] = "struct Base<void> ; "
"struct S : Base<void> :: Type { } ; "
"struct Base<void> { } ;";
ASSERT_EQUALS(exp, tok(code));
}
void template79() { // #5133
const char code[] = "class Foo {\n"
"public:\n"
" template<typename T> void foo() { bar<T>(); }\n"
"private:\n"
" template<typename T> void bar() { bazz(); }\n"
" void bazz() { }\n"
"};\n"
"void some_func() {\n"
" Foo x;\n"
" x.foo<int>();\n"
"}";
const char exp[] = "class Foo { "
"public: "
"void foo<int> ( ) ; "
"private: "
"void bar<int> ( ) ; "
"void bazz ( ) { } "
"} ; "
"void some_func ( ) { "
"Foo x ; "
"x . foo<int> ( ) ; "
"} "
"void Foo :: foo<int> ( ) { bar<int> ( ) ; } "
"void Foo :: bar<int> ( ) { bazz ( ) ; }";
ASSERT_EQUALS(exp, tok(code));
}
void template_specialization_1() { // #7868 - template specialization template <typename T> struct S<C<T>> {..};
const char code[] = "template <typename T> struct C {};\n"
"template <typename T> struct S {a};\n"
@ -1761,7 +1905,9 @@ private:
" template<class T> void Fred(T value) { }\n"
"}\n"
"Fred<int>(123);";
ASSERT_EQUALS("namespace { } "
ASSERT_EQUALS("namespace { "
"void Fred<int> ( int value ) ; "
"} "
"Fred<int> ( 123 ) ; "
"void Fred<int> ( int value ) { }", tok(code));
}
@ -1985,9 +2131,9 @@ private:
" }\n"
" void SomeFunction() { }\n"
"private:\n"
" template< typename T > void TemplatedMethod();\n"
" template< typename T > T TemplatedMethod(T);\n"
"};\n"
"template< typename T > void TestClass::TemplatedMethod() { }\n"
"template< typename T > T TestClass::TemplatedMethod(T t) { return t; }\n"
"}";
ASSERT_EQUALS("namespace MyNamespace { "
"class TestClass { "
@ -1998,9 +2144,10 @@ private:
"} "
"void SomeFunction ( ) { } "
"private: "
"template < typename T > void TemplatedMethod ( ) ; "
"int TemplatedMethod<int> ( int ) ; "
"template < typename T > T TemplatedMethod ( T ) ; " // this should be removed
"} ; "
"} void MyNamespace :: TestClass :: TemplatedMethod<int> ( ) { }", tok(code));
"} int MyNamespace :: TestClass :: TemplatedMethod<int> ( int t ) { return t ; }", tok(code));
}
unsigned int templateParameters(const char code[]) {

View File

@ -5041,7 +5041,14 @@ private:
"{\n"
" fn2<int>();\n"
"}\n";
ASSERT_EQUALS("int main ( )\n{\nfn2<int> ( ) ;\n} void fn2<int> ( int t = [ ] { return 1 ; } ( ) )\n{ }", tokenizeAndStringify(code));
ASSERT_EQUALS("void fn2<int> ( int t = [ ] { return 1 ; } ( ) ) ;\n"
"\n"
"\n"
"int main ( )\n"
"{\n"
"fn2<int> ( ) ;\n"
"} void fn2<int> ( int t = [ ] { return 1 ; } ( ) )\n"
"{ }", tokenizeAndStringify(code));
}
void cpp0xtemplate2() {