added clean C/C++ keyword lists and use them in `TokenList` (#3774)

This commit is contained in:
Oliver Stöneberg 2023-02-24 21:43:27 +01:00 committed by GitHub
parent da09a92b0a
commit 674231ae52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 293 additions and 91 deletions

View File

@ -227,6 +227,7 @@ LIBOBJ = $(libcppdir)/analyzerinfo.o \
$(libcppdir)/fwdanalysis.o \
$(libcppdir)/importproject.o \
$(libcppdir)/infer.o \
$(libcppdir)/keywords.o \
$(libcppdir)/library.o \
$(libcppdir)/mathlib.o \
$(libcppdir)/path.o \
@ -564,6 +565,9 @@ $(libcppdir)/importproject.o: lib/importproject.cpp externals/picojson/picojson.
$(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes.h lib/infer.h lib/mathlib.h lib/valueptr.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/infer.cpp
$(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp
$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp
@ -615,7 +619,7 @@ $(libcppdir)/token.o: lib/token.cpp lib/astutils.h lib/config.h lib/errortypes.h
$(libcppdir)/tokenize.o: lib/tokenize.cpp externals/simplecpp/simplecpp.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/tokenize.cpp
$(libcppdir)/tokenlist.o: lib/tokenlist.cpp externals/simplecpp/simplecpp.h lib/astutils.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(libcppdir)/tokenlist.o: lib/tokenlist.cpp externals/simplecpp/simplecpp.h lib/astutils.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/keywords.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/tokenlist.cpp
$(libcppdir)/utils.o: lib/utils.cpp lib/config.h lib/utils.h

View File

@ -83,6 +83,7 @@
<ClCompile Include="fwdanalysis.cpp" />
<ClCompile Include="importproject.cpp" />
<ClCompile Include="infer.cpp" />
<ClCompile Include="keywords.cpp" />
<ClCompile Include="library.cpp" />
<ClCompile Include="mathlib.cpp" />
<ClCompile Include="path.cpp" />
@ -149,6 +150,7 @@
<ClInclude Include="fwdanalysis.h" />
<ClInclude Include="importproject.h" />
<ClInclude Include="infer.h" />
<ClInclude Include="keywords.h" />
<ClInclude Include="library.h" />
<ClInclude Include="mathlib.h" />
<ClInclude Include="path.h" />

220
lib/keywords.cpp Normal file
View File

@ -0,0 +1,220 @@
/*
* Cppcheck - A tool for static C/C++ code analysis
* Copyright (C) 2007-2023 Cppcheck team.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "keywords.h"
// see https://en.cppreference.com/w/c/keyword
#define C90_KEYWORDS \
"auto", "break", "case", "char", "const", "continue", "default", \
"do", "double", "else", "enum", "extern", "float", "for", "goto", "if", "int", "long", \
"register", "return", "short", "signed", "sizeof", "static", "struct", "switch", "typedef", \
"union", "unsigned", "void", "volatile", "while"
#define C99_KEYWORDS \
"inline", "restrict", "_Bool", "_Complex", "_Imaginary"
#define C11_KEYWORDS \
"_Alignas", "_Alignof", "_Atomic", "_Generic", "_Noreturn", "_Static_assert", "_Thread_local"
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-macros"
#endif
#define C23_KEYWORDS \
"alignas", "alignof", "bool", "false", "nullptr", "static_assert", "thread_local", "true", "typeof", "typeof_unqual", \
"_BitInt", "_Decimal128", "_Decimal32", "_Decimal64"
#ifdef __clang__
#pragma clang diagnostic pop
#endif
static const std::unordered_set<std::string> c89_keywords_all = {
C90_KEYWORDS
};
static const std::unordered_set<std::string> c89_keywords = c89_keywords_all;
static const std::unordered_set<std::string> c99_keywords_all = {
C90_KEYWORDS, C99_KEYWORDS
};
static const std::unordered_set<std::string> c99_keywords = {
C99_KEYWORDS
};
static const std::unordered_set<std::string> c11_keywords_all = {
C90_KEYWORDS, C99_KEYWORDS, C11_KEYWORDS
};
static const std::unordered_set<std::string> c11_keywords = {
C11_KEYWORDS
};
/*
static const std::unordered_set<std::string> c23_keywords_all = {
C90_KEYWORDS, C99_KEYWORDS, C11_KEYWORDS, C23_KEYWORDS
};
static const std::unordered_set<std::string> c23_keywords = {
C23_KEYWORDS
};
*/
// see https://en.cppreference.com/w/cpp/keyword
#define CPP03_KEYWORDS \
"and", "and_eq", "asm", "auto", "bitand", "bitor", "bool", "break", "case", "catch", "char", \
"class", "compl", "const", "const_cast", "continue", "default", \
"delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", "export", "extern", "false", \
"float", "for", "friend", "goto", "if", "inline", "int", "long", \
"mutable", "namespace", "new", "not", "not_eq", "operator", \
"or", "or_eq", "private", "protected", "public", "register", "reinterpret_cast", \
"static_cast", "struct", "switch", "template", "this", "throw", \
"true", "try", "typedef", "typeid", "typename", "union", "unsigned", "using", \
"virtual", "void", "volatile", "wchar_t", "while", "xor", "xor_eq"
#define CPP11_KEYWORDS \
"alignas", "alignof", "char16_t", "char32_t", "constexpr", "decltype", \
"noexcept", "nullptr", "static_assert", "thread_local"
#define CPP20_KEYWORDS \
"concept", "consteval", "constinit", "co_await", \
"co_return", "co_yield", "requires"
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-macros"
#endif
#define CPP_TMTS_KEYWORDS \
"atomic_cancel", "atomic_commit", "atomic_noexcept", "synchronized"
#define CPP_REFL_TS_KEYWORDS \
"reflexpr"
#ifdef __clang__
#pragma clang diagnostic pop
#endif
static const std::unordered_set<std::string> cpp03_keywords_all = {
CPP03_KEYWORDS
};
static const std::unordered_set<std::string> cpp03_keywords = cpp03_keywords_all;
static const std::unordered_set<std::string> cpp11_keywords_all = {
CPP03_KEYWORDS, CPP11_KEYWORDS
};
static const std::unordered_set<std::string> cpp11_keywords = {
CPP11_KEYWORDS
};
static const std::unordered_set<std::string> cpp14_keywords_all = cpp11_keywords_all;
static const std::unordered_set<std::string> cpp14_keywords;
static const std::unordered_set<std::string> cpp17_keywords_all = cpp11_keywords_all;
static const std::unordered_set<std::string> cpp17_keywords;
static const std::unordered_set<std::string> cpp20_keywords_all = {
CPP03_KEYWORDS, CPP11_KEYWORDS, CPP20_KEYWORDS
};
static const std::unordered_set<std::string> cpp20_keywords = {
CPP20_KEYWORDS
};
static const std::unordered_set<std::string> cpp23_keywords;
static const std::unordered_set<std::string> cpp23_keywords_all = cpp20_keywords_all;
// cppcheck-suppress unusedFunction
const std::unordered_set<std::string>& Keywords::getAll(Standards::cstd_t cStd)
{
// cppcheck-suppress missingReturn
switch (cStd) {
case Standards::cstd_t::C89:
return c89_keywords_all;
case Standards::cstd_t::C99:
return c99_keywords_all;
case Standards::cstd_t::C11:
return c11_keywords_all;
/*case Standards::cstd_t::C23:
return c23_keywords_all;*/
}
}
// cppcheck-suppress unusedFunction
const std::unordered_set<std::string>& Keywords::getAll(Standards::cppstd_t cppStd) {
// cppcheck-suppress missingReturn
switch (cppStd) {
case Standards::cppstd_t::CPP03:
return cpp03_keywords_all;
case Standards::cppstd_t::CPP11:
return cpp11_keywords_all;
case Standards::cppstd_t::CPP14:
return cpp14_keywords_all;
case Standards::cppstd_t::CPP17:
return cpp17_keywords_all;
case Standards::cppstd_t::CPP20:
return cpp20_keywords_all;
case Standards::cppstd_t::CPP23:
return cpp23_keywords_all;
}
}
// cppcheck-suppress unusedFunction
const std::unordered_set<std::string>& Keywords::getOnly(Standards::cstd_t cStd)
{
// cppcheck-suppress missingReturn
switch (cStd) {
case Standards::cstd_t::C89:
return c89_keywords;
case Standards::cstd_t::C99:
return c99_keywords;
case Standards::cstd_t::C11:
return c11_keywords;
/*case Standards::cstd_t::C23:
return c23_keywords_all;*/
}
}
// cppcheck-suppress unusedFunction
const std::unordered_set<std::string>& Keywords::getOnly(Standards::cppstd_t cppStd)
{
// cppcheck-suppress missingReturn
switch (cppStd) {
case Standards::cppstd_t::CPP03:
return cpp03_keywords;
case Standards::cppstd_t::CPP11:
return cpp11_keywords;
case Standards::cppstd_t::CPP14:
return cpp14_keywords;
case Standards::cppstd_t::CPP17:
return cpp17_keywords;
case Standards::cppstd_t::CPP20:
return cpp20_keywords;
case Standards::cppstd_t::CPP23:
return cpp23_keywords;
}
}

37
lib/keywords.h Normal file
View File

@ -0,0 +1,37 @@
/*
* Cppcheck - A tool for static C/C++ code analysis
* Copyright (C) 2007-2023 Cppcheck team.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef keywordsH
#define keywordsH
#include "standards.h"
#include <string>
#include <unordered_set>
class Keywords
{
public:
static const std::unordered_set<std::string>& getAll(Standards::cstd_t cStd);
static const std::unordered_set<std::string>& getAll(Standards::cppstd_t cppStd);
static const std::unordered_set<std::string>& getOnly(Standards::cstd_t cStd);
static const std::unordered_set<std::string>& getOnly(Standards::cppstd_t cppStd);
};
#endif

View File

@ -44,6 +44,7 @@ HEADERS += $${PWD}/analyzer.h \
$${PWD}/fwdanalysis.h \
$${PWD}/importproject.h \
$${PWD}/infer.h \
$${PWD}/keywords.h \
$${PWD}/library.h \
$${PWD}/mathlib.h \
$${PWD}/path.h \
@ -110,6 +111,7 @@ SOURCES += $${PWD}/analyzerinfo.cpp \
$${PWD}/fwdanalysis.cpp \
$${PWD}/importproject.cpp \
$${PWD}/infer.cpp \
$${PWD}/keywords.cpp \
$${PWD}/library.cpp \
$${PWD}/mathlib.cpp \
$${PWD}/path.cpp \

View File

@ -22,6 +22,7 @@
#include "astutils.h"
#include "errorlogger.h"
#include "errortypes.h"
#include "keywords.h"
#include "library.h"
#include "mathlib.h"
#include "path.h"
@ -52,41 +53,6 @@ TokenList::TokenList(const Settings* settings) :
mIsCpp(false)
{
mTokensFrontBack.list = this;
mKeywords.insert("asm");
mKeywords.insert("auto");
mKeywords.insert("break");
mKeywords.insert("case");
//mKeywords.insert("char"); // type
mKeywords.insert("const");
mKeywords.insert("continue");
mKeywords.insert("default");
mKeywords.insert("do");
//mKeywords.insert("double"); // type
mKeywords.insert("else");
mKeywords.insert("enum");
mKeywords.insert("extern");
//mKeywords.insert("float"); // type
mKeywords.insert("for");
mKeywords.insert("goto");
mKeywords.insert("if");
mKeywords.insert("inline");
//mKeywords.insert("int"); // type
//mKeywords.insert("long"); // type
mKeywords.insert("register");
mKeywords.insert("restrict");
mKeywords.insert("return");
//mKeywords.insert("short"); // type
mKeywords.insert("signed");
mKeywords.insert("sizeof");
mKeywords.insert("static");
mKeywords.insert("struct");
mKeywords.insert("switch");
mKeywords.insert("typedef");
mKeywords.insert("union");
mKeywords.insert("unsigned");
mKeywords.insert("void");
mKeywords.insert("volatile");
mKeywords.insert("while");
}
TokenList::~TokenList()
@ -124,57 +90,6 @@ void TokenList::determineCppC()
mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath()));
mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath()));
}
if (mIsCpp) {
//mKeywords.insert("bool"); // type
mKeywords.insert("catch");
mKeywords.insert("class");
mKeywords.insert("constexpr");
mKeywords.insert("const_cast");
mKeywords.insert("decltype");
mKeywords.insert("delete");
mKeywords.insert("dynamic_cast");
mKeywords.insert("explicit");
mKeywords.insert("export");
//mKeywords.insert("false"); // literal
mKeywords.insert("friend");
mKeywords.insert("mutable");
mKeywords.insert("namespace");
mKeywords.insert("new");
mKeywords.insert("noexcept");
mKeywords.insert("operator");
mKeywords.insert("private");
mKeywords.insert("protected");
mKeywords.insert("public");
mKeywords.insert("reinterpret_cast");
mKeywords.insert("static_assert");
mKeywords.insert("static_cast");
mKeywords.insert("template");
mKeywords.insert("this");
mKeywords.insert("thread_local");
mKeywords.insert("throw");
//mKeywords.insert("true"); // literal
mKeywords.insert("try");
mKeywords.insert("typeid");
mKeywords.insert("typename");
mKeywords.insert("typeof");
mKeywords.insert("using");
mKeywords.insert("virtual");
//mKeywords.insert("wchar_t"); // type
if (!mSettings || mSettings->standards.cpp >= Standards::CPP20) {
mKeywords.insert("alignas");
mKeywords.insert("alignof");
mKeywords.insert("axiom");
mKeywords.insert("co_await");
mKeywords.insert("co_return");
mKeywords.insert("co_yield");
mKeywords.insert("concept");
mKeywords.insert("synchronized");
mKeywords.insert("consteval");
mKeywords.insert("reflexpr");
mKeywords.insert("requires");
}
}
}
int TokenList::appendFileIfNew(std::string fileName)
@ -2050,5 +1965,30 @@ void TokenList::simplifyStdType()
bool TokenList::isKeyword(const std::string &str) const
{
return mKeywords.find(str) != mKeywords.end();
if (mIsCpp) {
// TODO: integrate into keywords?
// types and literals are not handled as keywords
static const std::unordered_set<std::string> cpp_types = {"bool", "false", "true"};
if (cpp_types.find(str) != cpp_types.end())
return false;
// TODO: properly apply configured standard
if (!mSettings || mSettings->standards.cpp >= Standards::CPP20) {
static const auto& cpp20_keywords = Keywords::getAll(Standards::cppstd_t::CPP20);
return cpp20_keywords.find(str) != cpp20_keywords.end();
}
static const auto& cpp_keywords = Keywords::getAll(Standards::cppstd_t::CPP11);
return cpp_keywords.find(str) != cpp_keywords.end();
}
// TODO: integrate into Keywords?
// types are not handled as keywords
static const std::unordered_set<std::string> c_types = {"char", "double", "float", "int", "long", "short"};
if (c_types.find(str) != c_types.end())
return false;
// TODO: use configured standard
static const auto& c_keywords = Keywords::getAll(Standards::cstd_t::C99);
return c_keywords.find(str) != c_keywords.end();
}

View File

@ -27,7 +27,6 @@
#include <cstddef>
#include <iosfwd>
#include <string>
#include <unordered_set>
#include <vector>
class Settings;
@ -212,8 +211,6 @@ private:
/** settings */
const Settings* mSettings;
std::unordered_set<std::string> mKeywords;
/** File is known to be C/C++ code */
bool mIsC;
bool mIsCpp;