addons; add CTU infrastructure

.github/workflows/CI-unixish.yml

@@ -125,6 +125,7 @@ jobs:
         run: |
           ./cppcheck --addon=threadsafety addons/test/threadsafety
           ./cppcheck --addon=threadsafety --std=c++03 addons/test/threadsafety
+          ./cppcheck --addon=misra --inline-suppr --error-exitcode=1 addons/test/misra/misra-ctu-*-test.c
 
       - name: Build GUI on ubuntu
         if: contains(matrix.os, 'ubuntu')

.github/workflows/CI-windows.yml

@@ -127,3 +127,8 @@ jobs:
           python -m pytest test-more-projects.py || exit /b !errorlevel!
           python -m pytest test-proj2.py || exit /b !errorlevel!
           python -m pytest test-suppress-syntaxError.py || exit /b !errorlevel!
+
+      - name: Test addons
+        run: |
+          .\cppcheck.exe --addon=misra --inline-suppr --error-exitcode=1 addons/test/misra/misra-ctu-*-test.c
+

addons/cppcheckdata.py

@@ -559,6 +559,21 @@ class Variable:
         self.scope = IdMap[self.scopeId]
 
 
+class TypedefInfo:
+    """
+    TypedefInfo class -- information about typedefs
+    """
+    name = None
+    filename = None
+    lineNumber = None
+    used = None
+
+    def __init__(self, element):
+        self.name = element.get('name')
+        self.filename = element.get('file')
+        self.lineNumber = int(element.get('line'))
+        self.used = (element.get('used') == '1')
+
 class Value:
     """
     Value class
@@ -707,6 +722,7 @@ class Configuration:
     scopes = []
     functions = []
     variables = []
+    typedefInfo = []
     valueflow = []
     standards = None
 
@@ -717,6 +733,7 @@ class Configuration:
         self.scopes = []
         self.functions = []
         self.variables = []
+        self.typedefInfo = []
         self.valueflow = []
         self.standards = Standards()
 
@@ -939,6 +956,9 @@ class CppcheckData:
         # Iterating <varlist> in a <scope>.
         iter_scope_varlist = False
 
+        # Iterating <typedef-info>
+        iter_typedef_info = False
+
         # Use iterable objects to traverse XML tree for dump files incrementally.
         # Iterative approach is required to avoid large memory consumption.
         # Calling .clear() is necessary to let the element be garbage collected.
@@ -1013,6 +1033,12 @@ class CppcheckData:
                     else:
                         cfg_arguments.append(var)
 
+            # Parse typedef info
+            elif node.tag == 'typedef-info':
+                iter_typedef_info = (event == 'start')
+            elif iter_typedef_info and node.tag == 'info' and event == 'start':
+                cfg.typedefInfo.append(TypedefInfo(node))
+
             # Parse valueflows (list of values)
             elif node.tag == 'valueflow' and event == 'start':
                 continue
@@ -1163,3 +1189,10 @@ def reportError(location, severity, message, addon, errorId, extra=''):
         sys.stderr.write('%s (%s) %s [%s-%s]\n' % (loc, severity, message, addon, errorId))
         global EXIT_CODE
         EXIT_CODE = 1
+
+def reportSummary(dumpfile, summary_type, summary_data):
+    # dumpfile ends with ".dump"
+    ctu_info_file = dumpfile[:-4] + "ctu-info"
+    with open(ctu_info_file, 'at') as f:
+        msg = {'summary': summary_type, 'data': summary_data}
+        f.write(json.dumps(msg) + '\n')

addons/misra.py

@@ -17,6 +17,7 @@ from __future__ import print_function
 
 import cppcheckdata
 import itertools
+import json
 import sys
 import re
 import os
@@ -24,6 +25,8 @@ import argparse
 import codecs
 import string
 
+from collections import namedtuple
+
 try:
     from itertools import izip as zip
 except ImportError:
@@ -1336,6 +1339,15 @@ class MisraChecker:
                 self.reportError(scope.bodyStart, 5, 5)
 
 
+    def misra_5_6(self, data):
+        dumpfile = data[0]
+        typedefInfo = data[1]
+        summary = []
+        for ti in typedefInfo:
+            summary.append({ 'name': ti.name, 'file': ti.filename, 'line': ti.lineNumber })
+        if len(summary) > 0:
+            cppcheckdata.reportSummary(dumpfile, 'misra_5_6', summary)
+
     def misra_6_1(self, data):
         # Bitfield type must be bool or explicitly signed/unsigned int
         for token in data.tokenlist:
@@ -3250,6 +3262,7 @@ class MisraChecker:
             self.executeCheck(502, self.misra_5_2, cfg)
             self.executeCheck(504, self.misra_5_4, cfg)
             self.executeCheck(505, self.misra_5_5, cfg)
+            self.executeCheck(506, self.misra_5_6, (dumpfile, cfg.typedefInfo))
             self.executeCheck(601, self.misra_6_1, cfg)
             self.executeCheck(602, self.misra_6_2, cfg)
             if cfgNumber == 0:
@@ -3342,6 +3355,32 @@ class MisraChecker:
             self.executeCheck(2112, self.misra_21_12, cfg)
             # 22.4 is already covered by Cppcheck writeReadOnlyFile
 
+    def analyse_ctu_info(self, files):
+        data_misra_5_6 = []
+
+        Location = namedtuple('Location', 'file linenr column')
+
+        for filename in files:
+            if not filename.endswith('.ctu-info'):
+                continue
+            for line in open(filename, 'rt'):
+                if line.startswith('{'):
+                    s = json.loads(line)
+                    summary_type = s['summary']
+                    summary_data = s['data']
+
+                    # TODO break out info some function
+                    if summary_type == 'misra_5_6':
+                        for info1 in summary_data:
+                            found = False
+                            for info2 in data_misra_5_6:
+                                if info1['name'] == info2['name']:
+                                    found = True
+                                    if info1['file'] != info2['file'] or info1['line'] != info2['line']:
+                                        self.reportError(Location(info2['file'], info2['line'], 0), 5, 6)
+                                        self.reportError(Location(info1['file'], info1['line'], 0), 5, 6)
+                            if not found:
+                                data_misra_5_6.append(info1)
 
 RULE_TEXTS_HELP = '''Path to text file of MISRA rules
 
@@ -3434,6 +3473,9 @@ def main():
         checker.setSeverity(args.severity)
 
     for item in args.dumpfile:
+        if item.endswith('.ctu-info'):
+            continue
+
         checker.parseDump(item)
 
         if settings.verify:
@@ -3457,6 +3499,8 @@ def main():
             if exitCode != 0:
                 sys.exit(exitCode)
 
+    checker.analyse_ctu_info(args.dumpfile)
+
     if settings.verify:
         sys.exit(exitCode)
 

addons/test/misra/misra-ctu-1-test.c

@@ -0,0 +1,8 @@
+// Test with command:
+// ./cppcheck --addon=misra --inline-suppr addons/test/misra/misra-ctu-*-test.c
+
+#include "misra-ctu-test.h"
+
+// cppcheck-suppress misra-c2012-5.6
+typedef int MISRA_5_6_VIOLATION;
+

addons/test/misra/misra-ctu-2-test.c

@@ -0,0 +1,8 @@
+// Test with command:
+// ./cppcheck --addon=misra --inline-suppr addons/test/misra/misra-ctu-*-test.c
+
+#include "misra-ctu-test.h"
+
+// cppcheck-suppress misra-c2012-5.6
+typedef int MISRA_5_6_VIOLATION;
+

addons/test/misra/misra-ctu-test.h

@@ -0,0 +1,5 @@
+
+typedef int t1;
+typedef int t2;
+
+

lib/cppcheck.cpp

@@ -67,6 +67,7 @@ namespace {
         std::string scriptFile;
         std::string args;
         std::string python;
+        bool ctu = false;
 
         static std::string getFullPath(const std::string &fileName, const std::string &exename) {
             if (Path::fileExists(fileName))
@@ -102,6 +103,15 @@ namespace {
                     args += " " + v.get<std::string>();
             }
 
+            if (obj.count("ctu")) {
+                // ctu is specified in the config file
+                if (!obj["ctu"].is<bool>())
+                    return "Loading " + fileName + " failed. ctu must be array.";
+                ctu = obj["ctu"].get<bool>();
+            } else {
+                ctu = false;
+            }
+
             if (obj.count("python")) {
                 // Python was defined in the config file
                 if (obj["python"].is<picojson::array>()) {
@@ -187,6 +197,20 @@ static std::vector<std::string> split(const std::string &str, const std::string
     return ret;
 }
 
+static std::string getDumpFileName(const Settings& settings, const std::string& filename)
+{
+    if (!settings.dumpFile.empty())
+        return settings.dumpFile;
+    if (!settings.dump && !settings.buildDir.empty())
+        return AnalyzerInformation::getAnalyzerInfoFile(settings.buildDir, filename, "") + ".dump";
+    return filename + ".dump";
+}
+
+static std::string getCtuInfoFileName(const std::string &dumpFile)
+{
+    return dumpFile.substr(0, dumpFile.size()-4) + "ctu-info";
+}
+
 static void createDumpFile(const Settings& settings,
                            const std::string& filename,
                            const std::vector<std::string>& files,
@@ -196,16 +220,16 @@ static void createDumpFile(const Settings& settings,
 {
     if (!settings.dump && settings.addons.empty())
         return;
-    if (!settings.dumpFile.empty())
+    dumpFile = getDumpFileName(settings, filename);
-        dumpFile = settings.dumpFile;
-    else if (!settings.dump && !settings.buildDir.empty())
-        dumpFile = AnalyzerInformation::getAnalyzerInfoFile(settings.buildDir, filename, "") + ".dump";
-    else
-        dumpFile = filename + ".dump";
 
     fdump.open(dumpFile);
     if (!fdump.is_open())
         return;
+
+    {
+        std::ofstream fout(getCtuInfoFileName(dumpFile));
+    }
+
     fdump << "<?xml version=\"1.0\"?>" << std::endl;
     fdump << "<dumps>" << std::endl;
     fdump << "  <platform"
@@ -235,7 +259,7 @@ static void createDumpFile(const Settings& settings,
 
 static std::string executeAddon(const AddonInfo &addonInfo,
                                 const std::string &defaultPythonExe,
-                                const std::string &dumpFile,
+                                const std::vector<std::string> &files,
                                 std::function<bool(std::string,std::vector<std::string>,std::string,std::string*)> executeCommand)
 {
     const std::string redirect = "2>&1";
@@ -263,7 +287,9 @@ static std::string executeAddon(const AddonInfo &addonInfo,
             throw InternalError(nullptr, "Failed to auto detect python");
     }
 
-    const std::string args = cmdFileName(addonInfo.scriptFile) + " --cli" + addonInfo.args + " " + cmdFileName(dumpFile);
+    std::string args = cmdFileName(addonInfo.scriptFile) + " --cli" + addonInfo.args;
+    for (const std::string& filename: files)
+        args += " " + cmdFileName(filename);
     std::string result;
     if (!executeCommand(pythonExe, split(args), redirect, &result))
         throw InternalError(nullptr, "Failed to execute addon (command: '" + pythonExe + " " + args + "')");
@@ -1264,8 +1290,19 @@ void CppCheck::executeRules(const std::string &tokenlist, const Tokenizer &token
 
 void CppCheck::executeAddons(const std::string& dumpFile)
 {
+    if (!dumpFile.empty()) {
+        std::vector<std::string> f{dumpFile};
+        executeAddons(f);
+        if (!mSettings.dump && mSettings.buildDir.empty())
+            std::remove(dumpFile.c_str());
+    }
+}
+
+void CppCheck::executeAddons(const std::vector<std::string>& files)
+{
+    if (mSettings.addons.empty() || files.empty())
+        return;
 
-    if (!mSettings.addons.empty() && !dumpFile.empty()) {
     for (const std::string &addon : mSettings.addons) {
         struct AddonInfo addonInfo;
         const std::string &failedToGetAddonInfo = addonInfo.getAddonInfo(addon, mSettings.exename);
@@ -1274,8 +1311,11 @@ void CppCheck::executeAddons(const std::string& dumpFile)
             mExitCode = 1;
             continue;
         }
+        if (addon != "misra" && !addonInfo.ctu && endsWith(files.back(), ".ctu-info", 9))
+            continue;
+
         const std::string results =
-                executeAddon(addonInfo, mSettings.addonPython, dumpFile, mExecuteCommand);
+            executeAddon(addonInfo, mSettings.addonPython, files, mExecuteCommand);
         std::istringstream istr(results);
         std::string line;
 
@@ -1311,9 +1351,20 @@ void CppCheck::executeAddons(const std::string& dumpFile)
             reportErr(errmsg);
         }
     }
-        std::remove(dumpFile.c_str());
 }
 
+void CppCheck::executeAddonsWholeProgram(const std::map<std::string, std::size_t> &files)
+{
+    if (mSettings.addons.empty())
+        return;
+
+    std::vector<std::string> ctuInfoFiles;
+    for (const auto f: files) {
+        const std::string &dumpFileName = getDumpFileName(mSettings, f.first);
+        ctuInfoFiles.push_back(getCtuInfoFileName(dumpFileName));
+    }
+
+    executeAddons(ctuInfoFiles);
 }
 
 Settings &CppCheck::settings()
@@ -1578,7 +1629,7 @@ bool CppCheck::analyseWholeProgram()
 
 void CppCheck::analyseWholeProgram(const std::string &buildDir, const std::map<std::string, std::size_t> &files)
 {
-    (void)files;
+    executeAddonsWholeProgram(files);
     if (buildDir.empty())
         return;
     if (mSettings.checks.isEnabled(Checks::unusedFunction))

lib/cppcheck.h

@@ -172,8 +172,14 @@ private:
     /**
      * Execute addons
      */
+    void executeAddons(const std::vector<std::string>& files);
     void executeAddons(const std::string &dumpFile);
 
+    /**
+     * Execute addons
+     */
+    void executeAddonsWholeProgram(const std::map<std::string, std::size_t> &files);
+
     /**
      * @brief Execute rules, if any
      * @param tokenlist token list to use (normal / simple)

lib/tokenize.cpp

@@ -1044,6 +1044,13 @@ void Tokenizer::simplifyTypedef()
         bool done = false;
         bool ok = true;
 
+        TypedefInfo typedefInfo;
+        typedefInfo.name = typeName->str();
+        typedefInfo.filename = list.file(typeName);
+        typedefInfo.lineNumber = typeName->linenr();
+        typedefInfo.used = false;
+        mTypedefInfo.push_back(typedefInfo);
+
         while (!done) {
             std::string pattern = typeName->str();
             int scope = 0;
@@ -1254,6 +1261,8 @@ void Tokenizer::simplifyTypedef()
                 }
 
                 if (simplifyType) {
+                    mTypedefInfo.back().used = true;
+
                     // can't simplify 'operator functionPtr ()' and 'functionPtr operator ... ()'
                     if (functionPtr && (tok2->previous()->str() == "operator" ||
                                         (tok2->next() && tok2->next()->str() == "operator"))) {
@@ -5480,6 +5489,19 @@ void Tokenizer::dump(std::ostream &out) const
     mSymbolDatabase->printXml(out);
     if (list.front())
         list.front()->printValueFlow(true, out);
+
+    if (!mTypedefInfo.empty()) {
+        out << "  <typedef-info>" << std::endl;
+        for (const TypedefInfo &typedefInfo: mTypedefInfo) {
+            out << "    <info"
+                << " name=\"" << typedefInfo.name << "\""
+                << " file=\"" << typedefInfo.filename << "\""
+                << " line=\"" << typedefInfo.lineNumber << "\""
+                << " used=\"" << (typedefInfo.used?1:0) << "\""
+                << "/>" << std::endl;
+        }
+        out << "  </typedef-info>" << std::endl;
+    }
 }
 
 void Tokenizer::simplifyHeadersAndUnusedTemplates()

lib/tokenize.h

@@ -987,6 +987,14 @@ private:
     /** sizeof information for known types */
     std::map<std::string, int> mTypeSize;
 
+    struct TypedefInfo {
+        std::string name;
+        std::string filename;
+        int lineNumber;
+        bool used;
+    };
+    std::vector<TypedefInfo> mTypedefInfo;
+
     /** variable count */
     nonneg int mVarId;