Merge pull request #708 from siemens/preprocessor-directives-for-addons-v2

Add preprocessor directives dump and Y2038 addon

Makefile

@@ -287,6 +287,8 @@ tags:
 install: cppcheck
 	install -d ${BIN}
 	install cppcheck ${BIN}
+	install addons/*.py ${BIN}
+	install addons/*/*.py ${BIN}
 	install htmlreport/cppcheck-htmlreport ${BIN}
 ifdef CFGDIR 
 	install -d ${DESTDIR}${CFGDIR}

addons/cppcheckdata.py

@@ -6,6 +6,35 @@
 #
 
 import xml.etree.ElementTree as ET
+import argparse
+
+## Directive class. Contains information about each preprocessor directive in the source code.
+#
+# file and linenr denote the location where the directive is defined.
+#
+# To iterate through all directives use such code:
+# @code
+# data = cppcheckdata.parsedump(...)
+# for cfg in data.configurations:
+#   for directive in cfg.directives:
+#     print(directive.str)
+# @endcode
+#
+
+
+class Directive:
+    ## The directive line, with all C or C++ comments removed
+    str = None
+    ## name of (possibly included) file where directive is defined
+    file = None
+    ## line number in (possibly included) file where directive is defined
+    linenr = None
+
+    def __init__(self, element):
+        self.str = element.get('str')
+        self.file = element.get('file')
+        self.linenr = element.get('linenr')
+
 
 ## Token class. Contains information about each token in the source code.
 #
@@ -16,8 +45,9 @@ import xml.etree.ElementTree as ET
 # To iterate through all tokens use such code:
 # @code
 # data = cppcheckdata.parsedump(...)
+# for cfg in data.configurations:
 #   code = ''
-# for token in data.tokenlist:
+#   for token in cfg.tokenlist:
 #     code = code + token.str + ' '
 #   print(code)
 # @endcode
@@ -380,13 +410,15 @@ class ValueFlow:
             self.values.append(ValueFlow.Value(value))
 
 ## Configuration class
-# This class contains the tokens, scopes, functions, variables and
+# This class contains the directives, tokens, scopes, functions,
-# value flows for one configuration.
+# variables and value flows for one configuration.
 
 
 class Configuration:
     ## Name of the configuration, "" for default
     name = ''
+    ## List of Directive items
+    directives = []
     ## List of Token items
     tokenlist = []
     ## List of Scope items
@@ -400,6 +432,7 @@ class Configuration:
 
     def __init__(self, confignode):
         self.name = confignode.get('cfg')
+        self.directives = []
         self.tokenlist = []
         self.scopes = []
         self.functions = []
@@ -407,6 +440,10 @@ class Configuration:
         self.valueflow = []
 
         for element in confignode:
+            if element.tag == 'directivelist':
+                for directive in element:
+                    self.directives.append(Directive(directive))
+
             if element.tag == 'tokenlist':
                 for token in element:
                     self.tokenlist.append(Token(token))
@@ -534,3 +571,57 @@ def astIsFloat(token):
     if typeToken.str == 'float' or typeToken.str == 'double':
         return True
     return False
+
+# Create a cppcheck parser
+
+class CppCheckFormatter(argparse.HelpFormatter):
+    '''
+        Properly formats multiline argument helps
+    '''
+    def _split_lines(self, text, width):
+        # this is the RawTextHelpFormatter._split_lines
+        if text.startswith('R|'):
+            return text[2:].splitlines()  
+        return argparse.HelpFormatter._split_lines(self, text, width)
+
+def ArgumentParser():
+    '''
+        Returns an argparse argument parser with an already-added
+        argument definition for -t/--template
+    '''
+    parser = argparse.ArgumentParser(formatter_class=CppCheckFormatter)
+    parser.add_argument('-t', '--template', metavar='<text>',
+        default='{callstack}: ({severity}) {message}',
+        help="R|Format the error messages. E.g.\n" \
+             "'{file}:{line},{severity},{id},{message}' or\n" \
+             "'{file}({line}):({severity}) {message}' or\n" \
+             "'{callstack} {message}'\n" \
+             "Pre-defined templates: gcc, vs, edit")
+    return parser
+
+# Format an error message.
+ 
+def reportError(template, callstack=[], severity='', message='', id=''):
+    '''
+        Format an error message according to the template.
+        
+        :param template: format string, or 'gcc', 'vs' or 'edit'.
+        :param callstack: e.g. [['file1.cpp',10],['file2.h','20'], ... ]
+        :param severity: e.g. 'error', 'warning' ...
+        :param id: message ID.
+        :param message: message text.
+    '''
+    # expand predefined templates
+    if template == 'gcc':
+        template = '{file}:{line}: {severity}: {message}'
+    elif template == 'vs':
+        template = '{file}({line}): {severity}: {message}'
+    elif template == 'edit':
+        template = '{file} +{line}: {severity}: {message}'
+    # compute 'callstack}, {file} and {line} replacements
+    stack = ' -> '.join(['['+f+':'+str(l)+']' for (f,l) in callstack])
+    file = callstack[-1][0]
+    line = str(callstack[-1][1])
+    # format message
+    return template.format(callstack=stack, file=file, line=line,
+        severity=severity, message=message, id=id)

addons/y2038/README

@@ -0,0 +1,151 @@
+README of the Y2038 cppcheck addon
+==================================
+
+Contents
+
+1. What is Y2038?
+2. What is the Y2038 ccpcheck addon?
+3. How does the Y2038 cppcheck addon work?
+4. How to use the Y2038 cppcheck addon
+
+---
+
+1. What is Y2038?
+
+In a few words:
+
+In Linux, the current date and time is kept as the number of seconds elapsed
+since the Unich epoch, that is, since January 1st, 1970 at 00:00:00 GMT.
+
+Most of the time, this representation is stored as a 32-bit signed quantity.
+
+On January 19th, 2038 at 03:14:07 GMT, such 32-bit representations will reach
+their maximum positive value.
+
+What happens then is unpredictable: system time might roll back to December
+13th, 1901 at 19:55:13, or it might keep running on until February 7th, 2106
+at 06:28:15 GMT, or the computer may freeze, or just about anything you can
+think of, plus a few ones you can't.
+
+The workaround for this is to switch to a 64-bit signed representation of time
+as seconds from the Unix epoch. This representation will work for more than 250
+billion years.
+
+Working around Y2038 requires fixing the Linux kernel, the C libraries, and
+any user code around which uses 32-bit epoch representations.
+
+There is Y2038-proofing work in progress on the Linux and GNU glibc front.
+
+2. What is the Y2038 ccpcheck addon?
+
+The Y2038 cppcheck addon is a tool to help detect code which might need fixing
+because it is Y2038-sensitive. This may be because it uses types or functions
+from libc or from the Linux kernel which are known not to be Y2038-proof.
+
+3. How does the Y2038 cppcheck addon work?
+
+The Y2038 cppcheck addon takes XML dumps produced by cppcheck from source code
+files and looks for the names of types or functions which are known to be Y2038-
+sensitive, and emits diagnostics whenever it finds one.
+
+Of course, this is of little use if your code uses a Y2038-proof glibc and
+correctly configured Y2038-proof time support.
+
+This is why y2038.py takes into account two preprocessor directives:
+_TIME_BITS and __USE_TIME_BITS64.
+
+_TIME_BITS is defined equal to 64 by user code when it wants 64-bit time
+support from the GNU glibc. Code which does not define _TIME_BITS equal to 64
+(or defines it to something else than 64) runs a risk of not being Y2038-proof.
+
+__USE_TIME_BITS64 is defined by the GNU glibc when it actualy provides 64-bit
+time support. When this is defined, then all glibc symbols, barring bugs, are
+Y2038-proof (but your code might have its own Y2038 bugs, if it handles signed
+32-bit Unix epoch values).
+
+The Y2038 cppcheck performs the following checks:
+
+  1. Upon meeting a definition for _TIME_BITS, if that definition does not
+     set it equal to 64, this error diagnostic is emitted:
+
+	Error: _TIME_BITS must be defined equal to 64
+
+     This case is very unlikely but might result from a typo, so pointing
+     it out is quite useful. Note that definitions of _TIME_BITS as an
+     expression evaluating to 64 will be flagged too.
+
+  2. Upon meeting a definition for _USE_TIME_BITS64, if _TIME_BITS is not
+     defined equal to 64, this information diagnostic is emitted:
+
+	Warning: _USE_TIME_BITS64 is defined but _TIME_BITS was not
+
+     This reflects the fact that even though the glibc checked default to
+     64-bit time support, this was not requested by the user code, and
+     therefore the user code might fail Y2038 if built against a glibc
+     which defaults to 32-bit time support.
+
+  3. Upon meeting a symbol (type or function) which is known to be Y2038-
+     sensitive, if _USE_TIME_BITS64 is undefined or _TIME_BITS not properly
+     defined, this warning diagnostic is emitted:
+
+	Warning: <symbol> might be Y2038-sensitive
+
+     This reflects the fact that the user code is referring to a symbol
+     which, when glibc defaults to 32-bit time support, might fail Y2038.
+
+General note: y2038.py will handle multiple configurations, and will
+emit diagnostics for each configuration in turn.
+
+4. How to use the Y2038 cppcheck addon
+
+The Y2038 cppcheck addon is used like any other cppcheck addon:
+
+	cppcheck --dump file1.c [ file2.c [...]]]
+	y2038.py file1.c [ file2.c [...]]]
+
+Sample test C file is provided:
+
+	test/y2038-test-1-bad-time-bits.c
+	test/y2038-test-2-no-time-bits.c
+	test/y2038-test-3-no-use-time-bits.c
+	test/y2038-test-4-good.c
+
+These cover the cases described above. You can run them through cppcheck
+and y2038.py to see for yourself how the addon diagnostics look like. If
+this README is not outdated (and if it is, feel free to submit a patch),
+you can run cppcheck on these files as on any others:
+
+	cppcheck --dump addons/y2038/test/y2038-*.c
+	y2038.py addons/y2038/test/y2038-*.dump
+
+If you havve not installed cppcheck yet, you will have to run these
+commands from the root of the cppcheck repository:
+
+    make
+    sudo make install
+	./cppcheck --dump addons/y2038/test/y2038-*.c
+    PYTHONPATH=addons python addons/y2038/y2038.py addons/y2038/test/y2038-*.c.dump
+
+In both cases, y2038.py execution should result in the following:
+
+Checking addons/y2038/test/y2038-test-1-bad-time-bits.c.dump...
+Checking addons/y2038/test/y2038-test-1-bad-time-bits.c.dump, config ""...
+Checking addons/y2038/test/y2038-test-2-no-time-bits.c.dump...
+Checking addons/y2038/test/y2038-test-2-no-time-bits.c.dump, config ""...
+Checking addons/y2038/test/y2038-test-3-no-use-time-bits.c.dump...
+Checking addons/y2038/test/y2038-test-3-no-use-time-bits.c.dump, config ""...
+Checking addons/y2038/test/y2038-test-4-good.c.dump...
+Checking addons/y2038/test/y2038-test-4-good.c.dump, config ""...
+# Configuration "":
+# Configuration "":
+[addons/y2038/test/y2038-test-1-bad-time-bits.c:8]: (error) _TIME_BITS must be defined equal to 64
+[addons/y2038/test/y2038-inc.h:9]: (warning) _USE_TIME_BITS64 is defined but _TIME_BITS was not
+[addons/y2038/test/y2038-test-1-bad-time-bits.c:10]: (information) addons/y2038/test/y2038-inc.h was included from here
+[addons/y2038/test/y2038-inc.h:9]: (warning) _USE_TIME_BITS64 is defined but _TIME_BITS was not
+[addons/y2038/test/y2038-test-2-no-time-bits.c:8]: (information) addons/y2038/test/y2038-inc.h was included from here
+[addons/y2038/test/y2038-test-3-no-use-time-bits.c:13]: (warning) timespec might be Y2038-sensitive
+[addons/y2038/test/y2038-test-3-no-use-time-bits.c:15]: (warning) clock_gettime might be Y2038-sensitive
+
+Note: y2038.py recognizes option --template as cppcheck does, including
+pre-defined templates 'gcc', 'vs' and 'edit'. The short form -t is also
+recognized.

addons/y2038/test/y2038-inc.h

@@ -0,0 +1,11 @@
+#ifndef __INC2038
+#define _INC2038
+
+/*
+ * This file defines _USE_TIME_BITS64.
+ * It plays the role of a Y2038-proof glibc.
+ */
+
+#define _USE_TIME_BITS64
+
+#endif /* INC2038 */

addons/y2038/test/y2038-test-1-bad-time-bits.c

@@ -0,0 +1,18 @@
+#include <stdio.h>
+#include <fcntl.h>
+
+/*
+ * Define _TIME_BITS unequal to 64 to trigger error
+ */
+
+#define _TIME_BITS 62
+
+#include "y2038-inc.h"
+
+int main(int argc, char **argv)
+{
+    clockid_t my_clk_id;
+    struct timespec *my_tp;
+
+    return clock_gettime( my_clk_id, &my_tp);
+}

addons/y2038/test/y2038-test-2-no-time-bits.c

@@ -0,0 +1,16 @@
+#include <stdio.h>
+#include <fcntl.h>
+
+/*
+ * Do not define _TIME_BITS but have _USE_TIME_BITS64 defined
+ */
+
+#include "y2038-inc.h"
+
+int main(int argc, char **argv)
+{
+    clockid_t my_clk_id;
+    struct timespec *my_tp;
+
+    return clock_gettime( my_clk_id, &my_tp);
+}

addons/y2038/test/y2038-test-3-no-use-time-bits.c

@@ -0,0 +1,16 @@
+#include <stdio.h>
+#include <fcntl.h>
+
+/*
+ * Include bad _USE_TIME_BITS64 definition to trigger error
+ */
+
+#define _TIME_BITS 64
+
+int main(int argc, char **argv)
+{
+    clockid_t my_clk_id;
+    struct timespec *my_tp;
+
+    return clock_gettime( my_clk_id, &my_tp);
+}

addons/y2038/test/y2038-test-4-good.c

@@ -0,0 +1,15 @@
+/*
+ * Define _TIME_BITS equal to 64 so that glibc knows we want Y2038 support.
+ */
+
+#define _TIME_BITS 64
+
+#include "y2038-inc.h"
+
+int main(int argc, char **argv)
+{
+    clockid_t my_clk_id;
+    struct timespec *my_tp;
+
+    return clock_gettime( my_clk_id, &my_tp);
+}

addons/y2038/y2038.py

@@ -0,0 +1,188 @@
+#! /usr/bin/python
+#
+# cppcheck addon for Y2038 safeness detection
+#
+# Detects:
+#
+# 1. _TIME_BITS being defined to something else than 64 bits
+# 2. _USE_TIME_BITS64 being defined when _TIME_BITS is not
+# 3. Any Y2038-sensitive symbol when _USE_TIME_BITS64 is not defined.
+#
+# Example usage:
+# $ cppcheck --dump path-to-src/
+# $ y2038.py path-to-src/
+#
+# y2038.py will walk the source tree for .dump files.
+
+import cppcheckdata
+import sys
+import os
+import re
+import argparse
+
+#---------------
+# Error handling
+#---------------
+
+diagnostics = {}
+
+
+def reportDiagnostic(template, configuration, file, line, severity, message):
+    # collect diagnostics by configuration
+    if not configuration in diagnostics:
+        diagnostics[configuration] = []
+    # add error to this configuration
+    diagnostics[configuration].append(
+        cppcheckdata.reportError(template, [[file, line]], severity, message))
+
+
+def printDiagnostics():
+    for cfg in diagnostics:
+        sys.stderr.write('# Configuration "' + cfg + '":\n')
+        for diag in diagnostics[cfg]:
+            sys.stderr.write(diag + '\n')
+
+
+def reportDirDiag(template, cfg, filename, linenr, directive, severity, msg):
+    reportDiagnostic(template, cfg.name,
+                     directive.file, directive.linenr,
+                     severity, msg)
+    if (filename != directive.file) or (linenr != directive.linenr):
+        reportDiagnostic(template, cfg.name,
+                         filename, linenr, 'information',
+                         directive.file + ' was included from here')
+
+
+def reportTokDiag(template, cfg, token, severity, msg):
+    reportDiagnostic(template, cfg.name,
+                     token.file, token.linenr,
+                     severity, msg)
+
+#---------------------------------------------
+# #define/#undef detection regular expressions
+#---------------------------------------------
+
+# test for '#define _TIME_BITS 64'
+re_define_time_bits_64 = re.compile(r'^\s*#\s*define\s+_TIME_BITS\s+64\s*$')
+
+# test for '#define _TIME_BITS ...' (combine w/ above to test for 'not 64')
+re_define_time_bits = re.compile(r'^\s*#\s*define\s+_TIME_BITS\s+.*$')
+
+# test for '#undef _TIME_BITS' (if it ever happens)
+re_undef_time_bits = re.compile(r'^\s*#\s*undef\s+_TIME_BITS\s*$')
+
+# test for '#define _USE_TIME_BITS64'
+re_define_use_time_bits64 = re.compile(r'^\s*#\s*define\s+_USE_TIME_BITS64\s*$')
+
+# test for '#undef _USE_TIME_BITS64' (if it ever happens)
+re_undef_use_time_bits64 = re.compile(r'^\s*#\s*undef\s+_USE_TIME_BITS64\s*$')
+
+#------------------------------------
+# List of Y2038-sensitive identifiers
+#------------------------------------
+
+# This is WIP. Eventually it should contain all identifiers (types
+# and functions) which would be affected by the Y2038 bug.
+
+id_Y2038 = [
+    'time_t',
+    'timespec',
+    'clock_gettime'
+]
+
+# return all files ending in .dump among or under the given paths
+
+def find_dump_files(paths):
+    dumpfiles = []
+    for path in paths:
+        if path.endswith('.dump'):
+            if not path in dumpfiles:
+                dumpfiles.append(path)
+        else:
+            for (top, subdirs, files) in os.walk(path):
+                for file in files:
+                    if file.endswith('.dump'):
+                        f = top+'/'+file
+                        if not f in dumpfiles:
+                            dumpfiles.append(f)
+    dumpfiles.sort()
+    return dumpfiles
+
+#------------------
+# Let's get to work
+#------------------
+
+# extend cppcheck parser with our own options
+
+parser = cppcheckdata.ArgumentParser()
+parser.add_argument('-q', '--quiet', action='store_true',
+    help='do not print "Checking ..." lines')
+parser.add_argument('paths', nargs='+', metavar='path',
+    help='path to dump file or directory')
+
+# parse command line
+
+args = parser.parse_args()
+
+# now operate on each file in turn
+
+dumpfiles = find_dump_files(args.paths)
+
+for dumpfile in dumpfiles:
+    if not args.quiet:
+        print('Checking ' + dumpfile + '...')
+    srcfile = dumpfile.rstrip('.dump')
+    # at the start of the check, we don't know if code is Y2038 safe
+    y2038safe = False
+    # load XML from .dump file
+    data = cppcheckdata.parsedump(dumpfile)
+    # go through each configuration
+    for cfg in data.configurations:
+        if not args.quiet:
+            print('Checking ' + dumpfile + ', config "' + cfg.name + '"...')
+        safe_ranges = []
+        safe = -1
+        time_bits_defined = False
+        for directive in cfg.directives:
+            # track source line number
+            if directive.file == srcfile:
+                srclinenr = directive.linenr
+            # check for correct _TIME_BITS if present
+            if re_define_time_bits_64.match(directive.str):
+                time_bits_defined = True
+            elif re_define_time_bits.match(directive.str):
+                reportDirDiag(args.template, cfg, srcfile, srclinenr,
+                              directive, 'error',
+                              '_TIME_BITS must be defined equal to 64')
+                time_bits_defined = False
+            elif re_undef_time_bits.match(directive.str):
+                time_bits_defined = False
+            # check for _USE_TIME_BITS64 (un)definition
+            if re_define_use_time_bits64.match(directive.str):
+                safe = int(srclinenr)
+                # warn about _TIME_BITS not being defined
+                if time_bits_defined == False:
+                    reportDirDiag(args.template, 
+                        cfg, srcfile, srclinenr, directive, 'warning',
+                                  '_USE_TIME_BITS64 is defined but _TIME_BITS was not')
+            elif re_undef_use_time_bits64.match(directive.str):
+                unsafe = int(srclinenr)
+                # do we have a safe..unsafe area?
+                if safe > 0 and unsafe > safe:
+                    safe_ranges.append((safe, unsafe))
+                    safe = -1
+        # check end of source beyond last directive
+        if len(cfg.tokenlist) > 0:
+            unsafe = int(cfg.tokenlist[-1].linenr)
+            if safe > 0 and unsafe > safe:
+                safe_ranges.append((safe, unsafe))
+        # go through all tokens
+        for token in cfg.tokenlist:
+            if token.str in id_Y2038:
+                if not any(lower <= int(token.linenr) <= upper
+                           for (lower, upper) in safe_ranges):
+                    reportTokDiag(args.template, cfg, token, 'warning',
+                                  token.str + ' might be Y2038-sensitive')
+            token = token.next
+
+printDiagnostics()

lib/cppcheck.cpp

@@ -233,6 +233,7 @@ unsigned int CppCheck::processFile(const std::string& filename, std::istream& fi
                 // dump xml if --dump
                 if (_settings.dump && fdump.is_open()) {
                     fdump << "<dump cfg=\"" << cfg << "\">" << std::endl;
+                    preprocessor.dump(fdump);
                     _tokenizer.dump(fdump);
                     fdump << "</dump>" << std::endl;
                 }

lib/preprocessor.cpp

@@ -33,6 +33,41 @@
 #include <set>
 #include <stack>
 
+/**
+ * Remove heading and trailing whitespaces from the input parameter.
+ * If string is all spaces/tabs, return empty string.
+ * @param s The string to trim.
+ */
+static std::string trim(const std::string& s)
+{
+    const std::string::size_type beg = s.find_first_not_of(" \t");
+    if (beg == std::string::npos)
+        return "";
+    const std::string::size_type end = s.find_last_not_of(" \t");
+    return s.substr(beg, end - beg + 1);
+}
+
+Directive::Directive(const std::string &_file, const int _linenr, const std::string &_str):
+    file(_file),
+    linenr(_linenr),
+    str(_str)
+{
+    // strip C++ comment if there is one
+    std::size_t pos = str.find("//");
+    if (pos != std::string::npos)
+        str.erase(pos);
+    // strip any C comments
+    while ((pos = str.find("/*")) != std::string::npos) {
+        std::size_t end = str.find("*/", pos+2);
+        if (end != std::string::npos) {
+            str.erase(pos, end + 2 - pos);
+        } else { // treat '/*' as '//' if '*/' is missing
+            str.erase(pos);
+        }
+    }
+    str = trim(str);
+}
+
 bool Preprocessor::missingIncludeFlag;
 bool Preprocessor::missingSystemIncludeFlag;
 
@@ -1749,7 +1784,9 @@ bool Preprocessor::match_cfg_def(std::map<std::string, std::string> cfg, std::st
 
 std::string Preprocessor::getcode(const std::string &filedata, const std::string &cfg, const std::string &filename)
 {
-    // For the error report
+    // For the error report and preprocessor dump:
+    // line number relative to current (included) file
+    // (may decrease when popping back from an included file)
     unsigned int lineno = 0;
 
     std::ostringstream ret;
@@ -1766,9 +1803,28 @@ std::string Preprocessor::getcode(const std::string &filedata, const std::string
     std::stack<unsigned int> lineNumbers;
     std::istringstream istr(filedata);
     std::string line;
+    directives.clear();
     while (std::getline(istr, line)) {
         ++lineno;
 
+        if (line.empty()) {
+            ret << '\n';
+            continue;
+        }
+
+        // record directive for addons / checkers
+        if ((line[0] == '#')
+            && (line.compare(0, 6, "#line ") != 0)
+            && (line.compare(0, 8, "#endfile") != 0)) {
+            // for clarity, turn "#file ..." back into "#include ..."
+            std::string orig_line = line;
+            if (orig_line.compare(0, 6, "#file ")==0)
+                orig_line.replace(1, 4, "include");
+            // record directive and extra
+            directives.push_back(Directive(filenames.top(), lineno,
+                                           orig_line));
+        }
+
         if (_settings.terminated())
             return "";
 
@@ -2412,20 +2468,6 @@ static void skipstring(const std::string &line, std::string::size_type &pos)
     }
 }
 
-/**
- * Remove heading and trailing whitespaces from the input parameter.
- * If string is all spaces/tabs, return empty string.
- * @param s The string to trim.
- */
-static std::string trim(const std::string& s)
-{
-    const std::string::size_type beg = s.find_first_not_of(" \t");
-    if (beg == std::string::npos)
-        return "";
-    const std::string::size_type end = s.find_last_not_of(" \t");
-    return s.substr(beg, end - beg + 1);
-}
-
 /**
  * @brief get parameters from code. For example 'foo(1,2)' => '1','2'
  * @param line in: The code
@@ -3271,3 +3313,23 @@ void Preprocessor::getErrorMessages(ErrorLogger *errorLogger, const Settings *se
     preprocessor.validateCfgError("X", "X");
     preprocessor.error("", 1, "#error message");   // #error ..
 }
+
+void Preprocessor::dump(std::ostream &out) const
+{
+    // Create a xml directive dump.
+    // The idea is not that this will be readable for humans. It's a
+    // data dump that 3rd party tools could load and get useful info from.
+    std::list<Directive>::const_iterator it;
+
+    out << "  <directivelist>" << std::endl;
+
+    for (it = directives.begin(); it != directives.end(); ++it) {
+        out << "    <directive "
+            << "file=\"" << it->file << "\" "
+            << "linenr=\"" << it->linenr << "\" "
+            // str might contain characters such as '"', '<' or '>' which
+            // could result in invalid XML, so run it through toxml().
+            << "str=\"" << ErrorLogger::toxml(it->str) << "\"/>" << std::endl;
+    }
+    out << "  </directivelist>" << std::endl;
+}

lib/preprocessor.h

@@ -31,6 +31,30 @@
 class ErrorLogger;
 class Settings;
 
+/**
+ * @brief A preprocessor directive
+ * Each preprocessor directive (#include, #define, #undef,#if, #ifdef,
+ * #else, #endif) will be recorded as an instance of this class.
+ *
+ * file and linenr denote the location where where the directive is defined.
+ *
+ */
+
+class CPPCHECKLIB Directive {
+public:
+    /** name of (possibly included) file where directive is defined */
+    std::string file;
+
+    /** line number in (possibly included) file where directive is defined */
+    int linenr;
+
+    /** the actual directive text */
+    std::string str;
+
+    /** record a directive (possibly filtering src) */
+    Directive(const std::string &_file, const int _linenr, const std::string &_str);
+};
+
 /// @addtogroup Core
 /// @{
 
@@ -250,6 +274,11 @@ public:
         file0 = f;
     }
 
+    /**
+     * dump all directives present in source file
+     */
+    void dump(std::ostream &out) const;
+
 private:
     void missingInclude(const std::string &filename, unsigned int linenr, const std::string &header, HeaderTypes headerType);
 
@@ -271,6 +300,9 @@ private:
     Settings& _settings;
     ErrorLogger *_errorLogger;
 
+    /** list of all directives met while preprocessing file */
+    std::list<Directive> directives;
+
     /** filename for cpp/c file - useful when reporting errors */
     std::string file0;
 };

test/testpreprocessor.cpp

@@ -304,6 +304,10 @@ private:
 
         TEST_CASE(wrongPathOnUnicodeError); // see #6773
         TEST_CASE(wrongPathOnErrorDirective);
+
+        TEST_CASE(testDirectiveIncludeTypes);
+        TEST_CASE(testDirectiveIncludeLocations);
+        TEST_CASE(testDirectiveIncludeComments);
     }
 
     std::string preprocessorRead(const char* code) {
@@ -3743,6 +3747,91 @@ private:
         ASSERT_EQUALS("[test.c:1]: (error) #error hello world!\n", errout.str());
     }
 
+    void testDirectiveIncludeTypes() {
+        const char filedata[] = "#define macro some definition\n"
+                                "#undef macro\n"
+                                "#ifdef macro\n"
+                                "#elif some (complex) condition\n"
+                                "#else\n"
+                                "#endif\n"
+                                "#if some other condition\n"
+                                "#pragma some proprietary content\n"
+                                "#\n" /* may appear in old C code */
+                                "#ident some text\n" /* may appear in old C code */
+                                "#unknownmacro some unpredictable text\n"
+                                "#warning some warning message\n"
+                                "#error some error message\n";
+        const char dumpdata[] = "  <directivelist>\n"
+
+                                "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro some definition\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"2\" str=\"#undef macro\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"3\" str=\"#ifdef macro\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"4\" str=\"#elif some (complex) condition\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"5\" str=\"#else\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"6\" str=\"#endif\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"7\" str=\"#if some other condition\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"8\" str=\"#pragma some proprietary content\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"9\" str=\"#\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"10\" str=\"#ident some text\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"11\" str=\"#unknownmacro some unpredictable text\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"12\" str=\"#warning some warning message\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"13\" str=\"#error some error message\"/>\n"
+                                "  </directivelist>\n";
+
+        std::ostringstream ostr;
+        Settings settings;
+        Preprocessor preprocessor(settings, this);
+        preprocessor.getcode(filedata, "", "test.c");
+        preprocessor.dump(ostr);
+        ASSERT_EQUALS(dumpdata, ostr.str());
+    }
+
+    void testDirectiveIncludeLocations() {
+        const char filedata[] = "#define macro1 val\n"
+                                "#file \"inc1.h\"\n"
+                                "#define macro2 val\n"
+                                "#file \"inc2.h\"\n"
+                                "#define macro3 val\n"
+                                "#endfile\n"
+                                "#define macro4 val\n"
+                                "#endfile\n"
+                                "#define macro5 val\n";
+        const char dumpdata[] = "  <directivelist>\n"
+                                "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro1 val\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"2\" str=\"#include &quot;inc1.h&quot;\"/>\n"
+                                "    <directive file=\"inc1.h\" linenr=\"1\" str=\"#define macro2 val\"/>\n"
+                                "    <directive file=\"inc1.h\" linenr=\"2\" str=\"#include &quot;inc2.h&quot;\"/>\n"
+                                "    <directive file=\"inc2.h\" linenr=\"1\" str=\"#define macro3 val\"/>\n"
+                                "    <directive file=\"inc1.h\" linenr=\"3\" str=\"#define macro4 val\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"3\" str=\"#define macro5 val\"/>\n"
+                                "  </directivelist>\n";
+
+        std::ostringstream ostr;
+        Settings settings;
+        Preprocessor preprocessor(settings, this);
+        preprocessor.getcode(filedata, "", "test.c");
+        preprocessor.dump(ostr);
+        ASSERT_EQUALS(dumpdata, ostr.str());
+    }
+
+    void testDirectiveIncludeComments() {
+        const char filedata[] = "#ifdef macro2 /* this will be removed */\n"
+                                "#else /* this will be removed too */\n"
+                                "#endif /* this will also be removed */\n";
+        const char dumpdata[] = "  <directivelist>\n"
+                                "    <directive file=\"test.c\" linenr=\"1\" str=\"#ifdef macro2\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"2\" str=\"#else\"/>\n"
+                                "    <directive file=\"test.c\" linenr=\"3\" str=\"#endif\"/>\n"
+                                "  </directivelist>\n";
+
+        std::ostringstream ostr;
+        Settings settings;
+        Preprocessor preprocessor(settings, this);
+        preprocessor.getcode(filedata, "", "test.c");
+        preprocessor.dump(ostr);
+        ASSERT_EQUALS(dumpdata, ostr.str());
+    }
+
 };
 
 REGISTER_TEST(TestPreprocessor)

tools/dmake.cpp

@@ -430,6 +430,8 @@ int main(int argc, char **argv)
     fout << "install: cppcheck\n";
     fout << "\tinstall -d ${BIN}\n";
     fout << "\tinstall cppcheck ${BIN}\n";
+    fout << "\tinstall addons/*.py ${BIN}\n";
+    fout << "\tinstall addons/*/*.py ${BIN}\n";
     fout << "\tinstall htmlreport/cppcheck-htmlreport ${BIN}\n";
     fout << "ifdef CFGDIR \n";
     fout << "\tinstall -d ${DESTDIR}${CFGDIR}\n";