Dump compiler_info.json

A new flag is added to "CodeChecker analyze": --dump-compiler-info-file.
This flag is given a filename as parameter. This filename will be
created in the output folder. The file contains implicit include paths
and some other gcc-specific data that can be used to finetune analysis
later. When this flag is used, then analysis doesn't run.

The compiler_info.json was generated in the output directory by default.
Due to some bug, this file is empty in recent CodeChecker versions. The
reason is that parsing compile_commands.json is done in parallel with a
process pool. The dict object that collects this data must be handled by
multiprocessing.SyncManager() so it can be used in a process pool.

Author: bruntib

analyzer/codechecker_analyzer/buildlog/log_parser.py

@@ -7,7 +7,7 @@
 # -------------------------------------------------------------------------
 
 
-from collections import namedtuple
+from dataclasses import dataclass
 from shutil import which
 from enum import Enum
 from functools import lru_cache
@@ -322,9 +322,25 @@ class ImplicitCompilerInfo:
     # attributes to the implicit settings. In the future we may find that some
     # other attributes are also dependencies of implicit compiler info in which
     # case this tuple should be extended.
-    ImplicitInfoSpecifierKey = namedtuple(
-        'ImplicitInfoSpecifierKey',
-        ['compiler', 'language', 'compiler_flags'])
+    @dataclass
+    class ImplicitInfoSpecifierKey:
+        compiler: str
+        language: str
+        compiler_flags: list[str]
+
+        def __str__(self):
+            return json.dumps([
+                self.compiler,
+                self.language,
+                self.compiler_flags])
+
+        def __eq__(self, other):
+            return (self.compiler, self.language, self.compiler_flags) == \
+                   (other.compiler, other.language, other.compiler_flags)
+
+        def __hash__(self):
+            return hash(
+                (self.compiler, self.language, tuple(self.compiler_flags)))
 
     compiler_info: Dict[ImplicitInfoSpecifierKey, dict] = {}
     compiler_isexecutable = {}
@@ -333,14 +349,6 @@ class ImplicitCompilerInfo:
     # should be a clang version information object.
     compiler_versions = {}
 
-    @staticmethod
-    def c():
-        return "c"
-
-    @staticmethod
-    def cpp():
-        return "c++"
-
     @staticmethod
     def is_executable_compiler(compiler):
         if compiler not in ImplicitCompilerInfo.compiler_isexecutable:
@@ -378,11 +386,9 @@ def __get_compiler_err(cmd: List[str]) -> Optional[str]:
             _, err = proc.communicate("")
             return err
         except OSError as oerr:
-            # TODO: shlex.join(cmd) would be more elegant after upgrading to
-            # Python 3.8.
             LOG.error(
                 "Error during process execution: %s\n%s\n",
-                ' '.join(map(shlex.quote, cmd)), oerr.strerror)
+                shlex.join(cmd), oerr.strerror)
             return None
 
     @staticmethod
@@ -437,11 +443,7 @@ def get_compiler_includes(compiler, language, compiler_flags):
         """
         cmd = [compiler, *compiler_flags, '-E', '-x', language, '-', '-v']
 
-        # TODO: shlex.join(cmd) would be more elegant after upgrading to
-        # Python 3.8.
-        LOG.debug(
-            "Retrieving default includes via %s",
-            ' '.join(map(shlex.quote, cmd)))
+        LOG.debug("Retrieving default includes via %s", shlex.join(cmd))
         include_dirs = ImplicitCompilerInfo.__parse_compiler_includes(cmd)
 
         return list(map(os.path.normpath, include_dirs))
@@ -551,7 +553,7 @@ def get_compiler_standard(compiler, language):
     @staticmethod
     def dump_compiler_info(file_path: str):
         dumpable = {
-            json.dumps(k): v for k, v
+            str(k): v for k, v
             in ImplicitCompilerInfo.compiler_info.items()}
 
         with open(file_path, 'w', encoding="utf-8", errors="ignore") as f:
@@ -976,8 +978,7 @@ def parse_options(compilation_db_entry,
 
     if 'arguments' in compilation_db_entry:
         gcc_command = compilation_db_entry['arguments']
-        details['original_command'] = \
-            ' '.join([shlex.quote(x) for x in gcc_command])
+        details['original_command'] = shlex.join(gcc_command)
     elif 'command' in compilation_db_entry:
         details['original_command'] = compilation_db_entry['command']
         gcc_command = shlex.split(compilation_db_entry['command'])
@@ -1312,6 +1313,13 @@ def parse_unique_log(compilation_database,
                        keep_gcc_intrin)
                        for entry in entries)
 
+        # Here we overwrite ImplicitCompilerInfo.compiker_info with a dict type
+        # that can be used in multiprocess environment, since the next section
+        # is executed in a process pool.
+        manager = multiprocessing.SyncManager()
+        manager.start()
+        ImplicitCompilerInfo.compiler_info = manager.dict()
+
         # Process entries in parallel using imap_unordered with chunk size 1024
         with multiprocessing.Pool(jobs) as pool:
             # Convert generator to list for map function

analyzer/codechecker_analyzer/cli/analyze.py

@@ -229,6 +229,16 @@ def add_arguments_to_parser(parser):
                              "specified file rather than invoke the compiler "
                              "executable.")
 
+    parser.add_argument('--dump-compiler-info-file',
+                        dest="dump_compiler_info_file",
+                        required=False,
+                        default=argparse.SUPPRESS,
+                        help="Dump implicit gcc compiler info to a json file "
+                             "that can be used for fine-tuning analysis later."
+                             "These are information like the implicit include "
+                             "paths of standard headers, the default language "
+                             "version and the default target architecture.")
+
     parser.add_argument('--keep-gcc-include-fixed',
                         dest="keep_gcc_include_fixed",
                         required=False,
@@ -1371,6 +1381,17 @@ def main(args):
                     "compilation database or all of them were skipped.")
         sys.exit(0)
 
+    if args.dump_compiler_info_file:
+        compiler_info = Path(args.output_path) / 'compiler_info.json'
+        try:
+            os.rename(
+                compiler_info,
+                compiler_info.with_name(args.dump_compiler_info_file))
+        except ValueError as err:
+            LOG.error(err)
+            LOG.error("Provide a single filename.")
+        sys.exit(0)
+
     uniqued_compilation_db_file = os.path.join(
         args.output_path, "unique_compile_commands.json")
     with open(uniqued_compilation_db_file, 'w',

analyzer/codechecker_analyzer/cli/check.py

@@ -129,6 +129,23 @@ def add_arguments_to_parser(parser):
                              "analyzers' output will not be printed to the "
                              "standard output.")
 
+    parser.add_argument('--compiler-info-file',
+                        dest="compiler_info_file",
+                        required=False,
+                        default=argparse.SUPPRESS,
+                        help="Read the compiler includes and target from the "
+                             "specified file rather than invoke the compiler "
+                             "executable.")
+
+    parser.add_argument('--dump-compiler-info-file',
+                        dest="dump_compiler_info_file",
+                        required=False,
+                        help="Dump implicit gcc compiler info to a json file "
+                             "that can be used for fine-tuning analysis later."
+                             "These are information like the implicit include "
+                             "paths of standard headers, the default language "
+                             "version and the default target architecture.")
+
     parser.add_argument('--keep-gcc-include-fixed',
                         dest="keep_gcc_include_fixed",
                         required=False,
@@ -980,6 +997,8 @@ def __update_if_key_exists(source, target, key):
                           'compile_uniqueing',
                           'report_hash',
                           'add_gcc_include_dirs_with_isystem',
+                          'compiler_info_file',
+                          'dump_compiler_info_file',
                           'enable_z3',
                           'enable_z3_refutation']
         for key in args_to_update:

analyzer/codechecker_analyzer/pre_analysis_manager.py

@@ -43,8 +43,7 @@ def collect_statistics(action, source, clangsa_config, statistics_data):
         LOG.debug('Can not collect statistical data.')
         return None
 
-    # TODO: shlex.join() will be more convenient in Python 3.8.
-    LOG.debug_analyzer(' '.join(map(shlex.quote, cmd)))
+    LOG.debug_analyzer(shlex.join(cmd))
 
     ret_code, analyzer_out, analyzer_err = \
         analyzer_base.SourceAnalyzer.run_proc(cmd)

docs/analyzer/user_guide.md

@@ -1058,6 +1058,12 @@ optional arguments:
                         Read the compiler includes and target from the
                         specified file rather than invoke the compiler
                         executable.
+  --dump-compiler-info-file DUMP_COMPILER_INFO_FILE
+                        Dump implicit gcc compiler info to a json file that can
+                        be used for fine-tuning analysis later.These are
+                        information like the implicit include paths of standard
+                        headers, the default language version and the default
+                        target architecture.
   --keep-gcc-include-fixed
                         There are some implicit include paths which
                         are only used by GCC (include-fixed). This flag