Dump compiler_info.json

A new flag is added to "CodeChecker analyze": --dump-compiler-info-file.
This flag is given a filename as parameter. This filename will be
created in the output folder. The file contains implicit include paths
and some other gcc-specific data that can be used to finetune analysis
later. When this flag is used, then analysis doesn't run.

The compiler_info.json was generated in the output directory by default.
Due to some bug, this file is empty in recent CodeChecker versions. The
reason is that parsing compile_commands.json is done in parallel with a
process pool. The dict object that collects this data must be handled by
multiprocessing.SyncManager() so it can be used in a process pool.

Author: bruntib

analyzer/codechecker_analyzer/buildlog/log_parser.py

@@ -7,7 +7,7 @@
 # -------------------------------------------------------------------------
 
 
-from collections import namedtuple
+from dataclasses import dataclass
 from shutil import which
 from enum import Enum
 from functools import lru_cache
@@ -322,9 +322,25 @@ class ImplicitCompilerInfo:
     # attributes to the implicit settings. In the future we may find that some
     # other attributes are also dependencies of implicit compiler info in which
     # case this tuple should be extended.
-    ImplicitInfoSpecifierKey = namedtuple(
-        'ImplicitInfoSpecifierKey',
-        ['compiler', 'language', 'compiler_flags'])
+    @dataclass
+    class ImplicitInfoSpecifierKey:
+        compiler: str
+        language: str
+        compiler_flags: list[str]
+
+        def __str__(self):
+            return json.dumps([
+                self.compiler,
+                self.language,
+                self.compiler_flags])
+
+        def __eq__(self, other):
+            return (self.compiler, self.language, self.compiler_flags) == \
+                   (other.compiler, other.language, other.compiler_flags)
+
+        def __hash__(self):
+            return hash(
+                (self.compiler, self.language, tuple(self.compiler_flags)))
 
     compiler_info: Dict[ImplicitInfoSpecifierKey, dict] = {}
     compiler_isexecutable = {}
@@ -333,14 +349,6 @@ class ImplicitCompilerInfo:
     # should be a clang version information object.
     compiler_versions = {}
 
-    @staticmethod
-    def c():
-        return "c"
-
-    @staticmethod
-    def cpp():
-        return "c++"
-
     @staticmethod
     def is_executable_compiler(compiler):
         if compiler not in ImplicitCompilerInfo.compiler_isexecutable:
@@ -378,11 +386,9 @@ def __get_compiler_err(cmd: List[str]) -> Optional[str]:
             _, err = proc.communicate("")
             return err
         except OSError as oerr:
-            # TODO: shlex.join(cmd) would be more elegant after upgrading to
-            # Python 3.8.
             LOG.error(
                 "Error during process execution: %s\n%s\n",
-                ' '.join(map(shlex.quote, cmd)), oerr.strerror)
+                shlex.join(cmd), oerr.strerror)
             return None
 
     @staticmethod
@@ -437,11 +443,7 @@ def get_compiler_includes(compiler, language, compiler_flags):
         """
         cmd = [compiler, *compiler_flags, '-E', '-x', language, '-', '-v']
 
-        # TODO: shlex.join(cmd) would be more elegant after upgrading to
-        # Python 3.8.
-        LOG.debug(
-            "Retrieving default includes via %s",
-            ' '.join(map(shlex.quote, cmd)))
+        LOG.debug("Retrieving default includes via %s", shlex.join(cmd))
         include_dirs = ImplicitCompilerInfo.__parse_compiler_includes(cmd)
 
         return list(map(os.path.normpath, include_dirs))
@@ -551,7 +553,7 @@ def get_compiler_standard(compiler, language):
     @staticmethod
     def dump_compiler_info(file_path: str):
         dumpable = {
-            json.dumps(k): v for k, v
+            str(k): v for k, v
             in ImplicitCompilerInfo.compiler_info.items()}
 
         with open(file_path, 'w', encoding="utf-8", errors="ignore") as f:
@@ -976,8 +978,7 @@ def parse_options(compilation_db_entry,
 
     if 'arguments' in compilation_db_entry:
         gcc_command = compilation_db_entry['arguments']
-        details['original_command'] = \
-            ' '.join([shlex.quote(x) for x in gcc_command])
+        details['original_command'] = shlex.join(gcc_command)
     elif 'command' in compilation_db_entry:
         details['original_command'] = compilation_db_entry['command']
         gcc_command = shlex.split(compilation_db_entry['command'])
@@ -1223,7 +1224,6 @@ def _process_entry_worker(args):
 
 
 def parse_unique_log(compilation_database,
-                     report_dir,
                      compile_uniqueing="none",
                      compiler_info_file=None,
                      keep_gcc_include_fixed=False,
@@ -1251,8 +1251,6 @@ def parse_unique_log(compilation_database,
                             by "arguments" which is a split command. Older
                             versions of intercept-build provide the build
                             command this way.
-    report_dir  -- The output report directory. The compiler infos
-                   will be written to <report_dir>/compiler.info.json.
     compile_uniqueing -- Compilation database uniqueing mode.
                          If there are more than one compile commands for a
                          target file, only a single one is kept.
@@ -1312,6 +1310,13 @@ def parse_unique_log(compilation_database,
                        keep_gcc_intrin)
                        for entry in entries)
 
+        # Here we overwrite ImplicitCompilerInfo.compiker_info with a dict type
+        # that can be used in multiprocess environment, since the next section
+        # is executed in a process pool.
+        manager = multiprocessing.SyncManager()
+        manager.start()
+        ImplicitCompilerInfo.compiler_info = manager.dict()
+
         # Process entries in parallel using imap_unordered with chunk size 1024
         with multiprocessing.Pool(jobs) as pool:
             # Convert generator to list for map function
@@ -1390,9 +1395,6 @@ def parse_unique_log(compilation_database,
                                   compile_uniqueing)
                         sys.exit(1)
 
-        ImplicitCompilerInfo.dump_compiler_info(
-            os.path.join(report_dir, "compiler_info.json"))
-
         LOG.debug('Parsing log file done.')
         return list(uniqued_build_actions.values()), skipped_cmp_cmd_count
 

analyzer/codechecker_analyzer/cli/analyze.py

@@ -229,6 +229,17 @@ def add_arguments_to_parser(parser):
                              "specified file rather than invoke the compiler "
                              "executable.")
 
+    parser.add_argument('--dump-compiler-info-file',
+                        dest="dump_compiler_info_file",
+                        required=False,
+                        action='store_true',
+                        default=False,
+                        help="Dump implicit gcc compiler info to a json file "
+                             "that can be used for fine-tuning analysis later."
+                             "These are information like the implicit include "
+                             "paths of standard headers, the default language "
+                             "version and the default target architecture.")
+
     parser.add_argument('--keep-gcc-include-fixed',
                         dest="keep_gcc_include_fixed",
                         required=False,
@@ -1254,7 +1265,8 @@ def main(args):
 
     args.output_path = os.path.abspath(args.output_path)
     if os.path.exists(args.output_path) and \
-            not os.path.isdir(args.output_path):
+            not os.path.isdir(args.output_path) and \
+            not args.dump_compiler_info_file:
         LOG.error("The given output path is not a directory: " +
                   args.output_path)
         sys.exit(1)
@@ -1342,20 +1354,15 @@ def main(args):
                  "overwriting with current result", args.output_path)
         shutil.rmtree(args.output_path)
 
-    if not os.path.exists(args.output_path):
+    if not os.path.exists(args.output_path) and \
+            not args.dump_compiler_info_file:
         os.makedirs(args.output_path)
 
-    # TODO: I'm not sure that this directory should be created here.
-    fixit_dir = os.path.join(args.output_path, 'fixit')
-    if not os.path.exists(fixit_dir):
-        os.makedirs(fixit_dir)
-
     LOG.debug("args: %s", str(args))
     LOG.debug("Output will be stored to: '%s'", args.output_path)
 
     actions, skipped_cmp_cmd_count = log_parser.parse_unique_log(
         compile_commands,
-        args.output_path,
         args.compile_uniqueing,
         compiler_info_file,
         args.keep_gcc_include_fixed,
@@ -1371,6 +1378,19 @@ def main(args):
                     "compilation database or all of them were skipped.")
         sys.exit(0)
 
+    if args.dump_compiler_info_file:
+        log_parser.ImplicitCompilerInfo.dump_compiler_info(
+            args.output_path)
+        sys.exit(0)
+    else:
+        log_parser.ImplicitCompilerInfo.dump_compiler_info(
+            Path(args.output_path) / "compiler_info.json")
+
+    # TODO: I'm not sure that this directory should be created here.
+    fixit_dir = os.path.join(args.output_path, 'fixit')
+    if not os.path.exists(fixit_dir):
+        os.makedirs(fixit_dir)
+
     uniqued_compilation_db_file = os.path.join(
         args.output_path, "unique_compile_commands.json")
     with open(uniqued_compilation_db_file, 'w',

analyzer/codechecker_analyzer/cli/check.py

@@ -129,6 +129,25 @@ def add_arguments_to_parser(parser):
                              "analyzers' output will not be printed to the "
                              "standard output.")
 
+    parser.add_argument('--compiler-info-file',
+                        dest="compiler_info_file",
+                        required=False,
+                        default=argparse.SUPPRESS,
+                        help="Read the compiler includes and target from the "
+                             "specified file rather than invoke the compiler "
+                             "executable.")
+
+    parser.add_argument('--dump-compiler-info-file',
+                        dest="dump_compiler_info_file",
+                        required=False,
+                        action='store_true',
+                        default=False,
+                        help="Dump implicit gcc compiler info to a json file "
+                             "that can be used for fine-tuning analysis later."
+                             "These are information like the implicit include "
+                             "paths of standard headers, the default language "
+                             "version and the default target architecture.")
+
     parser.add_argument('--keep-gcc-include-fixed',
                         dest="keep_gcc_include_fixed",
                         required=False,
@@ -980,6 +999,8 @@ def __update_if_key_exists(source, target, key):
                           'compile_uniqueing',
                           'report_hash',
                           'add_gcc_include_dirs_with_isystem',
+                          'compiler_info_file',
+                          'dump_compiler_info_file',
                           'enable_z3',
                           'enable_z3_refutation']
         for key in args_to_update:

analyzer/codechecker_analyzer/pre_analysis_manager.py

@@ -43,8 +43,7 @@ def collect_statistics(action, source, clangsa_config, statistics_data):
         LOG.debug('Can not collect statistical data.')
         return None
 
-    # TODO: shlex.join() will be more convenient in Python 3.8.
-    LOG.debug_analyzer(' '.join(map(shlex.quote, cmd)))
+    LOG.debug_analyzer(shlex.join(cmd))
 
     ret_code, analyzer_out, analyzer_err = \
         analyzer_base.SourceAnalyzer.run_proc(cmd)

analyzer/tests/functional/analyze/test_analyze.py

@@ -266,6 +266,45 @@ def test_compiler_info_file_is_loaded(self):
         self.assertTrue("--target=FAKE_TARGET" in out)
         self.assertTrue("-idirafter /FAKE_INCLUDE_DIR" in out)
 
+    def test_compiler_info_file_is_dumped(self):
+        """
+        Test if the compiler info file is dumped properly.
+        """
+        build_json = os.path.join(self.test_workspace, "build_simple.json")
+        source_file = os.path.join(self.test_workspace, "simple.cpp")
+
+        build_log = [{
+            "directory": self.test_workspace,
+            "command": f"g++ -c {source_file}",
+            "file": source_file
+        }]
+
+        with open(build_json, 'w',
+                  encoding="utf-8", errors="ignore") as outfile:
+            json.dump(build_log, outfile)
+
+        with tempfile.NamedTemporaryFile() as f:
+            analyze_cmd = [
+                self._codechecker_cmd, "analyze", build_json, "-o", f.name,
+                "--dump-compiler-info-file"]
+
+            process = subprocess.Popen(
+                analyze_cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=self.test_dir,
+                encoding="utf-8",
+                errors="ignore")
+            out, _ = process.communicate()
+            print(out)
+
+            f.flush()
+            f.seek(0)
+
+            compiler_info = json.loads(f.read())
+
+        self.assertIn('["g++", "c++", []]', compiler_info)
+
     def test_capture_analysis_output(self):
         """
         Test if reports/success/<output_file>.[stdout,stderr].txt

analyzer/tests/unit/test_buildcmd_escaping.py

@@ -88,7 +88,7 @@ def test_analyzer_exec_double_quote(self):
             ' -DDEBUG \'-DMYPATH="/this/some/path/"\''
 
         comp_actions, _ = log_parser.\
-            parse_unique_log(self.__get_cmp_json(compile_cmd), self.tmp_dir)
+            parse_unique_log(self.__get_cmp_json(compile_cmd))
 
         for comp_action in comp_actions:
             cmd = [self.compiler]
@@ -114,7 +114,7 @@ def test_analyzer_ansic_double_quote(self):
         """
         compile_cmd = self.compiler + ''' '-DMYPATH=\"/some/other/path\"' '''
         comp_actions, _ = log_parser.\
-            parse_unique_log(self.__get_cmp_json(compile_cmd), self.tmp_dir)
+            parse_unique_log(self.__get_cmp_json(compile_cmd))
 
         for comp_action in comp_actions:
             cmd = [self.compiler]