seemingly working parse of compile_commands.json

README.md

@@ -31,3 +31,12 @@ command in your project to generate the documentation.
 For an easy introduction into using cldoc, please have a look at the
 [example project](https://github.com/jessevdk/cldoc/tree/master/example) and corresponding
 generated [documentation](http://jessevdk.github.com/cldoc/example/).
+
+# How to run for develpment purposes
+cd `realpath ${PROJECT_BUILD}`
+LD_LIBRARY_PATH=$(llvm-config-3.6 --libdir) PYTHONPATH=${HOME}/Codes/cldoc \
+    python -m cldoc.__init__ generate -- \
+    --compile_commands ${PROJECT_BUILD}/compile_commands.json \
+    --basedir `realpath ${PROJECT_DIR}` \
+    --output /tmp/cldoc
+

cldoc/clang/cindex.py

@@ -1098,6 +1098,18 @@ def __repr__(self):
 CursorKind.ANNOTATE_ATTR = CursorKind(406)
 CursorKind.ASM_LABEL_ATTR = CursorKind(407)
 CursorKind.PACKED_ATTR = CursorKind(408)
+CursorKind.PURE_ATTR = CursorKind(409)
+CursorKind.CONST_ATTR = CursorKind(410)
+CursorKind.NO_DUPLICATE_ATTR = CursorKind(411)
+CursorKind.CUDA_CONSTANT_ATTR = CursorKind(412)
+CursorKind.CUDA_DEVICE_ATTR = CursorKind(413)
+CursorKind.CUDA_GLOBAL_ATTR = CursorKind(414)
+CursorKind.CUDA_HOST_ATTR = CursorKind(415)
+CursorKind.CUDA_SHARED_ATTR = CursorKind(416)
+CursorKind.VISIBILITY_ATTR = CursorKind(417)
+CursorKind.DLL_EXPORT = CursorKind(418)
+CursorKind.DLL_IMPORT = CursorKind(419)
+
 
 ###
 # Preprocessing

cldoc/cmdgenerate.py

@@ -88,7 +88,10 @@ def run(args):
     parser.add_argument('--custom-css', default=[], metavar='FILES', action='append',
                           help='specify additional css files to be merged into the html (only for when --output is html)')
 
-    parser.add_argument('files', nargs='+', help='files to parse')
+    parser.add_argument('--compile_commands', default=None,
+                        help='Path to compile_commands.json')
+
+    parser.add_argument('files', nargs='*', help='files to parse')
 
     restargs = args[sep + 1:]
     cxxflags = args[:sep]
@@ -120,7 +123,7 @@ def run(args):
         cxxflags.append('-x')
         cxxflags.append(opts.language)
 
-    t = tree.Tree(opts.files, cxxflags)
+    t = tree.Tree(opts.basedir, opts.files, cxxflags, opts.compile_commands)
 
     t.process()
 

cldoc/nodes/method.py

@@ -40,7 +40,10 @@ def override(self):
             return self._override
 
         # Lookup in bases, recursively
-        bases = list(self.parent.bases)
+        if hasattr(self.parent, 'bases'):
+            bases = list(self.parent.bases)
+        else:
+            bases = list()
         mname = self.name
 
         self._override = []

cldoc/tree.py

@@ -21,6 +21,7 @@
 import nodes
 import includepaths
 import documentmerger
+import json
 
 from . import example
 from . import utf8
@@ -50,7 +51,7 @@
         if not lname is None:
             cindex.Config.set_library_file(lname)
 else:
-    versions = [None, '3.5', '3.4', '3.3', '3.2']
+    versions = [None, '3.6', '3.5', '3.4', '3.3', '3.2']
 
     for v in versions:
         name = 'clang'
@@ -72,15 +73,64 @@
     sys.stderr.write("\nFatal: Failed to locate libclang library. cldoc depends on libclang for parsing sources, please make sure you have libclang installed.\n" + str(e) + "\n\n")
     sys.exit(1)
 
+class ChangeDir(object):
+    """Provides 'with' semantics for os.chdir(). Will change to the desired directory and then
+       change back when leaving scope. Example:
+
+       os.chdir(os.expanduser('~'))
+       print os.getcwd()         # e.g. '/home/josh'
+       with ChangeDir('/tmp'):
+           print os.getcwd()     # '/tmp'
+       print os.getcwd()         # e.g. '/home/josh
+    """
+    # TODO(josh): investigate https://github.com/jaraco/path.py
+
+    def __init__(self, targetdir):
+        self.targetdir = targetdir
+        self.old_cwd = ''
+
+    def __enter__(self):
+        self.old_cwd = os.getcwd()
+        os.chdir(self.targetdir)
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        os.chdir(self.old_cwd)
+        return exception_value is None
+
+
+def strip_flag(flags, flag, nargs=0):
+    flag_index = flags.index(flag)
+    if flag_index != -1:
+        flags.pop(flag_index)
+        for i in range(nargs):
+            flags.pop(flag_index)
+
+
+def get_flags(command):
+    flags = includepaths.flags([]) + command.split()[1:]
+    strip_flag(flags, '-c', 1)
+    strip_flag(flags, '-o', 1)
+    # TODO(josh): remove these hacks
+    while '-Werror' in flags:
+        flags.remove('-Werror')
+    return flags + ['-Wno-mismatched-tags', '-Wno-absolute-value',
+                    '-Wno-ignored-qualifiers', '-Wno-unused-function']
+
 class Tree(documentmerger.DocumentMerger):
-    def __init__(self, files, flags):
+    def __init__(self, basedir, files, flags, command_db_path=None):
+        self.basedir = basedir
         self.processed = {}
         self.files, ok = self.expand_sources([os.path.realpath(f) for f in files])
 
         if not ok:
             sys.exit(1)
 
         self.flags = includepaths.flags(flags)
+        if command_db_path:
+            with open(command_db_path) as command_db_file:
+                self.compile_commands = json.load(command_db_file)
+        else:
+            self.compile_commands = None
 
         # Sort files on sources, then headers
         self.files.sort(lambda a, b: cmp(self.is_header(a), self.is_header(b)))
@@ -154,67 +204,96 @@ def find_node_comment(self, node):
 
         return None
 
-    def process(self):
+    def process_file(self, f, directory=None, command=None):
         """
-        process processes all the files with clang and extracts all relevant
-        nodes from the generated AST
+        process a single file
         """
+        if f in self.processed:
+            return
 
-        self.index = cindex.Index.create()
-        self.headers = {}
+        print('Processing {0}'.format(os.path.basename(f)))
 
-        for f in self.files:
-            if f in self.processed:
-                continue
+        if directory:
+            assert command is not None, "If directory is supplied, command must also be supplied"
+            with ChangeDir(directory):
+                tu = self.index.parse(f, get_flags(command))
+        else:
+            tu = self.index.parse(f, self.flags)
 
-            print('Processing {0}'.format(os.path.basename(f)))
+        if len(tu.diagnostics) != 0:
+            fatal = False
 
-            tu = self.index.parse(f, self.flags)
+            for d in tu.diagnostics:
+                sys.stderr.write(d.format)
+                sys.stderr.write("\n")
 
-            if len(tu.diagnostics) != 0:
-                fatal = False
+                if d.severity == cindex.Diagnostic.Fatal or \
+                   d.severity == cindex.Diagnostic.Error:
+                    fatal = True
 
-                for d in tu.diagnostics:
-                    sys.stderr.write(d.format)
-                    sys.stderr.write("\n")
+            if fatal:
+                sys.stderr.write("\nCould not generate documentation for %s due to parser errors\n" % (f,))
+                sys.exit(1)
+
+        if not tu:
+            sys.stderr.write("Could not parse file %s...\n" % (f,))
+            sys.exit(1)
 
-                    if d.severity == cindex.Diagnostic.Fatal or \
-                       d.severity == cindex.Diagnostic.Error:
-                        fatal = True
+        # Extract comments from files and included files that we are
+        # supposed to inspect
+        extractfiles = [f]
 
-                if fatal:
-                    sys.stderr.write("\nCould not generate documentation due to parser errors\n")
-                    sys.exit(1)
+        for inc in tu.get_includes():
+            filename = str(inc.include)
+            self.headers[filename] = True
 
-            if not tu:
-                sys.stderr.write("Could not parse file %s...\n" % (f,))
-                sys.exit(1)
+            if filename in self.processed or (not filename in self.files) or filename in extractfiles:
+                continue
 
-            # Extract comments from files and included files that we are
-            # supposed to inspect
-            extractfiles = [f]
+            extractfiles.append(filename)
 
-            for inc in tu.get_includes():
-                filename = str(inc.include)
-                self.headers[filename] = True
+        for e in extractfiles:
+            db = comment.CommentsDatabase(e, tu)
 
-                if filename in self.processed or (not filename in self.files) or filename in extractfiles:
-                    continue
+            self.add_categories(db.category_names)
+            self.commentsdbs[e] = db
+
+        self.visit(tu.cursor.get_children())
+
+        for f in self.processing:
+            self.processed[f] = True
+
+        self.processing = {}
+
+
+    def process(self):
+        """
+        process processes all the files with clang and extracts all relevant
+        nodes from the generated AST
+        """
 
-                extractfiles.append(filename)
+        self.index = cindex.Index.create()
+        self.headers = {}
 
-            for e in extractfiles:
-                db = comment.CommentsDatabase(e, tu)
 
-                self.add_categories(db.category_names)
-                self.commentsdbs[e] = db
+        num_files = len(self.files)
+        if self.compile_commands:
+            num_compile_commands = len(self.compile_commands)
+        else:
+            num_compile_commands = 0
 
-            self.visit(tu.cursor.get_children())
+        total_to_process = num_files + num_compile_commands
+        for i, f in enumerate(self.files):
+            progress_percent = 1e2 * i / total_to_process
+            sys.stdout.write('[{:6.2f}%] '.format(progress_percent))
+            self.process_file(f)
 
-            for f in self.processing:
-                self.processed[f] = True
+        if self.compile_commands:
+            for i, entry in enumerate(self.compile_commands):
+                progress_percent = 1e2 * (i + num_files) / total_to_process
+                sys.stdout.write('[{:6.2f}%] '.format(progress_percent))
+                self.process_file(entry['file'], entry['directory'], entry['command'])
 
-            self.processing = {}
 
         # Construct hierarchy of nodes.
         for node in self.all_nodes:
@@ -477,6 +556,20 @@ def visit(self, citer, parent=None):
 
             # Ignore files other than the ones we are scanning for
             if not str(item.location.file) in self.files:
+                # TODO(josh): re-enable this check after figuring out how to
+                # match files from compilation database
+                pass
+                # continue
+
+            realpath_to_file = os.path.realpath(str(item.location.file))
+            realpath_to_base = os.path.realpath(self.basedir)
+            relpath_from_basedir = os.path.relpath(realpath_to_file,
+                                                   realpath_to_base)
+
+            # Only include files that are a subpath of basedir
+            # NOTE(josh): item.location.file might be relative to the cwd
+            # of the command
+            if not len(relpath_from_basedir) < len(realpath_to_file):
                 continue
 
             # Ignore unexposed things