Get cpp repos & ccls graph server working correctly

Also, fix the parts of CGraphServer and GoGraphserver that broken by changing `files` attribute back to set

Merge branch 'hurthwell-branch' into 'master'

See merge request persper/code-analytics!16

persper/analytics/analyzer2.py

@@ -1,13 +1,13 @@
 import asyncio
-from abc import ABC
 import collections.abc
-from typing import Union, Set, Optional, List
+from abc import ABC
+from typing import List, Optional, Set, Union
 
 from git import Commit, Diff, DiffIndex, Repo
 
-from persper.analytics.git_tools import (diff_with_commit, get_contents)
-from persper.analytics.graph_server import CommitSeekingMode, GraphServer
 from persper.analytics.commit_classifier import CommitClassifier
+from persper.analytics.git_tools import diff_with_commit, get_contents
+from persper.analytics.graph_server import CommitSeekingMode, GraphServer
 from persper.analytics.score import commit_overall_scores
 
 
@@ -44,6 +44,10 @@ def __setstate__(self, state):
         self.terminalCommit = state["_terminalCommit"]
         self._s_visitedCommits = _ReadOnlySet(self._visitedCommits)
 
+    @property
+    def graphServer(self):
+        return self._graphServer
+
     @property
     def observer(self):
         """
@@ -155,6 +159,7 @@ def printCommitStatus(status: str):
                 # then go on with current commit
                 printCommitStatus("Going forward.")
                 await self._analyzeCommit(commit, parent, CommitSeekingMode.NormalForward)
+            self._visitedCommits.add(commit.hexsha)
             graphServerLastCommit = commit.hexsha
             analyzedCommits += 1
 

persper/analytics/call_commit_graph.py

@@ -7,6 +7,7 @@
 from networkx.readwrite import json_graph
 from persper.analytics.devrank import devrank
 from persper.analytics.score import normalize
+from typing import Union, Set, List, Dict, Optional
 
 
 class CommitIdGenerators:
@@ -29,14 +30,23 @@ class CallCommitGraph:
     and edit histories across commits.
     """
 
-    def __init__(self, node_link_data=None, commit_id_generator=CommitIdGenerators.fromHexsha):
-        if node_link_data:
-            self._digraph = json_graph.node_link_graph(node_link_data)
+    def __init__(self, graph_data: Optional[Dict] = None, commit_id_generator=CommitIdGenerators.fromHexsha):
+        if graph_data:
+            self._digraph = json_graph.node_link_graph(
+                CallCommitGraph._to_networkx_format(graph_data))
         else:
             self._digraph = self._new_graph()
         self._commit_id_generator = commit_id_generator
         self._current_commit_id = None
 
+    @staticmethod
+    def _to_networkx_format(graph_data: Dict) -> Dict:
+        graph_data['multigraph'] = False
+        graph_data['directed'] = True
+        for node in graph_data['nodes']:
+            node['files'] = set(node['files'])
+        return graph_data
+
     def reset(self):
         """Reset all internal states"""
         self._digraph = self._new_graph()
@@ -59,6 +69,10 @@ def commits(self):
         # https://networkx.github.io/documentation/stable/tutorial.html#graph-attributes
         return self._digraph.graph['commits']
 
+    def files(self, node: str) -> Set[str]:
+        """Provide read-only access to `files` attribute of a node"""
+        return self.nodes()[node]['files']
+
     def __contains__(self, node):
         """Implement membership check"""
         return node in self._digraph
@@ -82,8 +96,8 @@ def _next_cindex(self):
         return self._cur_cindex() + 1
 
     # TODO: remove the default value of files
-    def add_node(self, node, files=[]):
-        self._digraph.add_node(node, size=None, history={}, files=files)
+    def add_node(self, node: str, files: Union[Set[str], List[str]] = []):
+        self._digraph.add_node(node, size=None, history={}, files=set(files))
 
     # add_node must be called on source and target first
     def add_edge(self, source, target):
@@ -106,11 +120,11 @@ def update_node_history(self, node, num_adds, num_dels):
             node_history[self._current_commit_id] = {'adds': num_adds, 'dels': num_dels}
 
     # read/write access to node history are thourgh this function
-    def _get_node_history(self, node):
+    def _get_node_history(self, node: str) -> Dict[str, Dict[str, int]]:
         return self._digraph.nodes[node]['history']
 
-    def update_node_files(self, node, new_files):
-        self._digraph.nodes[node]['files'] = new_files
+    def update_node_files(self, node: str, new_files: Union[Set[str], List[str]]):
+        self._digraph.nodes[node]['files'] = set(new_files)
 
     # TODO: provide other options for computing a node's size
     def _set_all_nodes_size(self, black_set=None):

persper/analytics/call_graph/c.py

@@ -1,4 +1,5 @@
 from persper.analytics.call_graph.utils import ns, line_attr
+from typing import Set
 
 
 class NotFunctionCallError(Exception):
@@ -83,9 +84,10 @@ def update_graph(ccgraph, ast_list, change_stats):
             if caller_name not in ccgraph:
                 ccgraph.add_node(caller_name, [filename])
             else:
-                files = ccgraph.nodes()[caller_name]['files']
+                files: Set[str] = ccgraph.files(caller_name)
                 if filename not in files:
-                    ccgraph.update_node_files(caller_name, files + [filename])
+                    files.add(filename)
+                    ccgraph.update_node_files(caller_name, files)
 
             for call in function.xpath('.//srcml:call', namespaces=ns):
                 try:

persper/analytics/go.py

@@ -40,10 +40,7 @@ def update_graph(self, old_filename, old_src,
     def get_graph(self):
         graph_url = self.server_addr + '/callgraph'
         r = requests.get(graph_url)
-        graph_data = r.json()
-        graph_data['directed'] = True
-        graph_data['multigraph'] = False
-        return CallCommitGraph(graph_data)
+        return CallCommitGraph(graph_data=r.json())
 
     def reset_graph(self):
         reset_url = urllib.parse.urljoin(self.server_addr, '/reset')

persper/analytics/lsp_graph_server/callgraph/builder.py

@@ -58,20 +58,20 @@ def PopulateSymbols(symbols):
                 if filterResult == None:
                     continue
                 if filterResult == False:
-                    if isinstance(s, SymbolInformation):
+                    if isinstance(s, DocumentSymbol):
                         PopulateSymbols(s.children)
                     continue
                 if isinstance(s, DocumentSymbol):
                     # We assume selectionRange is exactly the range of symbol name
                     symbolKinds[s.selectionRange.start.toTuple()] = s.kind
                     self._scopes.append(CallGraphScope(s.detail or s.name, s.kind,
                                                        fileName, s.range.start, s.range.end))
+                    if s.children:
+                        PopulateSymbols(s.children)
                 elif isinstance(s, SymbolInformation):
                     symbolKinds[(s.location.range.start.line, s.name)] = (s.location.range.start.character, s.kind)
                     self._scopes.append(CallGraphScope(s.containerName, s.kind, fileName,
                                                        s.location.range.start, s.location.range.end))
-                    if s.children:
-                        PopulateSymbols(s.children)
                 else:
                     _logger.error("Invalid DocumentSymbol in %s: %s", fileName, s)
 
@@ -293,6 +293,11 @@ def filterSymbol(self, symbol: Union[DocumentSymbol, SymbolInformation]) -> bool
             False   symbol should be excluded, while its children will pass filterSymbol
             None    symbol and its children will be excluded
         """
+        if symbol.kind in {
+            SymbolKind.Parameter,
+            SymbolKind.TypeParameter
+        }:
+            return None
         return symbol.kind in {
             SymbolKind.Constructor,
             SymbolKind.Enum,

persper/analytics/lsp_graph_server/callgraph/manager.py

@@ -42,6 +42,7 @@ async def buildGraph(self, fileNames: Union[str, Iterable[str]] = None, globPatt
         branchCounter = 0       # with dups
         fileCounter = 0
         await self._builder.waitForFileSystem()
+
         def pushBranch(branch):
             nonlocal branchCounter
             try:
@@ -64,7 +65,7 @@ def pushBranch(branch):
                     pushBranch(b)
         if globPattern or not fileNames:
             async for b in self._builder.buildCallGraphInFiles(globPattern):
-                pushBranch(b)        
+                pushBranch(b)
         if fileNames and not globPattern:
             _logger.info("Added %d branches from %d files.", branchCounter, fileCounter)
         else:

persper/analytics/lsp_graph_server/languageclient/lspcontract.py

@@ -237,7 +237,7 @@ def getSymbolRange(self):
         return self.selectionRange
 
     def __str__(self):
-        return self.name + "[" + self.kind + "]"
+        return str.format("{0}({1}){2}", self.name, self.kind, self.children or "")
 
     def toDict(self):
         raise NotImplementedError()