Merge 2-0-445 to backup_sync

build.sbt

@@ -2,7 +2,7 @@ name                     := "joern"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.4.2"
 
-val cpgVersion = "1.6.16"
+val cpgVersion = "1.6.19"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb

console/src/main/scala/io/joern/console/Commit.scala

@@ -3,7 +3,7 @@ package io.joern.console
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
-import overflowdb.BatchedUpdate.DiffGraphBuilder
+import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
 
 object Commit {
   val overlayName: String = "commit"

console/src/main/scala/io/joern/console/Console.scala

@@ -12,7 +12,7 @@ import io.shiftleft.codepropertygraph.cpgloading.CpgLoader
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.dotextension.ImageViewer
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
-import overflowdb.traversal.help.Doc
+import io.shiftleft.codepropertygraph.generated.help.Doc
 import overflowdb.traversal.help.Table.AvailableWidthProvider
 
 import scala.sys.process.Process

console/src/main/scala/io/joern/console/ConsoleConfig.scala

@@ -1,6 +1,6 @@
 package io.joern.console
 
-import better.files._
+import better.files.*
 
 import scala.annotation.tailrec
 import scala.collection.mutable

console/src/main/scala/io/joern/console/PluginManager.scala

@@ -1,5 +1,5 @@
 package io.joern.console
-import better.files.Dsl._
+import better.files.Dsl.*
 import better.files.File
 import better.files.File.apply
 

console/src/main/scala/io/joern/console/Run.scala

@@ -6,7 +6,7 @@ import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
 import org.reflections8.Reflections
 import org.reflections8.util.{ClasspathHelper, ConfigurationBuilder}
 
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 
 object Run {
 
@@ -64,7 +64,7 @@ object Run {
          |
          |val opts = new OptsDynamic()
          |
-         | import _root_.overflowdb.BatchedUpdate.DiffGraphBuilder
+         | import _root_.io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
          | implicit def _diffGraph: DiffGraphBuilder = opts.commit.diffGraphBuilder
          | def diffGraph = _diffGraph
          |""".stripMargin

console/src/main/scala/io/joern/console/cpgcreation/CpgGenerator.scala

@@ -3,7 +3,7 @@ package io.joern.console.cpgcreation
 import better.files.File
 import io.shiftleft.codepropertygraph.generated.Cpg
 
-import scala.sys.process._
+import scala.sys.process.*
 import scala.util.Try
 
 /** A CpgGenerator generates Code Property Graphs from code. Each supported language implements a Generator, e.g.,

console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala

@@ -1,6 +1,6 @@
 package io.joern.console.cpgcreation
 
-import better.files.Dsl._
+import better.files.Dsl.*
 import better.files.File
 import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
 import io.shiftleft.codepropertygraph.generated.Languages

console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala

@@ -3,7 +3,7 @@ package io.joern.console.cpgcreation
 import io.joern.console.FrontendConfig
 
 import java.nio.file.Path
-import scala.sys.process._
+import scala.sys.process.*
 import scala.util.{Failure, Try}
 
 /** Language frontend for Java archives (JAR files). Translates Java archives into code property graphs.

console/src/main/scala/io/joern/console/workspacehandling/Project.scala

@@ -1,6 +1,6 @@
 package io.joern.console.workspacehandling
 
-import better.files.Dsl._
+import better.files.Dsl.*
 import better.files.File
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.Overlays

console/src/main/scala/io/joern/console/workspacehandling/WorkspaceManager.scala

@@ -1,7 +1,7 @@
 package io.joern.console.workspacehandling
 
-import better.files.Dsl._
-import better.files._
+import better.files.Dsl.*
+import better.files.*
 import io.joern.console
 import io.joern.console.defaultAvailableWidthProvider
 import io.joern.console.Reporting
@@ -315,7 +315,7 @@ class WorkspaceManager[ProjectType <: Project](path: String, loader: WorkspaceLo
       case Success(v) => Some(v)
       case Failure(ex) =>
         System.err.println("Error loading CPG")
-        System.err.println(ex)
+        ex.printStackTrace()
         None
     }
   }

console/src/test/scala/io/joern/console/ConsoleTests.scala

@@ -1,11 +1,11 @@
 package io.joern.console
 
-import better.files.Dsl._
-import better.files._
-import io.joern.console.testing._
+import better.files.Dsl.*
+import better.files.*
+import io.joern.console.testing.*
 import io.joern.x2cpg.X2Cpg.defaultOverlayCreators
 import io.joern.x2cpg.layers.{Base, CallGraph, ControlFlow, TypeRelations}
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec

console/src/test/scala/io/joern/console/LanguageHelperTests.scala

@@ -1,7 +1,7 @@
 package io.joern.console
 
-import better.files.Dsl._
-import better.files._
+import better.files.Dsl.*
+import better.files.*
 import io.shiftleft.codepropertygraph.generated.Languages
 import io.joern.console.cpgcreation.{guessLanguage, LlvmCpgGenerator}
 import org.scalatest.matchers.should.Matchers

console/src/test/scala/io/joern/console/PluginManagerTests.scala

@@ -1,7 +1,7 @@
 package io.joern.console
 
-import better.files.Dsl._
-import better.files._
+import better.files.Dsl.*
+import better.files.*
 import io.shiftleft.utils.ProjectRoot
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec

console/src/test/scala/io/joern/console/workspacehandling/WorkspaceManagerTests.scala

@@ -1,6 +1,6 @@
 package io.joern.console.workspacehandling
 
-import better.files._
+import better.files.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec

console/src/test/scala/io/joern/console/workspacehandling/WorkspaceTests.scala

@@ -1,6 +1,6 @@
 package io.joern.console.workspacehandling
 
-import better.files.Dsl._
+import better.files.Dsl.*
 import better.files.File
 import io.joern.console.testing.availableWidthProvider
 import io.shiftleft.semanticcpg.testing.MockCpg

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/dotgenerator/DdgGenerator.scala

@@ -1,15 +1,13 @@
 package io.joern.dataflowengineoss.dotgenerator
 
 import io.joern.dataflowengineoss.DefaultSemantics
-import io.shiftleft.codepropertygraph.generated.nodes._
+import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Properties}
-import io.joern.dataflowengineoss.language._
+import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.semanticsloader.Semantics
 import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph}
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.utils.MemberAccess.isGenericMemberAccessName
-import overflowdb.Node
-import overflowdb.traversal.jIteratortoTraversal
 
 import scala.collection.mutable
 
@@ -59,7 +57,7 @@ class DdgGenerator {
     }
   }
 
-  private def shouldBeDisplayed(v: Node): Boolean = !(
+  private def shouldBeDisplayed(v: StoredNode): Boolean = !(
     v.isInstanceOf[ControlStructure] ||
       v.isInstanceOf[JumpTarget]
   )

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/language/package.scala

@@ -4,7 +4,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.semanticcpg.language.*
 import io.joern.dataflowengineoss.language.dotextension.DdgNodeDot
 import io.joern.dataflowengineoss.language.nodemethods.{ExpressionMethods, ExtendedCfgNodeMethods}
-import overflowdb.traversal.help.Doc
+import io.shiftleft.codepropertygraph.generated.help.Doc
 
 import scala.language.implicitConversions
 

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpCpg14.scala

@@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.layers.dataflows
 
 import better.files.File
 import io.joern.dataflowengineoss.DefaultSemantics
-import io.joern.dataflowengineoss.language._
+import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.semanticsloader.Semantics
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
 
 case class Cpg14DumpOptions(var outDir: String) extends LayerCreatorOptions {}

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpDdg.scala

@@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.layers.dataflows
 
 import better.files.File
 import io.joern.dataflowengineoss.DefaultSemantics
-import io.joern.dataflowengineoss.language._
+import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.semanticsloader.Semantics
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
 
 case class DdgDumpOptions(var outDir: String) extends LayerCreatorOptions {}

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/layers/dataflows/DumpPdg.scala

@@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.layers.dataflows
 
 import better.files.File
 import io.joern.dataflowengineoss.DefaultSemantics
-import io.joern.dataflowengineoss.language._
+import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.semanticsloader.Semantics
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
 
 case class PdgDumpOptions(var outDir: String) extends LayerCreatorOptions {}

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/DdgGenerator.scala

@@ -7,7 +7,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators, PropertyNames}
 import io.shiftleft.semanticcpg.accesspath.MatchResult
 import io.shiftleft.semanticcpg.language.*
-import overflowdb.BatchedUpdate.DiffGraphBuilder
+import io.shiftleft.codepropertygraph.generated.DiffGraphBuilder
 
 import scala.collection.{Set, mutable}
 

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala

@@ -2,9 +2,9 @@ package io.joern.dataflowengineoss.passes.reachingdef
 
 import io.joern.dataflowengineoss.semanticsloader.Semantics
 import io.shiftleft.codepropertygraph.generated.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes._
+import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.passes.ForkJoinParallelCpgPass
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.mutable

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala

@@ -167,7 +167,7 @@ class ReachingDefTransferFunction(flowGraph: ReachingDefFlowGraph)
   val gen: Map[StoredNode, mutable.BitSet] =
     initGen(method).withDefaultValue(mutable.BitSet())
 
-  val kill: Map[StoredNode, Set[Definition]] =
+  val kill: Map[StoredNode, mutable.BitSet] =
     initKill(method, gen).withDefaultValue(mutable.BitSet())
 
   /** For a given flow graph node `n` and set of definitions, apply the transfer function to obtain the updated set of
@@ -224,7 +224,7 @@ class ReachingDefTransferFunction(flowGraph: ReachingDefFlowGraph)
     * All operations in our graph are represented by calls and non-operations such as identifiers or field-identifiers
     * have empty gen and kill sets, meaning that they just pass on definitions unaltered.
     */
-  private def initKill(method: Method, gen: Map[StoredNode, Set[Definition]]): Map[StoredNode, Set[Definition]] = {
+  private def initKill(method: Method, gen: Map[StoredNode, mutable.BitSet]): Map[StoredNode, mutable.BitSet] = {
 
     val allIdentifiers: Map[String, List[CfgNode]] = {
       val results             = mutable.Map.empty[String, List[CfgNode]]
@@ -259,42 +259,44 @@ class ReachingDefTransferFunction(flowGraph: ReachingDefFlowGraph)
     * calculate kill(call) based on gen(call).
     */
   private def killsForGens(
-    genOfCall: Set[Definition],
+    genOfCall: mutable.BitSet,
     allIdentifiers: Map[String, List[CfgNode]],
     allCalls: Map[String, List[Call]]
-  ): Set[Definition] = {
+  ): mutable.BitSet = {
 
-    def definitionsOfSameVariable(definition: Definition): Set[Definition] = {
+    def definitionsOfSameVariable(definition: Definition): Iterator[Definition] = {
       val definedNodes = flowGraph.numberToNode(definition) match {
         case param: MethodParameterIn =>
-          allIdentifiers(param.name)
+          allIdentifiers(param.name).iterator
             .filter(x => x.id != param.id)
         case identifier: Identifier =>
-          val sameIdentifiers = allIdentifiers(identifier.name)
+          val sameIdentifiers = allIdentifiers(identifier.name).iterator
             .filter(x => x.id != identifier.id)
 
           /** Killing an identifier should also kill field accesses on that identifier. For example, a reassignment `x =
             * new Box()` should kill any previous calls to `x.value`, `x.length()`, etc.
             */
-          val sameObjects: Iterable[Call] = allCalls.values.flatten
+          val sameObjects: Iterator[Call] = allCalls.valuesIterator.flatten
             .filter(_.name == Operators.fieldAccess)
             .filter(_.ast.isIdentifier.nameExact(identifier.name).nonEmpty)
 
           sameIdentifiers ++ sameObjects
         case call: Call =>
-          allCalls(call.code)
+          allCalls(call.code).iterator
             .filter(x => x.id != call.id)
-        case _ => Set()
+        case _ => Iterator.empty
       }
       definedNodes
         // It can happen that the CFG is broken and contains isolated nodes,
         // in which case they are not in `nodeToNumber`. Let's filter those.
-        .collect { case x if nodeToNumber.contains(x) => Definition.fromNode(x, nodeToNumber) }.toSet
+        .collect { case x if nodeToNumber.contains(x) => Definition.fromNode(x, nodeToNumber) }
     }
 
-    genOfCall.flatMap { definition =>
-      definitionsOfSameVariable(definition)
+    val res = mutable.BitSet()
+    for (definition <- genOfCall) {
+      res.addAll(definitionsOfSameVariable(definition))
     }
+    res
   }
 
 }

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/AccessPathUsage.scala

@@ -1,8 +1,8 @@
 package io.joern.dataflowengineoss.queryengine
 
-import io.shiftleft.codepropertygraph.generated.nodes._
-import io.shiftleft.semanticcpg.accesspath._
-import io.shiftleft.semanticcpg.language.{AccessPathHandling, toCallMethods}
+import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.semanticcpg.accesspath.*
+import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.utils.MemberAccess
 import org.slf4j.LoggerFactory
 

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/Engine.scala

@@ -254,7 +254,6 @@ object Engine {
   private def ddgInE(node: CfgNode, path: Vector[PathElement], callSiteStack: List[Call] = List()): Vector[Edge] = {
     node
       .inE(EdgeTypes.REACHING_DEF)
-      .asScala
       .filter { e =>
         e.outNode() match {
           case srcNode: CfgNode =>

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/HeldTaskCompletion.scala

@@ -1,7 +1,7 @@
 package io.joern.dataflowengineoss.queryengine
 
 import scala.collection.mutable
-import scala.collection.parallel.CollectionConverters._
+import scala.collection.parallel.CollectionConverters.*
 
 /** Complete held tasks using the result table. The result table is modified in the process.
   *

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/SourcesToStartingPoints.scala

@@ -32,7 +32,7 @@ object SourcesToStartingPoints {
         .map(src => {
           // We need to get Cpg wrapper from graph. Hence we are taking head element from source iterator.
           // This will also ensure if the source list is empty then these tasks are invoked.
-          val cpg                           = Cpg(src.graph())
+          val cpg                           = Cpg(src.graph)
           val (startingPoints, methodTasks) = calculateStartingPoints(sources, executorService)
           val startingPointFromUsageInOtherClasses =
             calculateStatingPointsWithUsageInOtherClasses(methodTasks, cpg, executorService)

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskCreator.scala

@@ -100,7 +100,7 @@ class TaskCreator(context: EngineContext) {
     */
 
   private def paramToMethodRefCallReceivers(param: MethodParameterIn): List[Expression] =
-    new Cpg(param.graph()).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l
+    new Cpg(param.graph).methodRef.methodFullNameExact(param.method.fullName).inCall.argument(0).l
 
   /** Create new tasks from all results that end in an output argument, including return arguments. In this case, we
     * want to traverse to corresponding method output parameters and method return nodes respectively.

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/queryengine/TaskSolver.scala

@@ -2,7 +2,7 @@ package io.joern.dataflowengineoss.queryengine
 
 import io.joern.dataflowengineoss.queryengine.QueryEngineStatistics.{PATH_CACHE_HITS, PATH_CACHE_MISSES}
 import io.joern.dataflowengineoss.semanticsloader.Semantics
-import io.shiftleft.codepropertygraph.generated.nodes._
+import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.semanticcpg.language.{toCfgNodeMethods, toExpressionMethods, _}
 
 import java.util.concurrent.Callable

dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/semanticsloader/Parser.scala

@@ -7,7 +7,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker
 import org.antlr.v4.runtime.{CharStream, CharStreams, CommonTokenStream}
 
 import scala.collection.mutable
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 
 object Semantics {