[gosrc2cpg] - Download dependency and process the dependencies - joer…

…nio#3688 (joernio#3718) * Download dependency and process the dependencies * ignoring one test becuase of high memory usage
Privado-Inc · Oct 5, 2023 · fc471ab · fc471ab
1 parent d3659d0
commit fc471ab
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 5 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -19,6 +19,10 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
       - name: Install Bundler
         run: gem install bundler
       - name: Delete `.rustup` directory

diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt
@@ -11,6 +11,7 @@ libraryDependencies ++= Seq(
   "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
   "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test,
   "com.lihaoyi"               %% "upickle"           % Versions.upickle,
+  "com.lihaoyi"               %% "os-lib"            % "0.9.1",
   "com.fasterxml.jackson.core" % "jackson-databind"  % "2.15.2",
   "com.typesafe"               % "config"            % "1.4.2",
   "com.michaelpollmeier"       % "versionsort"       % "1.0.11",

diff --git a/...ontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala b/...ontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala
@@ -1,12 +1,17 @@
 package io.joern.gosrc2cpg.passes
 
 import better.files.File
+import io.joern.gosrc2cpg.Config
 import io.joern.gosrc2cpg.model.GoModHelper
+import io.joern.gosrc2cpg.parser.GoAstJsonParser
+import io.joern.gosrc2cpg.utils.AstGenRunner
 import io.joern.gosrc2cpg.utils.AstGenRunner.getClass
 import io.joern.x2cpg.utils.ExternalCommand
 import org.slf4j.LoggerFactory
 
-import scala.util.{Failure, Success}
+import java.io.File as JFile
+import java.nio.file.Paths
+import scala.util.{Failure, Success, Try}
 
 class DownloadDependenciesPass(parentGoMod: GoModHelper) {
   private val logger = LoggerFactory.getLogger(getClass)
@@ -23,14 +28,31 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper) {
         ExternalCommand.run("go mod init joern.io/temp", prjDir) match
           case Success(_) =>
             mod.dependencies.foreach(dependency => {
-              val cmd = s"go get ${dependency.module}@${dependency.version}"
+              val dependencyStr = s"${dependency.module}@${dependency.version}"
+              val cmd           = s"go get $dependencyStr"
               ExternalCommand.run(cmd, prjDir) match
-                case Success(_) => print(". ")
+                case Success(_) =>
+                  print(". ")
+                  processDependency(dependencyStr)
                 case Failure(f) =>
                   logger.error(s"\t- command '${cmd}' failed", f)
             })
           case Failure(f) =>
             logger.error("\t- command 'go mod init joern.io/temp' failed", f)
       })
   }
+
+  private def processDependency(dependencyStr: String): Unit = {
+    val gopath             = Try(sys.env("GOPATH")).getOrElse(Seq(os.home, "go").mkString(JFile.separator))
+    val dependencyLocation = (Seq(gopath, "pkg", "mod") ++ dependencyStr.split("/")).mkString(JFile.separator)
+    File.usingTemporaryDirectory("godep") { astLocation =>
+      val config       = Config().withInputPath(dependencyLocation)
+      val astGenResult = new AstGenRunner(config).execute(astLocation)
+      val goMod = new GoModHelper(
+        Some(config),
+        astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x))
+      )
+      new MethodAndTypeCacheBuilderPass(astGenResult.parsedFiles, config, goMod).process()
+    }
+  }
 }
diff --git a/...rontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala b/...rontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala
@@ -6,14 +6,14 @@ import io.shiftleft.codepropertygraph.generated.Operators
 import io.shiftleft.semanticcpg.language.*
 
 class DownloadDependencyTest extends GoCodeToCpgSuite {
-  "Simple use case of thrid party dependency download use case" ignore {
+  "Simple use case of thrid party dependency download use case" should {
     val config = Config().withFetchDependencies(true)
     val cpg = code(
       """
         |module joern.io/sample
         |go 1.18
         |require (
-        |	github.com/google/uuid v1.3.1 // indirect
+        |	github.com/google/uuid v1.3.1
         |)
         |""".stripMargin,
       "go.mod"
@@ -32,6 +32,33 @@ class DownloadDependencyTest extends GoCodeToCpgSuite {
     }
   }
 
+  // NOTE: This test is ignored because of high memory usage on windows
+  "Download dependency example with different package and namespace name" ignore {
+    val config = Config().withFetchDependencies(true)
+    val cpg = code(
+      """
+        |module joern.io/sample
+        |go 1.18
+        |require (
+        |	github.com/aerospike/aerospike-client-go/v6 v6.14.0
+        |)
+        |""".stripMargin,
+      "go.mod"
+    ).moreCode("""
+          |package main
+          |import "github.com/aerospike/aerospike-client-go/v6"
+          |func main()  {
+          |  client, err := aerospike.NewClient("localhost", 3000)
+          |}
+          |""".stripMargin)
+      .withConfig(config)
+
+    "Check CALL Node" in {
+      val List(x) = cpg.call("NewClient").l
+      x.typeFullName shouldBe "*github.com/aerospike/aerospike-client-go/v6.Client"
+    }
+  }
+
   "unresolved dependency tests one" should {
     val cpg = code(
       """