diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5c8655c76679..9d1081d295a5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -19,6 +19,10 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.18 - name: Install Bundler run: gem install bundler - name: Delete `.rustup` directory diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt index 0bce88822b43..07aa5ea2c8cd 100644 --- a/joern-cli/frontends/gosrc2cpg/build.sbt +++ b/joern-cli/frontends/gosrc2cpg/build.sbt @@ -11,6 +11,7 @@ libraryDependencies ++= Seq( "io.shiftleft" %% "codepropertygraph" % Versions.cpg, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, "com.lihaoyi" %% "upickle" % Versions.upickle, + "com.lihaoyi" %% "os-lib" % "0.9.1", "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.2", "com.typesafe" % "config" % "1.4.2", "com.michaelpollmeier" % "versionsort" % "1.0.11", diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala index 04ca5da8615f..96a7c626963a 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala @@ -1,12 +1,17 @@ package io.joern.gosrc2cpg.passes import better.files.File +import io.joern.gosrc2cpg.Config import io.joern.gosrc2cpg.model.GoModHelper +import io.joern.gosrc2cpg.parser.GoAstJsonParser +import io.joern.gosrc2cpg.utils.AstGenRunner import io.joern.gosrc2cpg.utils.AstGenRunner.getClass import io.joern.x2cpg.utils.ExternalCommand import org.slf4j.LoggerFactory -import scala.util.{Failure, Success} +import java.io.File as JFile +import java.nio.file.Paths +import scala.util.{Failure, Success, Try} class DownloadDependenciesPass(parentGoMod: GoModHelper) { private val logger = LoggerFactory.getLogger(getClass) @@ -23,9 +28,12 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper) { ExternalCommand.run("go mod init joern.io/temp", prjDir) match case Success(_) => mod.dependencies.foreach(dependency => { - val cmd = s"go get ${dependency.module}@${dependency.version}" + val dependencyStr = s"${dependency.module}@${dependency.version}" + val cmd = s"go get $dependencyStr" ExternalCommand.run(cmd, prjDir) match - case Success(_) => print(". ") + case Success(_) => + print(". ") + processDependency(dependencyStr) case Failure(f) => logger.error(s"\t- command '${cmd}' failed", f) }) @@ -33,4 +41,18 @@ class DownloadDependenciesPass(parentGoMod: GoModHelper) { logger.error("\t- command 'go mod init joern.io/temp' failed", f) }) } + + private def processDependency(dependencyStr: String): Unit = { + val gopath = Try(sys.env("GOPATH")).getOrElse(Seq(os.home, "go").mkString(JFile.separator)) + val dependencyLocation = (Seq(gopath, "pkg", "mod") ++ dependencyStr.split("/")).mkString(JFile.separator) + File.usingTemporaryDirectory("godep") { astLocation => + val config = Config().withInputPath(dependencyLocation) + val astGenResult = new AstGenRunner(config).execute(astLocation) + val goMod = new GoModHelper( + Some(config), + astGenResult.parsedModFile.flatMap(modFile => GoAstJsonParser.readModFile(Paths.get(modFile)).map(x => x)) + ) + new MethodAndTypeCacheBuilderPass(astGenResult.parsedFiles, config, goMod).process() + } + } } diff --git a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala index d889afd1b752..691a2152155e 100644 --- a/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala +++ b/joern-cli/frontends/gosrc2cpg/src/test/scala/io/joern/go2cpg/passes/ast/DownloadDependencyTest.scala @@ -6,14 +6,14 @@ import io.shiftleft.codepropertygraph.generated.Operators import io.shiftleft.semanticcpg.language.* class DownloadDependencyTest extends GoCodeToCpgSuite { - "Simple use case of thrid party dependency download use case" ignore { + "Simple use case of thrid party dependency download use case" should { val config = Config().withFetchDependencies(true) val cpg = code( """ |module joern.io/sample |go 1.18 |require ( - | github.com/google/uuid v1.3.1 // indirect + | github.com/google/uuid v1.3.1 |) |""".stripMargin, "go.mod" @@ -32,6 +32,33 @@ class DownloadDependencyTest extends GoCodeToCpgSuite { } } + // NOTE: This test is ignored because of high memory usage on windows + "Download dependency example with different package and namespace name" ignore { + val config = Config().withFetchDependencies(true) + val cpg = code( + """ + |module joern.io/sample + |go 1.18 + |require ( + | github.com/aerospike/aerospike-client-go/v6 v6.14.0 + |) + |""".stripMargin, + "go.mod" + ).moreCode(""" + |package main + |import "github.com/aerospike/aerospike-client-go/v6" + |func main() { + | client, err := aerospike.NewClient("localhost", 3000) + |} + |""".stripMargin) + .withConfig(config) + + "Check CALL Node" in { + val List(x) = cpg.call("NewClient").l + x.typeFullName shouldBe "*github.com/aerospike/aerospike-client-go/v6.Client" + } + } + "unresolved dependency tests one" should { val cpg = code( """