From ee6c91d4765ad4f25e9090df8015e5cf956c58e5 Mon Sep 17 00:00:00 2001 From: crt-31 <126034653+crt-31@users.noreply.github.com> Date: Mon, 2 Oct 2023 03:47:23 -0700 Subject: [PATCH] Add SemanticDB support - cont. (#1508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add SemanticDB support * Semanticdb output works * fixup to make work with scalaopt phase * fixed tests * added tests * fix * Add Scalac3 semanticdb options * Fix scala3 failures * fix formating * fix scala3 support; add scala3 test * clean up dead code * bump, to retrigger PR checks * bump scalameta; refactor version string * more scala 3.1, refactor version string * update hashes * bump scalafmt * Revert "bump, to retrigger PR checks" This reverts commit 110442d8d00d168d9f39e2672fe181b6833121ee. * Revert "Revert "bump, to retrigger PR checks"" This reverts commit 39b10b291a2dd2337f753f6149eca2c1a6c242c3. * Revert "bump scalafmt" This reverts commit cb07f1593ee52606f0eec025c96e0ccdbc685413. * Revert "update hashes" This reverts commit 993aa30d32552146b4636cf579ec7f91252e045d. * Revert "more scala 3.1, refactor version string" This reverts commit 166c1f0e805043af11c07fb2315746d5af9f0664. * Revert "bump scalameta; refactor version string" This reverts commit 8fcbbc9afb04991a713f931ced843f01095a433e. * Revert "Bump jmh version: 1.20 -> 1.36 (#1466)" This reverts commit 1e4df56559ca8b68eb9a173cdf048a3c81bead09. * Bump jmh version: 1.20 -> 1.36 (#1466) Also bump versions of its dependencies: * org.ow2.asm:asm: 6.1.1 -> 9.0 * net.sf.jopt-simple:jopt-simple: 4.6 -> 5.0.4 * add to predefined outputs * revert predefined output * include semanticdb in class jar * fix merge * fix tests * lint * move to phases * fix bug * force rebuild * lint fix * Semanticdb to use phase, and other improvements * semanticdb: addressed pr review comments - fixed semanticdb example - added prefix '_' to semanticdb output path - renamed certain vars to not prepend with "scala_" - changed depsprovider var name to be plural instead of singular --------- Co-authored-by: Ólafur Páll Geirsson Co-authored-by: Matt Peterson Co-authored-by: Aish Co-authored-by: Aish Fenton Co-authored-by: Dmitry Komanov --- README.md | 1 + docs/scala_toolchain.md | 24 ++- examples/semanticdb/BUILD | 28 ++++ examples/semanticdb/Foo.scala | 3 + examples/semanticdb/Main.scala | 7 + examples/semanticdb/README.md | 13 ++ examples/semanticdb/WORKSPACE | 45 ++++++ examples/semanticdb/aspect.bzl | 24 +++ scala/BUILD | 9 ++ scala/private/macros/scala_repositories.bzl | 1 + .../private/macros/setup_scala_toolchain.bzl | 33 +++- scala/private/phases/phase_compile.bzl | 19 ++- scala/private/phases/phase_semanticdb.bzl | 78 +++++++++ scala/private/phases/phases.bzl | 4 + scala/private/rule_impls.bzl | 3 +- scala/private/rules/scala_binary.bzl | 2 + scala/private/rules/scala_junit_test.bzl | 2 + scala/private/rules/scala_library.bzl | 4 + scala/private/rules/scala_repl.bzl | 2 + scala/private/rules/scala_test.bzl | 2 + scala/private/toolchain_deps/BUILD | 6 + scala/scala_toolchain.bzl | 28 +++- scala/semanticdb_provider.bzl | 8 + scala_proto/private/scala_proto_aspect.bzl | 1 + test/semanticdb/A.scala | 1 + test/semanticdb/B.scala | 1 + test/semanticdb/BUILD | 51 ++++++ test/semanticdb/rules.bzl | 35 ++++ .../semantic_provider_vars.sh.template | 5 + test/shell/test_examples.sh | 11 +- test/shell/test_helper.sh | 9 ++ test/shell/test_semanticdb.sh | 153 ++++++++++++++++++ test_rules_scala.sh | 1 + third_party/repositories/scala_2_11.bzl | 7 + third_party/repositories/scala_2_12.bzl | 7 + third_party/repositories/scala_2_13.bzl | 7 + twitter_scrooge/twitter_scrooge.bzl | 1 + 37 files changed, 616 insertions(+), 20 deletions(-) create mode 100644 examples/semanticdb/BUILD create mode 100644 examples/semanticdb/Foo.scala create mode 100644 examples/semanticdb/Main.scala create mode 100644 examples/semanticdb/README.md create mode 100644 examples/semanticdb/WORKSPACE create mode 100644 examples/semanticdb/aspect.bzl create mode 100644 scala/private/phases/phase_semanticdb.bzl create mode 100644 scala/semanticdb_provider.bzl create mode 100644 test/semanticdb/A.scala create mode 100644 test/semanticdb/B.scala create mode 100644 test/semanticdb/BUILD create mode 100644 test/semanticdb/rules.bzl create mode 100644 test/semanticdb/semantic_provider_vars.sh.template create mode 100755 test/shell/test_semanticdb.sh diff --git a/README.md b/README.md index 7cafdc4f4..d6047a952 100644 --- a/README.md +++ b/README.md @@ -283,3 +283,4 @@ Here's a (non-exhaustive) list of companies that use `rules_scala` in production * [Twitter](https://twitter.com/) * [VSCO](https://vsco.co) * [Wix](https://www.wix.com/) + diff --git a/docs/scala_toolchain.md b/docs/scala_toolchain.md index e7eb74f3e..924e2bfed 100644 --- a/docs/scala_toolchain.md +++ b/docs/scala_toolchain.md @@ -79,7 +79,8 @@ scala_register_toolchains() Allows to configure dependencies lists by configuring DepInfo provider targets. Currently supported dep ids: scala_compile_classpath, scala_library_classpath, scala_macro_classpath, scala_xml, - parser_combinators. + parser_combinators, + semanticdb

@@ -143,6 +144,27 @@ scala_register_toolchains() List of target prefixes included for unused deps analysis. Exclude patetrns with '-'

+ + + enable_semanticdb + +

Boolean; optional (default False)

+

+ Enables semanticdb output. +

+ + + + semanticdb_bundle_in_jar + +

Boolean; optional (default False)

+

+ When False, *.semanticdb files are added to the filesystem in a directory. +

+

+ When True, *.semanticdb files will be bundled inside the jar file. +

+ diff --git a/examples/semanticdb/BUILD b/examples/semanticdb/BUILD new file mode 100644 index 000000000..281107111 --- /dev/null +++ b/examples/semanticdb/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_scala//scala:scala_toolchain.bzl", "scala_toolchain") +load("@io_bazel_rules_scala//scala:scala.bzl", "scala_binary", "scala_library") + +scala_toolchain( + name = "semanticdb_toolchain_impl", + enable_semanticdb = True, + semanticdb_bundle_in_jar = False, + visibility = ["//visibility:public"], +) + +toolchain( + name = "semanticdb_toolchain", + toolchain = "semanticdb_toolchain_impl", + toolchain_type = "@io_bazel_rules_scala//scala:toolchain_type", + visibility = ["//visibility:public"], +) + +scala_library( + name = "hello_lib", + srcs = ["Foo.scala"], +) + +scala_binary( + name = "hello", + srcs = ["Main.scala"], + main_class = "main", + deps = [":hello_lib"], +) diff --git a/examples/semanticdb/Foo.scala b/examples/semanticdb/Foo.scala new file mode 100644 index 000000000..222e82e0e --- /dev/null +++ b/examples/semanticdb/Foo.scala @@ -0,0 +1,3 @@ +class Foo(){ + def sayHello: String = "Hello!!" +} diff --git a/examples/semanticdb/Main.scala b/examples/semanticdb/Main.scala new file mode 100644 index 000000000..0ea328a25 --- /dev/null +++ b/examples/semanticdb/Main.scala @@ -0,0 +1,7 @@ + +object main{ + def main(args: Array[String]): Unit = { + val foo = new Foo() + println(foo.sayHello) + } +} \ No newline at end of file diff --git a/examples/semanticdb/README.md b/examples/semanticdb/README.md new file mode 100644 index 000000000..8aa4301e7 --- /dev/null +++ b/examples/semanticdb/README.md @@ -0,0 +1,13 @@ +# scala_rules Semanticdb example + +This example demonstrates using an aspect to access the semanticdb info for one or more targets. In this example, an aspect is used to generate a json file that contains the semanticdb info that could be consumed by a consumer such as an IDE. + +In this example, note that a scala_toolchain with enable_semanticdb=True is setup in the BUILD file. + +This command can be used to run the aspect (and not run the full build) + +``` +bazel build //... --aspects aspect.bzl%semanticdb_info_aspect --output_groups=json_output_file +``` + +The semanticdb_info.json file will be created for each target, and contains the semanticdb info for the target. \ No newline at end of file diff --git a/examples/semanticdb/WORKSPACE b/examples/semanticdb/WORKSPACE new file mode 100644 index 000000000..1d1fde5e9 --- /dev/null +++ b/examples/semanticdb/WORKSPACE @@ -0,0 +1,45 @@ +workspace(name = "specs2_junit_repositories") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +skylib_version = "1.4.1" + +http_archive( + name = "bazel_skylib", + sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7", + type = "tar.gz", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version), + "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version), + ], +) + +local_repository( + name = "io_bazel_rules_scala", + path = "../..", +) + +load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config") + +scala_config(scala_version = "2.13.6") + +load( + "@io_bazel_rules_scala//scala:scala.bzl", + "rules_scala_setup", + "rules_scala_toolchain_deps_repositories", +) + +rules_scala_setup() + +rules_scala_toolchain_deps_repositories(fetch_sources = True) + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + +rules_proto_dependencies() + +rules_proto_toolchains() + +#Register and use the custom toolchain that has semanticdb enabled +register_toolchains( + "//:semanticdb_toolchain", +) diff --git a/examples/semanticdb/aspect.bzl b/examples/semanticdb/aspect.bzl new file mode 100644 index 000000000..a03807ecd --- /dev/null +++ b/examples/semanticdb/aspect.bzl @@ -0,0 +1,24 @@ +#This aspect is an example of exposing semanticdb information for each target into a json file. +# An IDE could use a json file like this to consume the semanticdb data for each target. + +load("@io_bazel_rules_scala//scala:semanticdb_provider.bzl", "SemanticdbInfo") + +def semanticdb_info_aspect_impl(target, ctx): + if SemanticdbInfo in target: + output_struct = struct( + target_label = str(target.label), + semanticdb_target_root = target[SemanticdbInfo].target_root, + semanticdb_pluginjar = target[SemanticdbInfo].plugin_jar, + ) + + json_output_file = ctx.actions.declare_file("%s_semanticdb_info.json" % target.label.name) + ctx.actions.write(json_output_file, json.encode_indent(output_struct)) + + return [OutputGroupInfo(json_output_file = depset([json_output_file]))] + return [] + +semanticdb_info_aspect = aspect( + implementation = semanticdb_info_aspect_impl, + attr_aspects = ["deps"], + toolchains = ["@io_bazel_rules_scala//scala:toolchain_type"], +) diff --git a/scala/BUILD b/scala/BUILD index d83fc318e..82f1180f4 100644 --- a/scala/BUILD +++ b/scala/BUILD @@ -34,6 +34,8 @@ _PARSER_COMBINATORS_DEPS = ["@io_bazel_rules_scala_scala_parser_combinators"] _SCALA_XML_DEPS = ["@io_bazel_rules_scala_scala_xml"] +_SEMANTICDB_DEPS = ["@org_scalameta_semanticdb_scalac"] if SCALA_MAJOR_VERSION.startswith("2") else [] + setup_scala_toolchain( name = "default_toolchain", scala_compile_classpath = _SCALA_COMPILE_CLASSPATH_DEPS, @@ -108,3 +110,10 @@ declare_deps_provider( visibility = ["//visibility:public"], deps = _PARSER_COMBINATORS_DEPS, ) + +declare_deps_provider( + name = "semanticdb_provider", + deps_id = "semanticdb", + visibility = ["//visibility:public"], + deps = _SEMANTICDB_DEPS, +) diff --git a/scala/private/macros/scala_repositories.bzl b/scala/private/macros/scala_repositories.bzl index 4f7bb00ec..e35655c78 100644 --- a/scala/private/macros/scala_repositories.bzl +++ b/scala/private/macros/scala_repositories.bzl @@ -128,6 +128,7 @@ ARTIFACT_IDS = [ "io_bazel_rules_scala_scala_reflect", "io_bazel_rules_scala_scala_xml", "io_bazel_rules_scala_scala_parser_combinators", + "org_scalameta_semanticdb_scalac", ] if SCALA_MAJOR_VERSION.startswith("2") else [ "io_bazel_rules_scala_scala_library", "io_bazel_rules_scala_scala_compiler", diff --git a/scala/private/macros/setup_scala_toolchain.bzl b/scala/private/macros/setup_scala_toolchain.bzl index 93f2dac80..f8fa55d7d 100644 --- a/scala/private/macros/setup_scala_toolchain.bzl +++ b/scala/private/macros/setup_scala_toolchain.bzl @@ -8,6 +8,8 @@ def setup_scala_toolchain( scala_macro_classpath, scala_xml_deps = None, parser_combinators_deps = None, + semanticdb_deps = None, + enable_semanticdb = False, visibility = ["//visibility:public"], **kwargs): scala_xml_provider = "%s_scala_xml_provider" % name @@ -15,6 +17,7 @@ def setup_scala_toolchain( scala_compile_classpath_provider = "%s_scala_compile_classpath_provider" % name scala_library_classpath_provider = "%s_scala_library_classpath_provider" % name scala_macro_classpath_provider = "%s_scala_macro_classpath_provider" % name + semanticdb_deps_provider = "%s_semanticdb_deps_provider" % name declare_deps_provider( name = scala_compile_classpath_provider, @@ -57,15 +60,31 @@ def setup_scala_toolchain( else: parser_combinators_provider = "@io_bazel_rules_scala//scala:parser_combinators_provider" + dep_providers = [ + scala_xml_provider, + parser_combinators_provider, + scala_compile_classpath_provider, + scala_library_classpath_provider, + scala_macro_classpath_provider, + ] + + if enable_semanticdb == True: + if semanticdb_deps != None: + declare_deps_provider( + name = semanticdb_deps_provider, + deps_id = "semanticdb", + deps = [semanticdb_deps], + visibility = visibility, + ) + + dep_providers.append(semanticdb_deps_provider) + else: + dep_providers.append("@io_bazel_rules_scala//scala:semanticdb_provider") + scala_toolchain( name = "%s_impl" % name, - dep_providers = [ - scala_xml_provider, - parser_combinators_provider, - scala_compile_classpath_provider, - scala_library_classpath_provider, - scala_macro_classpath_provider, - ], + dep_providers = dep_providers, + enable_semanticdb = enable_semanticdb, visibility = visibility, **kwargs ) diff --git a/scala/private/phases/phase_compile.bzl b/scala/private/phases/phase_compile.bzl index 403118966..61e8151cb 100644 --- a/scala/private/phases/phase_compile.bzl +++ b/scala/private/phases/phase_compile.bzl @@ -129,10 +129,18 @@ def _phase_compile( jars2labels = p.collect_jars.jars2labels.jars_to_labels deps_providers = p.collect_jars.deps_providers default_classpath = p.scalac_provider.default_classpath + plugins = ctx.attr.plugins + additional_outputs = [] + scalacopts = p.scalacopts + + if (hasattr(p, "semanticdb")): + scalacopts += p.semanticdb.scalacopts + plugins = plugins + p.semanticdb.plugin + additional_outputs += p.semanticdb.outputs out = _compile_or_empty( ctx, - p.scalacopts, + scalacopts, manifest, jars, srcjars, @@ -144,6 +152,8 @@ def _phase_compile( deps_providers, default_classpath, unused_dependency_checker_ignored_targets, + plugins, + additional_outputs, ) # TODO: simplify the return values and use provider @@ -169,7 +179,9 @@ def _compile_or_empty( dependency_info, deps_providers, default_classpath, - unused_dependency_checker_ignored_targets): + unused_dependency_checker_ignored_targets, + plugins, + additional_outputs): # We assume that if a srcjar is present, it is not empty if len(ctx.files.srcs) + len(srcjars.to_list()) == 0: _build_nosrc_jar(ctx) @@ -200,7 +212,7 @@ def _compile_or_empty( jars, all_srcjars, transitive_compile_jars, - ctx.attr.plugins, + plugins, ctx.attr.resource_strip_prefix, ctx.files.resources, ctx.files.resource_jars, @@ -212,6 +224,7 @@ def _compile_or_empty( ctx.executable._scalac, dependency_info, unused_dependency_checker_ignored_targets, + additional_outputs, ) # build ijar if needed diff --git a/scala/private/phases/phase_semanticdb.bzl b/scala/private/phases/phase_semanticdb.bzl new file mode 100644 index 000000000..ed7a2deba --- /dev/null +++ b/scala/private/phases/phase_semanticdb.bzl @@ -0,0 +1,78 @@ +load( + "@io_bazel_rules_scala//scala/private/toolchain_deps:toolchain_deps.bzl", + "find_deps_info_on", +) +load("@io_bazel_rules_scala//scala:semanticdb_provider.bzl", "SemanticdbInfo") +load("@io_bazel_rules_scala_config//:config.bzl", "SCALA_MAJOR_VERSION") +load("@bazel_skylib//lib:paths.bzl", "paths") + +def phase_semanticdb(ctx, p): + #semanticdb_bundle_in_jar feature: enables bundling the semanticdb files within the output jar. + + #Scala 2: Uses the semanticdb compiler plugin. Will output semanticdb files into the specified 'targetroot' which defaults to be under the '_scalac/classes' dir. When targetroot is under the _scalac/classes dir scalac bundles the *.semanticdb files into the jar. + + #Scala3: Semanticdb is built into scalac. Currently, if semanticdb-target is used, the semanticdb files are written and not bundled, otherwise, the semanticdb files are not written as files and only available inside the jar. + + toolchain = ctx.toolchains["@io_bazel_rules_scala//scala:toolchain_type"] + toolchain_type_label = "@io_bazel_rules_scala//scala:toolchain_type" + + if toolchain.enable_semanticdb == True: + scalacopts = [] + semanticdb_deps = [] + output_files = [] + plugin_jar_path = "" + + target_output_path = paths.dirname(ctx.outputs.jar.path) + + semanticdb_intpath = "_scalac/" + ctx.label.name + "/classes" if toolchain.semanticdb_bundle_in_jar == True else "_semanticdb/" + ctx.label.name + + semanticdb_target_root = "%s/%s" % (target_output_path, semanticdb_intpath) + + #declare all the semanticdb files + if (not toolchain.semanticdb_bundle_in_jar): + semanticdb_outpath = "META-INF/semanticdb" + + for currSrc in ctx.files.srcs: + if currSrc.extension == "scala": + outputfilename = "%s/%s/%s.semanticdb" % (semanticdb_intpath, semanticdb_outpath, currSrc.path) + output_files.append(ctx.actions.declare_file(outputfilename)) + + if SCALA_MAJOR_VERSION.startswith("2"): + semanticdb_deps = find_deps_info_on(ctx, toolchain_type_label, "semanticdb").deps + + if len(semanticdb_deps) == 0: + fail("semanticdb enabled, but semanticdb plugin jar not specified in scala_toolchain") + if len(semanticdb_deps) != 1: + fail("more than one semanticdb plugin jar was specified in scala_toolchain. Expect a single semanticdb plugin jar") + + plugin_jar_path = semanticdb_deps[0][JavaInfo].java_outputs[0].class_jar.path + + scalacopts += [ + #note: Xplugin parameter handled in scalacworker, + "-Yrangepos", + "-P:semanticdb:failures:error", + "-P:semanticdb:targetroot:" + semanticdb_target_root, + ] + else: + #Note: In Scala3, semanticdb is built-in to compiler, so no need for plugin + + scalacopts.append("-Ysemanticdb") + + if toolchain.semanticdb_bundle_in_jar == False: + scalacopts.append("-semanticdb-target:" + semanticdb_target_root) + + semanticdb_provider = SemanticdbInfo( + semanticdb_enabled = True, + target_root = None if toolchain.semanticdb_bundle_in_jar else semanticdb_target_root, + is_bundled_in_jar = toolchain.semanticdb_bundle_in_jar, + plugin_jar = plugin_jar_path, + ) + + return struct( + scalacopts = scalacopts, + plugin = semanticdb_deps, + outputs = output_files, + external_providers = {"SemanticdbInfo": semanticdb_provider}, + ) + + return None diff --git a/scala/private/phases/phases.bzl b/scala/private/phases/phases.bzl index fbe7bd674..a497f4d07 100644 --- a/scala/private/phases/phases.bzl +++ b/scala/private/phases/phases.bzl @@ -66,6 +66,7 @@ load("@io_bazel_rules_scala//scala/private:phases/phase_scalacopts.bzl", _phase_ load("@io_bazel_rules_scala//scala/private:phases/phase_coverage_runfiles.bzl", _phase_coverage_runfiles = "phase_coverage_runfiles") load("@io_bazel_rules_scala//scala/private:phases/phase_scalafmt.bzl", _phase_scalafmt = "phase_scalafmt") load("@io_bazel_rules_scala//scala/private:phases/phase_test_environment.bzl", _phase_test_environment = "phase_test_environment") +load("@io_bazel_rules_scala//scala/private:phases/phase_semanticdb.bzl", _phase_semanticdb = "phase_semanticdb") # API run_phases = _run_phases @@ -146,3 +147,6 @@ phase_test_environment = _phase_test_environment # scalafmt phase_scalafmt = _phase_scalafmt + +# semanticdb +phase_semanticdb = _phase_semanticdb diff --git a/scala/private/rule_impls.bzl b/scala/private/rule_impls.bzl index 7044fc3cb..3af8cab7a 100644 --- a/scala/private/rule_impls.bzl +++ b/scala/private/rule_impls.bzl @@ -56,6 +56,7 @@ def compile_scala( scalac, dependency_info, unused_dependency_checker_ignored_targets, + additional_outputs, stamp_target_label = None): # look for any plugins: input_plugins = plugins @@ -112,7 +113,7 @@ def compile_scala( if dependency_info.unused_deps_mode != "off" or dependency_info.strict_deps_mode != "off": args.add_all("--UnusedDepsIgnoredTargets", unused_dependency_checker_ignored_targets) - outs = [output, statsfile, diagnosticsfile, scaladepsfile] + outs = [output, statsfile, diagnosticsfile, scaladepsfile] + additional_outputs ins = depset( direct = [manifest] + sources + classpath_resources + resources + resource_jars, diff --git a/scala/private/rules/scala_binary.bzl b/scala/private/rules/scala_binary.bzl index 7604461fb..a36f7606b 100644 --- a/scala/private/rules/scala_binary.bzl +++ b/scala/private/rules/scala_binary.bzl @@ -23,6 +23,7 @@ load( "phase_runfiles_common", "phase_scalac_provider", "phase_scalacopts", + "phase_semanticdb", "phase_write_executable_common", "phase_write_manifest", "run_phases", @@ -40,6 +41,7 @@ def _scala_binary_impl(ctx): ("java_wrapper", phase_java_wrapper_common), ("declare_executable", phase_declare_executable), ("scalacopts", phase_scalacopts), + ("semanticdb", phase_semanticdb), # no need to build an ijar for an executable ("compile", phase_compile_binary), ("coverage", phase_coverage_common), diff --git a/scala/private/rules/scala_junit_test.bzl b/scala/private/rules/scala_junit_test.bzl index 58567464e..c45d4592d 100644 --- a/scala/private/rules/scala_junit_test.bzl +++ b/scala/private/rules/scala_junit_test.bzl @@ -24,6 +24,7 @@ load( "phase_runfiles_common", "phase_scalac_provider", "phase_scalacopts", + "phase_semanticdb", "phase_test_environment", "phase_write_executable_junit_test", "phase_write_manifest", @@ -46,6 +47,7 @@ def _scala_junit_test_impl(ctx): ("java_wrapper", phase_java_wrapper_common), ("declare_executable", phase_declare_executable), ("scalacopts", phase_scalacopts), + ("semanticdb", phase_semanticdb), # no need to build an ijar for an executable ("compile", phase_compile_junit_test), ("coverage", phase_coverage_common), diff --git a/scala/private/rules/scala_library.bzl b/scala/private/rules/scala_library.bzl index 1a4e6887d..4a7e84a8c 100644 --- a/scala/private/rules/scala_library.bzl +++ b/scala/private/rules/scala_library.bzl @@ -34,6 +34,7 @@ load( "phase_runfiles_library", "phase_scalac_provider", "phase_scalacopts", + "phase_semanticdb", "phase_write_manifest", "run_phases", ) @@ -66,6 +67,7 @@ def _scala_library_impl(ctx): ("dependency", phase_dependency_common), ("collect_jars", phase_collect_jars_common), ("scalacopts", phase_scalacopts), + ("semanticdb", phase_semanticdb), ("compile", phase_compile_library), ("coverage", phase_coverage_library), ("merge_jars", phase_merge_jars), @@ -149,6 +151,7 @@ def _scala_library_for_plugin_bootstrapping_impl(ctx): ("dependency", phase_dependency_library_for_plugin_bootstrapping), ("collect_jars", phase_collect_jars_common), ("scalacopts", phase_scalacopts), + #("semanticdb", phase_semanticdb), noneed for semanticdb in bootstrap ("compile", phase_compile_library_for_plugin_bootstrapping), ("merge_jars", phase_merge_jars), ("runfiles", phase_runfiles_library), @@ -219,6 +222,7 @@ def _scala_macro_library_impl(ctx): ("dependency", phase_dependency_common), ("collect_jars", phase_collect_jars_macro_library), ("scalacopts", phase_scalacopts), + ("semanticdb", phase_semanticdb), ("compile", phase_compile_macro_library), ("coverage", phase_coverage_common), ("merge_jars", phase_merge_jars), diff --git a/scala/private/rules/scala_repl.bzl b/scala/private/rules/scala_repl.bzl index 9b4695653..67e042742 100644 --- a/scala/private/rules/scala_repl.bzl +++ b/scala/private/rules/scala_repl.bzl @@ -23,6 +23,7 @@ load( "phase_runfiles_common", "phase_scalac_provider", "phase_scalacopts", + "phase_semanticdb", "phase_write_executable_repl", "phase_write_manifest", "run_phases", @@ -41,6 +42,7 @@ def _scala_repl_impl(ctx): ("java_wrapper", phase_java_wrapper_repl), ("declare_executable", phase_declare_executable), ("scalacopts", phase_scalacopts), + ("semanticdb", phase_semanticdb), # no need to build an ijar for an executable ("compile", phase_compile_repl), ("coverage", phase_coverage_common), diff --git a/scala/private/rules/scala_test.bzl b/scala/private/rules/scala_test.bzl index 4f1948705..4f77f88af 100644 --- a/scala/private/rules/scala_test.bzl +++ b/scala/private/rules/scala_test.bzl @@ -24,6 +24,7 @@ load( "phase_runfiles_scalatest", "phase_scalac_provider", "phase_scalacopts", + "phase_semanticdb", "phase_test_environment", "phase_write_executable_scalatest", "phase_write_manifest", @@ -43,6 +44,7 @@ def _scala_test_impl(ctx): ("declare_executable", phase_declare_executable), ("scalacopts", phase_scalacopts), # no need to build an ijar for an executable + ("semanticdb", phase_semanticdb), ("compile", phase_compile_scalatest), ("coverage", phase_coverage_common), ("merge_jars", phase_merge_jars), diff --git a/scala/private/toolchain_deps/BUILD b/scala/private/toolchain_deps/BUILD index a5bc109d5..9cca1b572 100644 --- a/scala/private/toolchain_deps/BUILD +++ b/scala/private/toolchain_deps/BUILD @@ -23,3 +23,9 @@ common_toolchain_deps( deps_id = "parser_combinators", visibility = ["//visibility:public"], ) + +common_toolchain_deps( + name = "semanticdb", + deps_id = "semanticdb", + visibility = ["//visibility:public"], +) diff --git a/scala/scala_toolchain.bzl b/scala/scala_toolchain.bzl index 5386ec579..2b7accd28 100644 --- a/scala/scala_toolchain.bzl +++ b/scala/scala_toolchain.bzl @@ -5,6 +5,7 @@ load( load( "@io_bazel_rules_scala_config//:config.bzl", "ENABLE_COMPILER_DEPENDENCY_TRACKING", + "SCALA_MAJOR_VERSION", ) def _compute_strict_deps_mode(input_strict_deps_mode, dependency_mode): @@ -97,22 +98,30 @@ def _scala_toolchain_impl(ctx): enable_diagnostics_report = enable_diagnostics_report, jacocorunner = ctx.attr.jacocorunner, enable_stats_file = enable_stats_file, + enable_semanticdb = ctx.attr.enable_semanticdb, + semanticdb_bundle_in_jar = ctx.attr.semanticdb_bundle_in_jar, use_argument_file_in_runner = ctx.attr.use_argument_file_in_runner, ) return [toolchain] +def _default_dep_providers(): + dep_providers = [ + "@io_bazel_rules_scala//scala:scala_xml_provider", + "@io_bazel_rules_scala//scala:parser_combinators_provider", + "@io_bazel_rules_scala//scala:scala_compile_classpath_provider", + "@io_bazel_rules_scala//scala:scala_library_classpath_provider", + "@io_bazel_rules_scala//scala:scala_macro_classpath_provider", + ] + if SCALA_MAJOR_VERSION.startswith("2"): + dep_providers.append("@io_bazel_rules_scala//scala:semanticdb_provider") + return dep_providers + scala_toolchain = rule( _scala_toolchain_impl, attrs = { "scalacopts": attr.string_list(), "dep_providers": attr.label_list( - default = [ - "@io_bazel_rules_scala//scala:scala_xml_provider", - "@io_bazel_rules_scala//scala:parser_combinators_provider", - "@io_bazel_rules_scala//scala:scala_compile_classpath_provider", - "@io_bazel_rules_scala//scala:scala_library_classpath_provider", - "@io_bazel_rules_scala//scala:scala_macro_classpath_provider", - ], + default = _default_dep_providers(), providers = [_DepsInfo], ), "dependency_mode": attr.string( @@ -155,6 +164,11 @@ scala_toolchain = rule( default = True, doc = "Enable writing of statsfile", ), + "enable_semanticdb": attr.bool( + default = False, + doc = "Enable SemanticDb", + ), + "semanticdb_bundle_in_jar": attr.bool(default = False, doc = "Option to bundle the semanticdb files inside the output jar file"), "use_argument_file_in_runner": attr.bool( default = False, doc = "Changes java binaries scripts (including tests) to use argument files and not classpath jars to improve performance, requires java > 8", diff --git a/scala/semanticdb_provider.bzl b/scala/semanticdb_provider.bzl new file mode 100644 index 000000000..6b1ecfc0b --- /dev/null +++ b/scala/semanticdb_provider.bzl @@ -0,0 +1,8 @@ +SemanticdbInfo = provider( + fields = { + "semanticdb_enabled": "boolean", + "target_root": "directory containing the semanticdb files (relative to execroot).", + "is_bundled_in_jar": "boolean: whether the semanticdb files are bundled inside the jar", + "plugin_jar": "path to semanticdb plugin jar (relative to execroot)", + }, +) diff --git a/scala_proto/private/scala_proto_aspect.bzl b/scala_proto/private/scala_proto_aspect.bzl index bd80d428c..9832a75f3 100644 --- a/scala_proto/private/scala_proto_aspect.bzl +++ b/scala_proto/private/scala_proto_aspect.bzl @@ -122,6 +122,7 @@ def _compile_sources(ctx, toolchain, proto, src_jars, deps, scalacopts, stamp_la dependency_info = legacy_unclear_dependency_info_for_protobuf_scrooge(ctx), unused_dependency_checker_ignored_targets = [], stamp_target_label = stamp_label, + additional_outputs = [], ) return JavaInfo( diff --git a/test/semanticdb/A.scala b/test/semanticdb/A.scala new file mode 100644 index 000000000..a869c2849 --- /dev/null +++ b/test/semanticdb/A.scala @@ -0,0 +1 @@ +class A {} diff --git a/test/semanticdb/B.scala b/test/semanticdb/B.scala new file mode 100644 index 000000000..3a43d09b4 --- /dev/null +++ b/test/semanticdb/B.scala @@ -0,0 +1 @@ +class B {} diff --git a/test/semanticdb/BUILD b/test/semanticdb/BUILD new file mode 100644 index 000000000..5cec87149 --- /dev/null +++ b/test/semanticdb/BUILD @@ -0,0 +1,51 @@ +load("//scala:scala_toolchain.bzl", "scala_toolchain") +load("//scala:scala.bzl", "scala_library") +load("//test/semanticdb:rules.bzl", "semanticdb_vars_script") + +scala_toolchain( + name = "semanticdb_bundle_toolchain_impl", + enable_semanticdb = True, + semanticdb_bundle_in_jar = True, + visibility = ["//visibility:public"], +) + +toolchain( + name = "semanticdb_bundle_toolchain", + toolchain = "semanticdb_bundle_toolchain_impl", + toolchain_type = "@io_bazel_rules_scala//scala:toolchain_type", + visibility = ["//visibility:public"], +) + +scala_toolchain( + name = "semanticdb_nobundle_toolchain_impl", + enable_semanticdb = True, + semanticdb_bundle_in_jar = False, + visibility = ["//visibility:public"], +) + +toolchain( + name = "semanticdb_nobundle_toolchain", + toolchain = "semanticdb_nobundle_toolchain_impl", + toolchain_type = "@io_bazel_rules_scala//scala:toolchain_type", + visibility = ["//visibility:public"], +) + +scala_library( + name = "all_lib", + srcs = glob(["**/*.scala"]), +) + +semanticdb_vars_script( + name = "semantic_provider_vars_all", + dep = "all_lib", +) + +scala_library( + name = "empty_lib", + srcs = [], +) + +semanticdb_vars_script( + name = "semantic_provider_vars_empty", + dep = "empty_lib", +) diff --git a/test/semanticdb/rules.bzl b/test/semanticdb/rules.bzl new file mode 100644 index 000000000..50ff80d2c --- /dev/null +++ b/test/semanticdb/rules.bzl @@ -0,0 +1,35 @@ +load("//scala:semanticdb_provider.bzl", "SemanticdbInfo") + +def semanticdb_vars_script_impl(ctx): + if SemanticdbInfo in ctx.attr.dep: + out_script = ctx.actions.declare_file("%s.sh" % ctx.label.name) + semanticdb_info = ctx.attr.dep[SemanticdbInfo] + + ctx.actions.expand_template( + output = out_script, + template = ctx.file._script, + substitutions = { + "%TARGETROOT%": "" if semanticdb_info.target_root == None else semanticdb_info.target_root, + "%ENABLED%": "1" if semanticdb_info.semanticdb_enabled else "0", + "%ISBUNDLED%": "1" if semanticdb_info.is_bundled_in_jar else "0", + "%PLUGINPATH%": semanticdb_info.plugin_jar, + }, + ) + return [ + DefaultInfo(files = depset( + [out_script], + transitive = [ctx.attr.dep[DefaultInfo].files], + )), + ] + return None + +semanticdb_vars_script = rule( + implementation = semanticdb_vars_script_impl, + attrs = { + "dep": attr.label(mandatory = True), + "_script": attr.label( + allow_single_file = True, + default = "semantic_provider_vars.sh.template", + ), + }, +) diff --git a/test/semanticdb/semantic_provider_vars.sh.template b/test/semanticdb/semantic_provider_vars.sh.template new file mode 100644 index 000000000..e8602c506 --- /dev/null +++ b/test/semanticdb/semantic_provider_vars.sh.template @@ -0,0 +1,5 @@ +#!/bin/bash +semanticdb_target_root="%TARGETROOT%" +semanticdb_enabled=%ENABLED% +semanticdb_is_bundled=%ISBUNDLED% +semanticdb_pluginjarpath="%PLUGINPATH%" \ No newline at end of file diff --git a/test/shell/test_examples.sh b/test/shell/test_examples.sh index 8730f5f17..d492ea437 100755 --- a/test/shell/test_examples.sh +++ b/test/shell/test_examples.sh @@ -28,9 +28,18 @@ function scala3_3_example() { (cd examples/scala3; bazel build --repo_env=SCALA_VERSION=3.3.0 //...) } +function semanticdb_example() { + set -e + ( cd examples/semanticdb; + bazel build //... --aspects aspect.bzl%semanticdb_info_aspect --output_groups=json_output_file; + bazel build //... + ) +} + $runner scalatest_repositories_example $runner specs2_junit_repositories_example $runner multi_framework_toolchain_example +$runner semanticdb_example $runner scala3_1_example $runner scala3_2_example -$runner scala3_3_example +$runner scala3_3_example \ No newline at end of file diff --git a/test/shell/test_helper.sh b/test/shell/test_helper.sh index fe10dbd8c..8bd3c49db 100755 --- a/test/shell/test_helper.sh +++ b/test/shell/test_helper.sh @@ -111,3 +111,12 @@ test_scala_library_expect_failure_on_missing_direct_deps() { test_expect_failure_or_warning_on_missing_direct_deps_with_expected_message "${expected_message}" $test_target "--extra_toolchains=//test/toolchains:high_level_transitive_deps_strict_deps_error" } + +jar_contains_files() { + for arg in "${@:2}" + do + if ! jar tf $1 | grep $arg; then + return 1 + fi + done +} \ No newline at end of file diff --git a/test/shell/test_semanticdb.sh b/test/shell/test_semanticdb.sh new file mode 100755 index 000000000..6ee5f095e --- /dev/null +++ b/test/shell/test_semanticdb.sh @@ -0,0 +1,153 @@ +# shellcheck source=./test_runner.sh + +dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +. "${dir}"/test_runner.sh +. "${dir}"/test_helper.sh +runner=$(get_test_runner "${1:-local}") + +FILES=("A.scala.semanticdb" "B.scala.semanticdb") + +jar_contains_files() { + for arg in "${@:2}" + do + if ! jar tf $1 | grep $arg; then + return 1 + fi + done +} + +test_produces_semanticdb(){ + set -e + + local scala_majver=$1 + local is_bundle=$2 + + + if [ $is_bundle -eq 1 ]; then + local toolchain="--extra_toolchains=//test/semanticdb:semanticdb_bundle_toolchain" + else + local toolchain="--extra_toolchains=//test/semanticdb:semanticdb_nobundle_toolchain" + fi + + if [ $scala_majver -eq 3 ]; then + local version_opt="--repo_env=SCALA_VERSION=3.3.0" + fi + + + bazel build //test/semanticdb:semantic_provider_vars_all ${toolchain} ${version_opt} + + #semantic_provider_vars.sh contains the SemanticdbInfo data + . $(bazel info bazel-bin)/test/semanticdb/semantic_provider_vars_all.sh + + #Check the Provider variables + if [ $semanticdb_enabled -ne 1 ]; then + echo "Error: SemanticdbInfo.semanticdb_enabled not equal to true" + exit 1 + fi + + + if [ $semanticdb_is_bundled -ne $is_bundle ]; then + echo "Error: SemanticdbInfo.is_bundled_in_jar is incorrect." + exit 1 + fi + + if [ $is_bundle -eq 0 ]; then + if [[ $semanticdb_target_root == "" ]]; then + echo "Error: SemanticdbInfo.target_root expected to have a value" + exit 1 + fi + else + if [[ $semanticdb_target_root != "" ]]; then + echo "Error: SemanticdbInfo.target_root expected to be empty string" + exit 1 + fi + fi + + if [[ $scala_majver == 3 ]] && [[ $semanticdb_pluginjarpath != "" ]]; then + echo "Error: SemanticdbInfo.pluginjarpath expected to be empty for scala 3" + exit 1 + fi + if [[ $scala_majver == 2 ]] && [[ $semanticdb_pluginjarpath == "" ]]; then + echo "Error: SemanticdbInfo.pluginjarpath expected to be set for scala 2" + exit 1 + fi + + if [ $is_bundle -eq 0 ]; then + + semanticdb_path="$(bazel info execution_root)/${semanticdb_target_root}/META-INF/semanticdb/test/semanticdb/" + + for arg in $FILES + do + if ! [ -f "${semanticdb_path}${arg}" ]; then + echo "Error: Expected Semanticdb file not found: ${semanticdb_path}${arg}" + exit 1; + + fi + done + fi + + local JAR="$(bazel info bazel-bin)/test/semanticdb/all_lib.jar" + + if [ $is_bundle -eq 0 ]; then + if jar_contains_files $JAR $FILES; then + echo "Error: SemanticDB output erroneously included in jar: $JAR" + exit 1 + fi + else + if ! jar_contains_files $JAR $FILES; then + echo "Error: SemanticDB output not included in jar: $JAR" + exit 1 + fi + fi +} + +test_empty_semanticdb(){ + #just make sure this special case of semanticdb with no source files builds fine + + set -e + + bazel build //test/semanticdb:semantic_provider_vars_empty --extra_toolchains=//test/semanticdb:semanticdb_nobundle_toolchain +} + +test_no_semanticdb() { + #verify no semanticdb files have been generated in the bin dir or bundled in the jar + + set -e + + local jar="$(bazel info bazel-bin)/test/semanticdb/all_lib.jar" + local targetout_path="$(bazel info bazel-bin)/test/semanticdb" + + rm -rf $targetout_path #clean out the output dir for clean slate + + #bazel clean + bazel build //test/semanticdb:all_lib + + #there should be no *.semanticdb files under the target's output dir + if [ $( find $targetout_path -type f -name *.semanticdb | wc -l ) -gt 0 ] ; then + echo "Error: Semanticdb files erroneously found in target output" + exit 1 + fi + + if jar_contains_files $jar "${FILES[@]}"; then + echo "Error: Semanticdb included in jar $JAR, but wasn't expected to be" + exit 1 + fi +} + +run_semanticdb_tests() { + local bundle=1; local nobundle=0 + local scala3=3; local scala2=2 + + $runner test_produces_semanticdb $scala2 $bundle + $runner test_produces_semanticdb $scala2 $nobundle + + $runner test_empty_semanticdb + + $runner test_produces_semanticdb $scala3 $bundle + $runner test_produces_semanticdb $scala3 $nobundle + + $runner test_no_semanticdb + +} + +run_semanticdb_tests diff --git a/test_rules_scala.sh b/test_rules_scala.sh index ac067dcd9..3d962dedb 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -56,3 +56,4 @@ $runner bazel build //test_statsfile:SimpleNoStatsFile_statsfile --extra_toolcha . "${test_dir}"/test_twitter_scrooge.sh . "${test_dir}"/test_inherited_environment.sh . "${test_dir}"/test_persistent_worker.sh +. "${test_dir}"/test_semanticdb.sh diff --git a/third_party/repositories/scala_2_11.bzl b/third_party/repositories/scala_2_11.bzl index a4e6e4ebb..4c90faa44 100644 --- a/third_party/repositories/scala_2_11.bzl +++ b/third_party/repositories/scala_2_11.bzl @@ -77,6 +77,13 @@ artifacts = { "@io_bazel_rules_scala_scala_library", ], }, + "org_scalameta_semanticdb_scalac": { + "artifact": "org.scalameta:semanticdb-scalac_%s:4.8.6" % scala_version, + "sha256": "1253abd1e8c8b3ec0720995d378846bbc439d168e7ea815fe91da6bef8b1de9d", + "deps": [ + "@io_bazel_rules_scala_scala_library", + ], + }, "org_scalameta_fastparse": { "artifact": "org.scalameta:fastparse_2.11:1.0.1", "sha256": "49ecc30a4b47efc0038099da0c97515cf8f754ea631ea9f9935b36ca7d41b733", diff --git a/third_party/repositories/scala_2_12.bzl b/third_party/repositories/scala_2_12.bzl index 4e186d75a..bf03c623a 100644 --- a/third_party/repositories/scala_2_12.bzl +++ b/third_party/repositories/scala_2_12.bzl @@ -77,6 +77,13 @@ artifacts = { "@io_bazel_rules_scala_scala_library", ], }, + "org_scalameta_semanticdb_scalac": { + "artifact": "org.scalameta:semanticdb-scalac_%s:4.8.4" % scala_version, + "sha256": "f31614cd13b6dc5c97804aa814b6f7ad4d67124290c08d0c9296b53e46d744e0", + "deps": [ + "@io_bazel_rules_scala_scala_library", + ], + }, "org_scalameta_fastparse": { "artifact": "org.scalameta:fastparse_2.12:1.0.1", "sha256": "387ced762e93915c5f87fed59d8453e404273f49f812d413405696ce20273aa5", diff --git a/third_party/repositories/scala_2_13.bzl b/third_party/repositories/scala_2_13.bzl index c89abeb8f..cbdd506d3 100644 --- a/third_party/repositories/scala_2_13.bzl +++ b/third_party/repositories/scala_2_13.bzl @@ -81,6 +81,13 @@ artifacts = { "@io_bazel_rules_scala_scala_library", ], }, + "org_scalameta_semanticdb_scalac": { + "artifact": "org.scalameta:semanticdb-scalac_%s:4.8.4" % scala_version, + "sha256": "acc1f667d4e98b42ba851309feb391cf301afeeeeddb8c172874a2184b18b057", + "deps": [ + "@io_bazel_rules_scala_scala_library", + ], + }, "org_scalameta_fastparse": { "artifact": "org.scalameta:fastparse_2.13:1.0.1", "sha256": "b43b99244d5b51948daf1467083b3850dc2727c604de98dc426dec14244fd18e", diff --git a/twitter_scrooge/twitter_scrooge.bzl b/twitter_scrooge/twitter_scrooge.bzl index becc721a7..6dc61850a 100644 --- a/twitter_scrooge/twitter_scrooge.bzl +++ b/twitter_scrooge/twitter_scrooge.bzl @@ -285,6 +285,7 @@ def _compile_generated_scala( scalac = ctx.executable._scalac, dependency_info = legacy_unclear_dependency_info_for_protobuf_scrooge(ctx), unused_dependency_checker_ignored_targets = [], + additional_outputs = [], ) return _create_java_info_provider(scrooge_jar, all_deps, output)