From 8be595ac05e44d32f98a12cddd1efbebec798e55 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Thu, 14 Mar 2024 16:22:34 +0000 Subject: [PATCH 01/10] Remove rules docker --- WORKSPACE | 67 ------------------- cli/BUILD | 34 +--------- cli/index_test.ts | 2 +- cloudbuild-test.json | 13 ---- cloudbuild/BUILD | 37 ---------- cloudbuild/Dockerfile | 13 ++++ cloudbuild/cloudbuild.yaml | 6 ++ .../bazel_test => cloudbuild/run_tests.sh | 3 + packages/@dataform/cli/BUILD | 1 - packages/index.bzl | 4 +- scripts/publish | 4 -- 11 files changed, 26 insertions(+), 158 deletions(-) delete mode 100644 cloudbuild-test.json delete mode 100644 cloudbuild/BUILD create mode 100644 cloudbuild/Dockerfile create mode 100644 cloudbuild/cloudbuild.yaml rename scripts/cloudbuild/bazel_test => cloudbuild/run_tests.sh (73%) diff --git a/WORKSPACE b/WORKSPACE index d1c6876b4..936f321f4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -96,65 +96,6 @@ load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") gazelle_dependencies() -# Docker base images. -http_archive( - name = "io_bazel_rules_docker", - sha256 = "b1e80761a8a8243d03ebca8845e9cc1ba6c82ce7c5179ce2b295cd36f7e394bf", - urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.25.0/rules_docker-v0.25.0.tar.gz"], -) - -load( - "@io_bazel_rules_docker//toolchains/docker:toolchain.bzl", - docker_toolchain_configure = "toolchain_configure", -) - -# Force Docker toolchain to use 'which' to find Docker binary. -docker_toolchain_configure( - name = "docker_config", -) - -load( - "@io_bazel_rules_docker//repositories:repositories.bzl", - container_repositories = "repositories", -) - -container_repositories() - -load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") - -container_deps() - -load("@io_bazel_rules_docker//repositories:py_repositories.bzl", "py_deps") - -py_deps() - -load( - "@io_bazel_rules_docker//container:container.bzl", - "container_pull", -) - -container_pull( - name = "nodejs_base", - # This digest is for tag "18.13.0". - digest = "sha256:d9061fd0205c20cd47f70bdc879a7a84fb472b822d3ad3158aeef40698d2ce36", - registry = "index.docker.io", - repository = "library/node", -) - -container_pull( - name = "nginx_base", - digest = "sha256:8c3cdb5acd050a5a46be0bb5637e23d192f4ef010b4fb6c5af40e45c5b7a0a71", - registry = "index.docker.io", - repository = "library/nginx", -) - -load( - "@io_bazel_rules_docker//nodejs:image.bzl", - _nodejs_image_repos = "repositories", -) - -_nodejs_image_repos() - # Gcloud SDK binaries. load("//tools/gcloud:repository_rules.bzl", "gcloud_sdk") @@ -185,11 +126,3 @@ go_repository( sum = "h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=", version = "v0.3.2", ) - -container_pull( - name = "debian_base", - # docker manifest inspect index.docker.io/debian:bullseye - digest = "sha256:c11d2593cb741ae8a36d0de9cd240d13518e95f50bccfa8d00a668c006db181e", - registry = "index.docker.io", - repository = "library/debian", -) diff --git a/cli/BUILD b/cli/BUILD index 69e644d18..f0d5423e5 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -1,7 +1,5 @@ load("//tools:ts_library.bzl", "ts_library") load("//tools/common:copy.bzl", "copy_file") -load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_push") -load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit") load("//:version.bzl", "DF_VERSION") load("//testing:index.bzl", "ts_test_suite") load("//tools:node_modules.bzl", "node_modules") @@ -49,8 +47,8 @@ ts_test_suite( srcs = ["index_test.ts"], data = [ ":node_modules", - "//test_credentials:bigquery.json", "//packages/@dataform/core:package_tar", + "//test_credentials:bigquery.json", "@nodejs//:node", "@nodejs//:npm", ], @@ -73,33 +71,3 @@ copy_file( src = "//:readme.md", out = "readme.md", ) - -container_run_and_commit( - name = "image_with_dataform_cli", - commands = [ - "npm i -g @dataform/cli@{dataform_version}".format(dataform_version = DF_VERSION), - ], - image = "@nodejs_base//image", - tags = [ - "no-remote", - ], -) - -container_image( - name = "image_with_dataform_entrypoint", - base = ":image_with_dataform_cli_commit.tar", - cmd = [], - entrypoint = ["dataform"], - tags = [ - "no-remote", - ], -) - -container_push( - name = "push", - format = "Docker", - image = ":image_with_dataform_entrypoint", - registry = "docker.io", - repository = "dataformco/dataform", - tag = DF_VERSION, -) diff --git a/cli/index_test.ts b/cli/index_test.ts index 8c1302bdb..05844a487 100644 --- a/cli/index_test.ts +++ b/cli/index_test.ts @@ -17,7 +17,7 @@ suite("@dataform/cli", ({ afterEach }) => { const nodePath = `external/${platformPath}/bin/node`; const cliEntryPointPath = "cli/node_modules/@dataform/cli/bundle.js"; const npmPath = `external/${platformPath}/bin/npm`; - const corePackageTarPath = "packages/@dataform/core/package.tgz"; + const corePackageTarPath = "packages/@dataform/core/package.tar.gz"; test( "compile throws an error when dataformCoreVersion not in workflow_settings.yaml and no " + diff --git a/cloudbuild-test.json b/cloudbuild-test.json deleted file mode 100644 index dd36630dc..000000000 --- a/cloudbuild-test.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "steps": [ - { - "name": "gcr.io/dataform-public/dataform-builder@sha256:413b7e076ae77f1bed9b254434919e40d4d4112c00e1a12ea076da112a7012e9", - "entrypoint": "bash", - "args": ["./scripts/cloudbuild/bazel_test"] - } - ], - "timeout": "3600s", - "options": { - "machineType": "N1_HIGHCPU_8" - } -} diff --git a/cloudbuild/BUILD b/cloudbuild/BUILD deleted file mode 100644 index f6b3bd68e..000000000 --- a/cloudbuild/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit") -load("@io_bazel_rules_docker//container:container.bzl", "container_push") - -# The following defines our base builder image for the dataform repo. -container_run_and_commit( - name = "builder", - commands = [ - # Install apt packages. - "apt update", - "apt install -yq build-essential cmake git ca-certificates apt-transport-https gnupg2 lsb-release wget curl python2 python3.9 default-jre default-jdk", - # Update certificates. - "update-ca-certificates", - # Install docker. - "curl -fsSL https://get.docker.com -o get-docker.sh", - "chmod +x get-docker.sh", - "./get-docker.sh", - # Install bazelisk. - "wget https://github.com/bazelbuild/bazelisk/releases/download/v1.2.1/bazelisk-linux-amd64 -O /usr/bin/bazel", - "chmod +x /usr/bin/bazel", - # Set python versions. - "ln -sf /usr/bin/python2.7 /usr/bin/python", - "ln -sf /usr/bin/python3.9 /usr/bin/python3" - ], - image = "@debian_base//image", - tags = [ - "no-remote", - ], -) - -container_push( - name = "builder.push", - format = "Docker", - image = ":builder_commit.tar", - registry = "gcr.io", - repository = "dataform-public/dataform-builder", - tag = "latest", -) diff --git a/cloudbuild/Dockerfile b/cloudbuild/Dockerfile new file mode 100644 index 000000000..568bd6995 --- /dev/null +++ b/cloudbuild/Dockerfile @@ -0,0 +1,13 @@ +# TODO(ekrekr): update to image `gcr.io/cloud-builders/bazel` once we're at Bazel version 7+. +FROM gcr.io/cloud-builders/bazel:5.4.0 + +# Install apt packages. +RUN apt update +RUN apt install -yq build-essential cmake git ca-certificates apt-transport-https gnupg2 lsb-release wget curl python3 default-jre default-jdk + +# Update certificates. +RUN update-ca-certificates + +# Install bazelisk. +RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.2.1/bazelisk-linux-amd64 -O /usr/bin/bazel +RUN chmod +x /usr/bin/bazel diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml new file mode 100644 index 000000000..ac69ccb2c --- /dev/null +++ b/cloudbuild/cloudbuild.yaml @@ -0,0 +1,6 @@ +steps: + - name: "gcr.io/dataform-public/dataform-builder" + args: + ["build", "-t", "gcr.io/dataform-public/dataform-builder", "-f", "cloudbuild/Dockerfile", "."] + - name: + args: ["./cloudbuild/run_tests.sh"] diff --git a/scripts/cloudbuild/bazel_test b/cloudbuild/run_tests.sh similarity index 73% rename from scripts/cloudbuild/bazel_test rename to cloudbuild/run_tests.sh index 960f16d65..601ef58cf 100755 --- a/scripts/cloudbuild/bazel_test +++ b/cloudbuild/run_tests.sh @@ -1,6 +1,9 @@ #!/bin/bash set -e +# # Cloud build expects the dataform repo to be mounted in a `/dataform` directory. +# cd dataform + # Run tslint. bazel run @nodejs//:yarn bazel build @npm//tslint/bin:tslint && bazel-bin/external/npm/tslint/bin/tslint.sh --project . diff --git a/packages/@dataform/cli/BUILD b/packages/@dataform/cli/BUILD index ec1c6f5fd..6eb043aa7 100644 --- a/packages/@dataform/cli/BUILD +++ b/packages/@dataform/cli/BUILD @@ -90,6 +90,5 @@ pkg_npm_tar( ":bundle", ":package.json", ":worker_bundle", - ], ) diff --git a/packages/index.bzl b/packages/index.bzl index db3360bb4..145945a15 100644 --- a/packages/index.bzl +++ b/packages/index.bzl @@ -63,8 +63,8 @@ def pkg_npm_tar(name, srcs = [], deps = []): native.genrule( name = name + "_tar", srcs = [":" + name], - outs = [name + ".tgz"], - cmd = "tar -cvzf $(location {name}.tgz) -C $(location :{name})/.. --dereference {name}" + outs = [name + ".tar.gz"], + cmd = "tar -cvzf $(location {name}.tar.gz) -C $(location :{name})/.. --dereference {name}" .format(name = name), ) diff --git a/scripts/publish b/scripts/publish index c3b115538..d43ac47a4 100755 --- a/scripts/publish +++ b/scripts/publish @@ -32,7 +32,3 @@ bazel run cli:push git tag $VERSION git push origin $VERSION - -docker pull dataformco/dataform:$VERSION -docker tag dataformco/dataform:$VERSION dataformco/dataform:$TAG -docker push dataformco/dataform:$TAG From 970b7f64c21da9e0c0bc5797b2175cbea2e805d3 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Thu, 14 Mar 2024 16:24:17 +0000 Subject: [PATCH 02/10] Tidy --- cloudbuild/run_tests.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudbuild/run_tests.sh b/cloudbuild/run_tests.sh index 601ef58cf..960f16d65 100755 --- a/cloudbuild/run_tests.sh +++ b/cloudbuild/run_tests.sh @@ -1,9 +1,6 @@ #!/bin/bash set -e -# # Cloud build expects the dataform repo to be mounted in a `/dataform` directory. -# cd dataform - # Run tslint. bazel run @nodejs//:yarn bazel build @npm//tslint/bin:tslint && bazel-bin/external/npm/tslint/bin/tslint.sh --project . From fa84a49af5cfbc3452bf14b931e65ea25d215586 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Fri, 15 Mar 2024 10:51:01 +0000 Subject: [PATCH 03/10] Swap to using just scripts --- cloudbuild/Dockerfile | 13 ------------- cloudbuild/cloudbuild.yaml | 5 ++--- cloudbuild/install_requirements.sh | 12 ++++++++++++ 3 files changed, 14 insertions(+), 16 deletions(-) delete mode 100644 cloudbuild/Dockerfile mode change 100644 => 100755 cloudbuild/cloudbuild.yaml create mode 100755 cloudbuild/install_requirements.sh diff --git a/cloudbuild/Dockerfile b/cloudbuild/Dockerfile deleted file mode 100644 index 568bd6995..000000000 --- a/cloudbuild/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -# TODO(ekrekr): update to image `gcr.io/cloud-builders/bazel` once we're at Bazel version 7+. -FROM gcr.io/cloud-builders/bazel:5.4.0 - -# Install apt packages. -RUN apt update -RUN apt install -yq build-essential cmake git ca-certificates apt-transport-https gnupg2 lsb-release wget curl python3 default-jre default-jdk - -# Update certificates. -RUN update-ca-certificates - -# Install bazelisk. -RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.2.1/bazelisk-linux-amd64 -O /usr/bin/bazel -RUN chmod +x /usr/bin/bazel diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml old mode 100644 new mode 100755 index ac69ccb2c..e5a11053d --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -1,6 +1,5 @@ steps: - - name: "gcr.io/dataform-public/dataform-builder" - args: - ["build", "-t", "gcr.io/dataform-public/dataform-builder", "-f", "cloudbuild/Dockerfile", "."] + - name: + args: ["./cloudbuild/install_requirements.sh"] - name: args: ["./cloudbuild/run_tests.sh"] diff --git a/cloudbuild/install_requirements.sh b/cloudbuild/install_requirements.sh new file mode 100755 index 000000000..ef36201c8 --- /dev/null +++ b/cloudbuild/install_requirements.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +apt update +apt install -yq build-essential cmake git ca-certificates apt-transport-https gnupg2 lsb-release wget curl python3 default-jre default-jdk + +# Update certificates. +update-ca-certificates + +# Install bazelisk. +wget https://github.com/bazelbuild/bazelisk/releases/download/v1.2.1/bazelisk-linux-amd64 -O /usr/bin/bazel +chmod +x /usr/bin/bazel From 0e5f80a203f742738cff11e4ed1f3a756a9fd3c9 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Fri, 15 Mar 2024 11:14:28 +0000 Subject: [PATCH 04/10] Move script to base layout, use prebuilt bazel image --- cloudbuild.yaml | 4 ++++ cloudbuild/cloudbuild.yaml | 5 ----- cloudbuild/install_requirements.sh | 12 ------------ cloudbuild/run_tests.sh => scripts/run_tests | 0 4 files changed, 4 insertions(+), 17 deletions(-) create mode 100755 cloudbuild.yaml delete mode 100755 cloudbuild/cloudbuild.yaml delete mode 100755 cloudbuild/install_requirements.sh rename cloudbuild/run_tests.sh => scripts/run_tests (100%) diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100755 index 000000000..e91d6e903 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,4 @@ +steps: + - name: gcr.io/cloud-builders/bazel:5.4.0 + entrypoint: bash + args: ["./scripts/run_tests"] diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml deleted file mode 100755 index e5a11053d..000000000 --- a/cloudbuild/cloudbuild.yaml +++ /dev/null @@ -1,5 +0,0 @@ -steps: - - name: - args: ["./cloudbuild/install_requirements.sh"] - - name: - args: ["./cloudbuild/run_tests.sh"] diff --git a/cloudbuild/install_requirements.sh b/cloudbuild/install_requirements.sh deleted file mode 100755 index ef36201c8..000000000 --- a/cloudbuild/install_requirements.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -apt update -apt install -yq build-essential cmake git ca-certificates apt-transport-https gnupg2 lsb-release wget curl python3 default-jre default-jdk - -# Update certificates. -update-ca-certificates - -# Install bazelisk. -wget https://github.com/bazelbuild/bazelisk/releases/download/v1.2.1/bazelisk-linux-amd64 -O /usr/bin/bazel -chmod +x /usr/bin/bazel diff --git a/cloudbuild/run_tests.sh b/scripts/run_tests similarity index 100% rename from cloudbuild/run_tests.sh rename to scripts/run_tests From 12ad252d0fab5229725ad1b32bf2ffed1d53d344 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Fri, 15 Mar 2024 11:16:50 +0000 Subject: [PATCH 05/10] Trigger Build From 760413b9d065ccfb4757f00e30223fabc758a5da Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Fri, 15 Mar 2024 11:18:56 +0000 Subject: [PATCH 06/10] Swap back to JSON with the same name --- cloudbuild-test.json | 9 +++++++++ cloudbuild.yaml | 4 ---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 cloudbuild-test.json delete mode 100755 cloudbuild.yaml diff --git a/cloudbuild-test.json b/cloudbuild-test.json new file mode 100644 index 000000000..b2f0000f3 --- /dev/null +++ b/cloudbuild-test.json @@ -0,0 +1,9 @@ +{ + "steps": [ + { + "name": "gcr.io/cloud-builders/bazel:5.4.0", + "entrypoint": "bash", + "args": ["./scripts/run_tests"] + } + ] +} diff --git a/cloudbuild.yaml b/cloudbuild.yaml deleted file mode 100755 index e91d6e903..000000000 --- a/cloudbuild.yaml +++ /dev/null @@ -1,4 +0,0 @@ -steps: - - name: gcr.io/cloud-builders/bazel:5.4.0 - entrypoint: bash - args: ["./scripts/run_tests"] From 67bb829943ee63717ed3c9d8b3f8ac7035e8ad35 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Fri, 15 Mar 2024 11:45:27 +0000 Subject: [PATCH 07/10] Bump to v3 --- version.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.bzl b/version.bzl index 71a5a450d..6c596dd52 100644 --- a/version.bzl +++ b/version.bzl @@ -1,3 +1,3 @@ # NOTE: If you change the format of this line, you must change the bash command # in /scripts/publish to extract the version string correctly. -DF_VERSION = "3.0.0-beta.1" +DF_VERSION = "3.0.0-beta.2" From 04eda7658cf07542e9c8413dd63633756428ef5a Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Fri, 15 Mar 2024 11:52:31 +0000 Subject: [PATCH 08/10] Tidy --- scripts/publish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/publish b/scripts/publish index d43ac47a4..13b644e32 100755 --- a/scripts/publish +++ b/scripts/publish @@ -26,7 +26,7 @@ fi echo "Publishing as '$TAG' based on version: $VERSION" -bazel run packages/@dataform/cli:package.publish -- --tag=$TAG +# bazel run packages/@dataform/cli:package.publish -- --tag=$TAG bazel run packages/@dataform/core:package.publish -- --tag=$TAG bazel run cli:push From d80d02e91a6b957c621e02b52069458e0eef068a Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Mon, 25 Mar 2024 15:41:37 +0000 Subject: [PATCH 09/10] Tidy publish script --- scripts/publish | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/publish b/scripts/publish index 13b644e32..1372907c4 100755 --- a/scripts/publish +++ b/scripts/publish @@ -26,9 +26,8 @@ fi echo "Publishing as '$TAG' based on version: $VERSION" -# bazel run packages/@dataform/cli:package.publish -- --tag=$TAG +bazel run packages/@dataform/cli:package.publish -- --tag=$TAG bazel run packages/@dataform/core:package.publish -- --tag=$TAG -bazel run cli:push git tag $VERSION git push origin $VERSION From 7807983b720d6eacf7e92c049eb8744afe02ce99 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Mon, 25 Mar 2024 17:25:08 +0000 Subject: [PATCH 10/10] Re-add machine options for cloudbuild --- cloudbuild-test.json | 6 +++++- readme.md | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cloudbuild-test.json b/cloudbuild-test.json index b2f0000f3..54b95589e 100644 --- a/cloudbuild-test.json +++ b/cloudbuild-test.json @@ -5,5 +5,9 @@ "entrypoint": "bash", "args": ["./scripts/run_tests"] } - ] + ], + "timeout": "3600s", + "options": { + "machineType": "N1_HIGHCPU_8" + } } diff --git a/readme.md b/readme.md index c7a5a40ce..9c23579c4 100644 --- a/readme.md +++ b/readme.md @@ -35,9 +35,8 @@ npm i -g @dataform/cli - Write [data quality checks](https://cloud.google.com/dataform/docs/assertions). - Enable [scripting](https://cloud.google.com/dataform/docs/develop-workflows-js) and code re-use with a JavaScript API. - Import [pre-defined packages](https://dataform-co.github.io/dataform/docs/packages), or create your own. -- Import [pre-defined packages](https://dataform-co.github.io/dataform/docs/packages), or create your own. - View the [Dataform Core reference](https://cloud.google.com/dataform/docs/reference/dataform-core-reference). -- View the [Dataform configs reference](docs/configs-reference). +- View the [Dataform configs reference](https://dataform-co.github.io/dataform/docs/configs-reference). _Note: this readme can also be viewed on https://dataform-co.github.io/dataform._