From 55906074bcea5f2b4fdc021d4625cd4994411355 Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Mon, 2 Sep 2024 16:07:31 +0530 Subject: [PATCH 1/3] Add docker-compose for rill cloud --- Makefile | 9 +++-- web-admin/tests/setup/admin/Dockerfile | 14 +++++++ web-admin/tests/setup/binary/Dockerfile | 29 +++++++++++++++ web-admin/tests/setup/cloud/Dockerfile | 10 +++++ web-admin/tests/setup/docker-compose.yaml | 45 +++++++++++++++++++++++ web-admin/tests/setup/runtime/Dockerfile | 16 ++++++++ 6 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 web-admin/tests/setup/admin/Dockerfile create mode 100644 web-admin/tests/setup/binary/Dockerfile create mode 100644 web-admin/tests/setup/cloud/Dockerfile create mode 100644 web-admin/tests/setup/docker-compose.yaml create mode 100644 web-admin/tests/setup/runtime/Dockerfile diff --git a/Makefile b/Makefile index 6442a1dc41b..32ce5d7b9d5 100644 --- a/Makefile +++ b/Makefile @@ -2,16 +2,19 @@ all: cli .PHONY: cli -cli: cli.prepare +cli: cli.prepare.ui cli.prepare go build -o rill cli/main.go -.PHONY: cli.prepare -cli.prepare: +.PHONY: cli.prepare.ui +cli.prepare.ui: npm install npm run build rm -rf cli/pkg/web/embed/dist || true mkdir -p cli/pkg/web/embed/dist cp -r web-local/build/* cli/pkg/web/embed/dist + +.PHONY: cli.prepare +cli.prepare: rm -rf runtime/pkg/examples/embed/dist || true mkdir -p runtime/pkg/examples/embed/dist git clone --quiet https://github.com/rilldata/rill-examples.git runtime/pkg/examples/embed/dist diff --git a/web-admin/tests/setup/admin/Dockerfile b/web-admin/tests/setup/admin/Dockerfile new file mode 100644 index 00000000000..1b19f71e58a --- /dev/null +++ b/web-admin/tests/setup/admin/Dockerfile @@ -0,0 +1,14 @@ +# syntax = docker/dockerfile:1.1-experimental +FROM ubuntu + +WORKDIR /usr/src/app + +RUN apt-get update && apt-get install -y ca-certificates + +COPY rill . +COPY web-admin/tests/setup/admin/.env . +RUN cp rill /usr/local/bin +RUN chmod 777 /usr/local/bin/rill + +ENTRYPOINT ["rill"] +CMD ["admin", "start"] diff --git a/web-admin/tests/setup/binary/Dockerfile b/web-admin/tests/setup/binary/Dockerfile new file mode 100644 index 00000000000..67093f4777c --- /dev/null +++ b/web-admin/tests/setup/binary/Dockerfile @@ -0,0 +1,29 @@ +FROM golang:1.22.0 + +WORKDIR /usr/src/app + +# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change +COPY go.mod go.sum ./ +RUN go mod download && go mod verify + +# copy code files +COPY cli cli +COPY proto proto +COPY runtime runtime +COPY admin admin + +# copy ui files +RUN mkdir -p cli/pkg/web/embed/dist +COPY web-local/build/* cli/pkg/web/embed/dist + +# build the binary, using a dummy version to enable admin and runtime commands +RUN go build -o rill cli/main.go -ldflags="-s -w -X main.Version=v0.0.1" + +RUN cp rill /usr/local/bin +RUN chmod 777 /usr/local/bin/rill + +RUN mkdir project +RUN touch project/rill.yaml + +ENTRYPOINT ["rill"] +CMD ["start", "project"] diff --git a/web-admin/tests/setup/cloud/Dockerfile b/web-admin/tests/setup/cloud/Dockerfile new file mode 100644 index 00000000000..8d3caf44ff4 --- /dev/null +++ b/web-admin/tests/setup/cloud/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.9 + +WORKDIR /usr/src/app + +# copy ui files +COPY web-admin/build . + +# start a simple http server using python +ENTRYPOINT ["python3"] +CMD ["-m", "http.server", "3000"] diff --git a/web-admin/tests/setup/docker-compose.yaml b/web-admin/tests/setup/docker-compose.yaml new file mode 100644 index 00000000000..49370ebbedd --- /dev/null +++ b/web-admin/tests/setup/docker-compose.yaml @@ -0,0 +1,45 @@ +version: "3.9" +services: + postgres: + image: postgres + restart: always + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + ports: + - '5432:5432' + redis: + image: redis + restart: always + ports: + - '6379:6379' + + runtime: + image: rilldata/runtime + build: + context: ../../../ + dockerfile: web-admin/tests/setup/runtime/Dockerfile + ports: + - '8081:8081' + - '9091:9091' + + admin: + image: rilldata/admin + build: + context: ../../../ + dockerfile: web-admin/tests/setup/admin/Dockerfile + depends_on: + - postgres + - redis + ports: + - '8080:8080' + + cloud: + image: rilldata/cloud + build: + context: ../../../ + dockerfile: web-admin/tests/setup/cloud/Dockerfile + depends_on: + - admin + ports: + - '3000:3000' diff --git a/web-admin/tests/setup/runtime/Dockerfile b/web-admin/tests/setup/runtime/Dockerfile new file mode 100644 index 00000000000..7dcf66c75ea --- /dev/null +++ b/web-admin/tests/setup/runtime/Dockerfile @@ -0,0 +1,16 @@ +# syntax = docker/dockerfile:1.1-experimental +FROM ubuntu + +WORKDIR /usr/src/app + +RUN apt-get update && apt-get install -y ca-certificates + +COPY rill . +COPY web-admin/tests/setup/admin/.env . +RUN cp rill /usr/local/bin +RUN chmod 777 /usr/local/bin/rill + +RUN rill runtime install-duckdb-extensions + +ENTRYPOINT ["rill"] +CMD ["runtime", "start"] From cdbf17fb87f48774752e085d048335002598e453 Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Tue, 3 Sep 2024 15:13:43 +0530 Subject: [PATCH 2/3] Add oauth, email and git --- web-admin/tests/setup/docker-compose.yaml | 26 +++++++++++++++++++++++ web-admin/tests/setup/email/Dockerfile | 6 ++++++ web-admin/tests/setup/git/.gitignore | 1 + web-admin/tests/setup/git/Dockerfile | 8 +++++++ web-admin/tests/setup/runtime/Dockerfile | 1 - 5 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 web-admin/tests/setup/email/Dockerfile create mode 100644 web-admin/tests/setup/git/.gitignore create mode 100644 web-admin/tests/setup/git/Dockerfile diff --git a/web-admin/tests/setup/docker-compose.yaml b/web-admin/tests/setup/docker-compose.yaml index 49370ebbedd..4f1122456c1 100644 --- a/web-admin/tests/setup/docker-compose.yaml +++ b/web-admin/tests/setup/docker-compose.yaml @@ -8,12 +8,38 @@ services: - POSTGRES_PASSWORD=postgres ports: - '5432:5432' + redis: image: redis restart: always ports: - '6379:6379' + oauth: + image: public.ecr.aws/primaassicurazioni/localauth0:0.8.0 + restart: always + ports: + - '3010:3000' + + git: + image: rilldata/git + build: + context: . + dockerfile: git/Dockerfile + ports: + - '9418:9418' + volumes: + - ./git/repos:/usr/src/app + + email: + image: rilldata/email + build: + context: . + dockerfile: email/Dockerfile + ports: + - '1025:1025' + - '1080:1080' + runtime: image: rilldata/runtime build: diff --git a/web-admin/tests/setup/email/Dockerfile b/web-admin/tests/setup/email/Dockerfile new file mode 100644 index 00000000000..abd27d5cab8 --- /dev/null +++ b/web-admin/tests/setup/email/Dockerfile @@ -0,0 +1,6 @@ +FROM ruby:3.3 + +WORKDIR /usr/src/app +RUN gem install mailcatcher + +ENTRYPOINT ["mailcatcher"] diff --git a/web-admin/tests/setup/git/.gitignore b/web-admin/tests/setup/git/.gitignore new file mode 100644 index 00000000000..c05e7776daf --- /dev/null +++ b/web-admin/tests/setup/git/.gitignore @@ -0,0 +1 @@ +repos diff --git a/web-admin/tests/setup/git/Dockerfile b/web-admin/tests/setup/git/Dockerfile new file mode 100644 index 00000000000..928026e7e30 --- /dev/null +++ b/web-admin/tests/setup/git/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu + +WORKDIR /usr/src/app + +RUN apt-get update && apt-get install -y git + +ENTRYPOINT ["git"] +CMD ["daemon", "--verbose", "--base-path=/usr/src/app/", "--export-all", "--enable=upload-archive", "--enable=receive-pack", "/usr/src/app/"] diff --git a/web-admin/tests/setup/runtime/Dockerfile b/web-admin/tests/setup/runtime/Dockerfile index 7dcf66c75ea..87faaa4780f 100644 --- a/web-admin/tests/setup/runtime/Dockerfile +++ b/web-admin/tests/setup/runtime/Dockerfile @@ -1,4 +1,3 @@ -# syntax = docker/dockerfile:1.1-experimental FROM ubuntu WORKDIR /usr/src/app From 8f5cb255d42fcb61dfe1d66622c905edbf0b7858 Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Thu, 5 Sep 2024 15:20:19 +0530 Subject: [PATCH 3/3] Adding .env --- cli/cmd/admin/start.go | 12 ++++++---- web-admin/tests/setup/admin/.env | 28 ++++++++++++++++++++++++ web-admin/tests/setup/runtime/.env | 17 ++++++++++++++ web-admin/tests/setup/runtime/Dockerfile | 2 +- 4 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 web-admin/tests/setup/admin/.env create mode 100644 web-admin/tests/setup/runtime/.env diff --git a/cli/cmd/admin/start.go b/cli/cmd/admin/start.go index d8c6702af4a..488461d6791 100644 --- a/cli/cmd/admin/start.go +++ b/cli/cmd/admin/start.go @@ -69,6 +69,7 @@ type Config struct { GithubAppWebhookSecret string `split_words:"true"` GithubClientID string `split_words:"true"` GithubClientSecret string `split_words:"true"` + GithubAppDisabled bool `split_words:"true"` AssetsBucket string `split_words:"true"` // AssetsBucketGoogleCredentialsJSON is only required to be set for local development. // For production use cases the service account will be directly attached to pods which is the recommended way of setting credentials. @@ -216,10 +217,13 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command { } emailClient := email.New(sender) - // Init github client - gh, err := admin.NewGithub(conf.GithubAppID, conf.GithubAppPrivateKey) - if err != nil { - logger.Fatal("error creating github client", zap.Error(err)) + var gh admin.Github + if !conf.GithubAppDisabled { + // Init github client + gh, err = admin.NewGithub(conf.GithubAppID, conf.GithubAppPrivateKey) + if err != nil { + logger.Fatal("error creating github client", zap.Error(err)) + } } // Init AI client diff --git a/web-admin/tests/setup/admin/.env b/web-admin/tests/setup/admin/.env new file mode 100644 index 00000000000..de4cd5b45cf --- /dev/null +++ b/web-admin/tests/setup/admin/.env @@ -0,0 +1,28 @@ +# admin +RILL_ADMIN_DATABASE_DRIVER=postgres +RILL_ADMIN_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres +RILL_ADMIN_RIVER_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres +RILL_ADMIN_REDIS_URL=redis://redis:6379 +RILL_ADMIN_PROVISIONER_SPEC='{"runtimes":[{"host":"http://runtime:9091","slots":50,"data_dir":"dev-cloud-state","audience_url":"http://runtime:8081"}]}' +RILL_ADMIN_PROVISIONER_SET_JSON='{"static":{"type":"static","spec":{"runtimes":[{"host":"http://runtime:9091","slots":50,"data_dir":"dev-cloud-state","audience_url":"http://runtime:8081"}]}}}' +RILL_ADMIN_DEFAULT_PROVISIONER=static +RILL_ADMIN_HTTP_PORT=8080 +RILL_ADMIN_GRPC_PORT=9090 +RILL_ADMIN_DEBUG_PORT=6062 +RILL_ADMIN_PSQL_PORT=25432 +RILL_ADMIN_EXTERNAL_URL=http://localhost:8080 +RILL_ADMIN_EXTERNAL_GRPC_URL=http://localhost:9090 +RILL_ADMIN_FRONTEND_URL=http://localhost:3000 +RILL_ADMIN_SESSION_KEY_PAIRS=1234567890abcdef1234567890abcdef,1234567890abcdef1234567890abcdef +RILL_ADMIN_SIGNING_JWKS='{"keys":[{"p":"xhq80n_cNqtMs6fHKeudRfCNkHwUtGxXnJrsbohnOuNJy2aN3O1xBv5uMDXGvIayTKJX7dZuambj-8F-CxGym_2mX9XrWE5BoSkFvpRmp4zqOcqoBGASYH3zBeHBe4bX4L_Sv0SvqpkDdRraSHNWnAQtLgKObndVINKGyzqSloM","kty":"RSA","q":"uBVrB7-DEjd6SwMh-OuedsDM-MAfLbS5rmYth5ySoGY3UVmyeciLIZ-6UDEL5xwjPKGZmLYkqznt6ao1w6EYBsLXpHWkLlWmlBBhH4RnQDPXaU-CRXclwIE-_lxJWQIgmgkEjS9Sdu4dWcRCJJ5QH1S8EgOUc7iMruhVm_PG728","d":"BfvlO0oKo9hxqcXxK5YRpiRRztfu-YHlnrtP9yDNhhHOVeF-JT9bDvpyc1uMVAzraurTAg3DHwXM0j0djO284NKPz0KJCsJLmaE5DCt6BH5ivtqO-AUQtz-uU2pl3AjZR7hIEhPFBAgx23hhr5I7UFtds6e0YtfjU8_RqJZhyLOBSkZXKJqEZEcqTyE6icxENcSWjFJ5kd9WtWEsK7Giq5_J70E61DQIdgR0v7jF0-ls9MlLPuCqfn92pLe3oYxDuOtoDEpxwVDAAFoQFV0jO_ZRsmxAFK3aTBZtEBJq3ZhZLGUrEvSUAP-C-O2nA5NxSBqp1Z84H8LIzqgETpkG3Q","e":"AQAB","use":"sig","kid":"e2e_key","qi":"uAUYnt5K9P2DhbXdkyKjS_GSt99wgUjbVyjYj6-k7Sp5yZfBBt2UqVNfrPLsnL55tS_dZvH-Vyo7PfVbcsmB0fCKR3rgbiwhcus6o5ZIcWTT8FQsdmjY5JdOUvoUIVzau1VXVugtvRJ8hwU4OBiHlONnf5-6zGaXinxlcaQSwR0","dp":"hKk9Dz2eS2HVIG3XWoQ0srRPKJVfDB-NYAsDD1X6wx2XukT39R1wVj2lrlUku_T7i4Qfm96_3DMAswO1E1aLpBhtBLptOppVKsYc2a3pPISKCZ9jemKXi9RsHOE8Pytif1gVmg42EOkcvpv31xKdyial1qzRkroI4Mz6hTe5GF8","alg":"RS256","dq":"hi5-jPBJP08XInSPH93H7XIONsf8MhnJO1mZmH9F1guMCQ1j7ieDWygN7AREU9sLyFdXGZyzT-hzBl8ev5G-bLc77tIdsfSW7tidoxalGIRxYX8mSzJVX2QWodxtTqchUoZ-PdefubdJx7QqLn0DlEL0sADSBiQg0CHkvT_N45s","n":"jnPKu_UFSjah0mYBEm8k77lSxpn5QLVvLk1umv3r_tJhPORic6HsUToQuE4U-nGVaz1F6q8djPb3K2uRt_XA0btuu97sf-RSkVN5exLk7_NIXjoCSRfH3RKmvTU3LtbtjEfEtMG0qPs_S2_JsJROuUxZjQKDuVp-n60wq9kqOZfbKDebqRIyLWgbbcYMVvuTOQGq45X6Jxp1wNWqWHlhB7XjqNVQzMEsECfUM6JV9eW0PwnMfj72rY42hjzoBCXKYe8WxZvOCKlgYXx7SexeUuTxAN39rP7wK_2VoZXvyDktw_HAVFwMSW5byp5TtD1EvluHR8q0tp2EW0BdAmWPzQ"}]}' +RILL_ADMIN_SIGNING_KEY_ID=e2e_key +RILL_ADMIN_AUTH_DOMAIN=oauth:3010 +RILL_ADMIN_AUTH_CLIENT_ID=client +RILL_ADMIN_AUTH_CLIENT_SECRET=secret +RILL_ADMIN_GITHUB_APP_DISABLED=true +RILL_ADMIN_EMAIL_SMTP_HOST=email +RILL_ADMIN_EMAIL_SMTP_PORT=1025 +RILL_ADMIN_EMAIL_SMTP_USERNAME=smtp +RILL_ADMIN_EMAIL_SMTP_PASSWORD=smtp +RILL_ADMIN_EMAIL_SENDER_EMAIL="admin@rilldata.com" +RILL_ADMIN_EMAIL_SENDER_NAME="Rill Cloud dev" diff --git a/web-admin/tests/setup/runtime/.env b/web-admin/tests/setup/runtime/.env new file mode 100644 index 00000000000..8468fb279a3 --- /dev/null +++ b/web-admin/tests/setup/runtime/.env @@ -0,0 +1,17 @@ +RILL_RUNTIME_METASTORE_URL=dev-cloud-state/meta.db +RILL_RUNTIME_REDIS_URL=redis://redis:6379 +RILL_RUNTIME_HTTP_PORT=8081 +RILL_RUNTIME_GRPC_PORT=9091 +RILL_RUNTIME_PSQL_PORT=15432 +RILL_RUNTIME_DEBUG_PORT=6061 +RILL_RUNTIME_SESSION_KEY_PAIRS=1234567890abcdef1234567890abcdef,1234567890abcdef1234567890abcdef +RILL_RUNTIME_AUTH_ENABLE=true +RILL_RUNTIME_AUTH_ISSUER_URL=http://localhost:8080 +RILL_RUNTIME_AUTH_AUDIENCE_URL=http://localhost:8081 +RILL_RUNTIME_EMAIL_SMTP_HOST=email +RILL_RUNTIME_EMAIL_SMTP_PORT=1025 +RILL_RUNTIME_EMAIL_SMTP_USERNAME=smtp +RILL_RUNTIME_EMAIL_SMTP_PASSWORD=smtp +RILL_RUNTIME_EMAIL_SENDER_EMAIL="admin@rilldata.com" +RILL_RUNTIME_EMAIL_SENDER_NAME="Rill Cloud dev" +RILL_RUNTIME_DATA_DIR=dev-cloud-state diff --git a/web-admin/tests/setup/runtime/Dockerfile b/web-admin/tests/setup/runtime/Dockerfile index 87faaa4780f..32b3ec89f32 100644 --- a/web-admin/tests/setup/runtime/Dockerfile +++ b/web-admin/tests/setup/runtime/Dockerfile @@ -9,7 +9,7 @@ COPY web-admin/tests/setup/admin/.env . RUN cp rill /usr/local/bin RUN chmod 777 /usr/local/bin/rill -RUN rill runtime install-duckdb-extensions +#RUN rill runtime install-duckdb-extensions ENTRYPOINT ["rill"] CMD ["runtime", "start"]