feat: Rill cloud E2E setup

Makefile

@@ -2,16 +2,19 @@
 all: cli
 
 .PHONY: cli
-cli: cli.prepare
+cli: cli.prepare.ui cli.prepare
 	go build -o rill cli/main.go 
 
-.PHONY: cli.prepare
-cli.prepare:
+.PHONY: cli.prepare.ui
+cli.prepare.ui:
 	npm install
 	npm run build
 	rm -rf cli/pkg/web/embed/dist || true
 	mkdir -p cli/pkg/web/embed/dist
 	cp -r web-local/build/* cli/pkg/web/embed/dist
+
+.PHONY: cli.prepare
+cli.prepare:
 	rm -rf runtime/pkg/examples/embed/dist || true
 	mkdir -p runtime/pkg/examples/embed/dist
 	git clone --quiet https://github.com/rilldata/rill-examples.git runtime/pkg/examples/embed/dist

cli/cmd/admin/start.go

@@ -69,6 +69,7 @@ type Config struct {
 	GithubAppWebhookSecret string                 `split_words:"true"`
 	GithubClientID         string                 `split_words:"true"`
 	GithubClientSecret     string                 `split_words:"true"`
+	GithubAppDisabled      bool                   `split_words:"true"`
 	AssetsBucket           string                 `split_words:"true"`
 	// AssetsBucketGoogleCredentialsJSON is only required to be set for local development.
 	// For production use cases the service account will be directly attached to pods which is the recommended way of setting credentials.
@@ -216,10 +217,13 @@ func StartCmd(ch *cmdutil.Helper) *cobra.Command {
 			}
 			emailClient := email.New(sender)
 
-			// Init github client
-			gh, err := admin.NewGithub(conf.GithubAppID, conf.GithubAppPrivateKey)
-			if err != nil {
-				logger.Fatal("error creating github client", zap.Error(err))
+			var gh admin.Github
+			if !conf.GithubAppDisabled {
+				// Init github client
+				gh, err = admin.NewGithub(conf.GithubAppID, conf.GithubAppPrivateKey)
+				if err != nil {
+					logger.Fatal("error creating github client", zap.Error(err))
+				}
 			}
 
 			// Init AI client

web-admin/tests/setup/admin/.env

@@ -0,0 +1,28 @@
+# admin
+RILL_ADMIN_DATABASE_DRIVER=postgres
+RILL_ADMIN_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres
+RILL_ADMIN_RIVER_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres
+RILL_ADMIN_REDIS_URL=redis://redis:6379
+RILL_ADMIN_PROVISIONER_SPEC='{"runtimes":[{"host":"http://runtime:9091","slots":50,"data_dir":"dev-cloud-state","audience_url":"http://runtime:8081"}]}'
+RILL_ADMIN_PROVISIONER_SET_JSON='{"static":{"type":"static","spec":{"runtimes":[{"host":"http://runtime:9091","slots":50,"data_dir":"dev-cloud-state","audience_url":"http://runtime:8081"}]}}}'
+RILL_ADMIN_DEFAULT_PROVISIONER=static
+RILL_ADMIN_HTTP_PORT=8080
+RILL_ADMIN_GRPC_PORT=9090
+RILL_ADMIN_DEBUG_PORT=6062
+RILL_ADMIN_PSQL_PORT=25432
+RILL_ADMIN_EXTERNAL_URL=http://localhost:8080
+RILL_ADMIN_EXTERNAL_GRPC_URL=http://localhost:9090
+RILL_ADMIN_FRONTEND_URL=http://localhost:3000
+RILL_ADMIN_SESSION_KEY_PAIRS=1234567890abcdef1234567890abcdef,1234567890abcdef1234567890abcdef
+RILL_ADMIN_SIGNING_JWKS='{"keys":[{"p":"xhq80n_cNqtMs6fHKeudRfCNkHwUtGxXnJrsbohnOuNJy2aN3O1xBv5uMDXGvIayTKJX7dZuambj-8F-CxGym_2mX9XrWE5BoSkFvpRmp4zqOcqoBGASYH3zBeHBe4bX4L_Sv0SvqpkDdRraSHNWnAQtLgKObndVINKGyzqSloM","kty":"RSA","q":"uBVrB7-DEjd6SwMh-OuedsDM-MAfLbS5rmYth5ySoGY3UVmyeciLIZ-6UDEL5xwjPKGZmLYkqznt6ao1w6EYBsLXpHWkLlWmlBBhH4RnQDPXaU-CRXclwIE-_lxJWQIgmgkEjS9Sdu4dWcRCJJ5QH1S8EgOUc7iMruhVm_PG728","d":"BfvlO0oKo9hxqcXxK5YRpiRRztfu-YHlnrtP9yDNhhHOVeF-JT9bDvpyc1uMVAzraurTAg3DHwXM0j0djO284NKPz0KJCsJLmaE5DCt6BH5ivtqO-AUQtz-uU2pl3AjZR7hIEhPFBAgx23hhr5I7UFtds6e0YtfjU8_RqJZhyLOBSkZXKJqEZEcqTyE6icxENcSWjFJ5kd9WtWEsK7Giq5_J70E61DQIdgR0v7jF0-ls9MlLPuCqfn92pLe3oYxDuOtoDEpxwVDAAFoQFV0jO_ZRsmxAFK3aTBZtEBJq3ZhZLGUrEvSUAP-C-O2nA5NxSBqp1Z84H8LIzqgETpkG3Q","e":"AQAB","use":"sig","kid":"e2e_key","qi":"uAUYnt5K9P2DhbXdkyKjS_GSt99wgUjbVyjYj6-k7Sp5yZfBBt2UqVNfrPLsnL55tS_dZvH-Vyo7PfVbcsmB0fCKR3rgbiwhcus6o5ZIcWTT8FQsdmjY5JdOUvoUIVzau1VXVugtvRJ8hwU4OBiHlONnf5-6zGaXinxlcaQSwR0","dp":"hKk9Dz2eS2HVIG3XWoQ0srRPKJVfDB-NYAsDD1X6wx2XukT39R1wVj2lrlUku_T7i4Qfm96_3DMAswO1E1aLpBhtBLptOppVKsYc2a3pPISKCZ9jemKXi9RsHOE8Pytif1gVmg42EOkcvpv31xKdyial1qzRkroI4Mz6hTe5GF8","alg":"RS256","dq":"hi5-jPBJP08XInSPH93H7XIONsf8MhnJO1mZmH9F1guMCQ1j7ieDWygN7AREU9sLyFdXGZyzT-hzBl8ev5G-bLc77tIdsfSW7tidoxalGIRxYX8mSzJVX2QWodxtTqchUoZ-PdefubdJx7QqLn0DlEL0sADSBiQg0CHkvT_N45s","n":"jnPKu_UFSjah0mYBEm8k77lSxpn5QLVvLk1umv3r_tJhPORic6HsUToQuE4U-nGVaz1F6q8djPb3K2uRt_XA0btuu97sf-RSkVN5exLk7_NIXjoCSRfH3RKmvTU3LtbtjEfEtMG0qPs_S2_JsJROuUxZjQKDuVp-n60wq9kqOZfbKDebqRIyLWgbbcYMVvuTOQGq45X6Jxp1wNWqWHlhB7XjqNVQzMEsECfUM6JV9eW0PwnMfj72rY42hjzoBCXKYe8WxZvOCKlgYXx7SexeUuTxAN39rP7wK_2VoZXvyDktw_HAVFwMSW5byp5TtD1EvluHR8q0tp2EW0BdAmWPzQ"}]}'
+RILL_ADMIN_SIGNING_KEY_ID=e2e_key
+RILL_ADMIN_AUTH_DOMAIN=oauth:3010
+RILL_ADMIN_AUTH_CLIENT_ID=client
+RILL_ADMIN_AUTH_CLIENT_SECRET=secret
+RILL_ADMIN_GITHUB_APP_DISABLED=true
+RILL_ADMIN_EMAIL_SMTP_HOST=email
+RILL_ADMIN_EMAIL_SMTP_PORT=1025
+RILL_ADMIN_EMAIL_SMTP_USERNAME=smtp
+RILL_ADMIN_EMAIL_SMTP_PASSWORD=smtp
+RILL_ADMIN_EMAIL_SENDER_EMAIL="[email protected]"
+RILL_ADMIN_EMAIL_SENDER_NAME="Rill Cloud dev"

web-admin/tests/setup/admin/Dockerfile

@@ -0,0 +1,14 @@
+# syntax = docker/dockerfile:1.1-experimental
+FROM ubuntu
+
+WORKDIR /usr/src/app
+
+RUN apt-get update && apt-get install -y ca-certificates
+
+COPY rill .
+COPY web-admin/tests/setup/admin/.env .
+RUN cp rill /usr/local/bin
+RUN chmod 777 /usr/local/bin/rill
+
+ENTRYPOINT ["rill"]
+CMD ["admin", "start"]

web-admin/tests/setup/binary/Dockerfile

@@ -0,0 +1,29 @@
+FROM golang:1.22.0
+
+WORKDIR /usr/src/app
+
+# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
+COPY go.mod go.sum ./
+RUN go mod download && go mod verify
+
+# copy code files
+COPY cli cli
+COPY proto proto
+COPY runtime runtime
+COPY admin admin
+
+# copy ui files
+RUN mkdir -p cli/pkg/web/embed/dist
+COPY web-local/build/* cli/pkg/web/embed/dist
+
+# build the binary, using a dummy version to enable admin and runtime commands
+RUN go build -o rill cli/main.go -ldflags="-s -w -X main.Version=v0.0.1"
+
+RUN cp rill /usr/local/bin
+RUN chmod 777 /usr/local/bin/rill
+
+RUN mkdir project
+RUN touch project/rill.yaml
+
+ENTRYPOINT ["rill"]
+CMD ["start", "project"]

web-admin/tests/setup/cloud/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.9
+
+WORKDIR /usr/src/app
+
+# copy ui files
+COPY web-admin/build .
+
+# start a simple http server using python
+ENTRYPOINT ["python3"]
+CMD ["-m", "http.server", "3000"]

web-admin/tests/setup/docker-compose.yaml

@@ -0,0 +1,71 @@
+version: "3.9"
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+    ports:
+      - '5432:5432'
+
+  redis:
+    image: redis
+    restart: always
+    ports:
+      - '6379:6379'
+
+  oauth:
+    image: public.ecr.aws/primaassicurazioni/localauth0:0.8.0
+    restart: always
+    ports:
+      - '3010:3000'
+
+  git:
+    image: rilldata/git
+    build:
+      context: .
+      dockerfile: git/Dockerfile
+    ports:
+      - '9418:9418'
+    volumes:
+      - ./git/repos:/usr/src/app
+
+  email:
+    image: rilldata/email
+    build:
+      context: .
+      dockerfile: email/Dockerfile
+    ports:
+      - '1025:1025'
+      - '1080:1080'
+
+  runtime:
+    image: rilldata/runtime
+    build:
+      context: ../../../
+      dockerfile: web-admin/tests/setup/runtime/Dockerfile
+    ports:
+      - '8081:8081'
+      - '9091:9091'
+
+  admin:
+    image: rilldata/admin
+    build:
+      context: ../../../
+      dockerfile: web-admin/tests/setup/admin/Dockerfile
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - '8080:8080'
+
+  cloud:
+    image: rilldata/cloud
+    build:
+      context: ../../../
+      dockerfile: web-admin/tests/setup/cloud/Dockerfile
+    depends_on:
+      - admin
+    ports:
+      - '3000:3000'

web-admin/tests/setup/email/Dockerfile

@@ -0,0 +1,6 @@
+FROM ruby:3.3
+
+WORKDIR /usr/src/app
+RUN gem install mailcatcher
+
+ENTRYPOINT ["mailcatcher"]

web-admin/tests/setup/git/.gitignore

@@ -0,0 +1 @@
+repos

web-admin/tests/setup/git/Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu
+
+WORKDIR /usr/src/app
+
+RUN apt-get update && apt-get install -y git
+
+ENTRYPOINT ["git"]
+CMD ["daemon", "--verbose", "--base-path=/usr/src/app/", "--export-all", "--enable=upload-archive", "--enable=receive-pack", "/usr/src/app/"]

web-admin/tests/setup/runtime/.env

@@ -0,0 +1,17 @@
+RILL_RUNTIME_METASTORE_URL=dev-cloud-state/meta.db
+RILL_RUNTIME_REDIS_URL=redis://redis:6379
+RILL_RUNTIME_HTTP_PORT=8081
+RILL_RUNTIME_GRPC_PORT=9091
+RILL_RUNTIME_PSQL_PORT=15432
+RILL_RUNTIME_DEBUG_PORT=6061
+RILL_RUNTIME_SESSION_KEY_PAIRS=1234567890abcdef1234567890abcdef,1234567890abcdef1234567890abcdef
+RILL_RUNTIME_AUTH_ENABLE=true
+RILL_RUNTIME_AUTH_ISSUER_URL=http://localhost:8080
+RILL_RUNTIME_AUTH_AUDIENCE_URL=http://localhost:8081
+RILL_RUNTIME_EMAIL_SMTP_HOST=email
+RILL_RUNTIME_EMAIL_SMTP_PORT=1025
+RILL_RUNTIME_EMAIL_SMTP_USERNAME=smtp
+RILL_RUNTIME_EMAIL_SMTP_PASSWORD=smtp
+RILL_RUNTIME_EMAIL_SENDER_EMAIL="[email protected]"
+RILL_RUNTIME_EMAIL_SENDER_NAME="Rill Cloud dev"
+RILL_RUNTIME_DATA_DIR=dev-cloud-state

web-admin/tests/setup/runtime/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu
+
+WORKDIR /usr/src/app
+
+RUN apt-get update && apt-get install -y ca-certificates
+
+COPY rill .
+COPY web-admin/tests/setup/admin/.env .
+RUN cp rill /usr/local/bin
+RUN chmod 777 /usr/local/bin/rill
+
+#RUN rill runtime install-duckdb-extensions
+
+ENTRYPOINT ["rill"]
+CMD ["runtime", "start"]