Removes separate DB Migrations Container

.github/workflows/artifact-release.yml

@@ -53,53 +53,6 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
-  docker-backend-migrations:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      id-token: write
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/nucleuscloud/neosync/api-migrations
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=schedule
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=semver,pattern={{raw}}
-            type=sha
-
-      - name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Depot CLI
-        uses: depot/setup-action@v1
-
-      - name: Build and push
-        uses: depot/build-push-action@v1
-        with:
-          file: docker/Dockerfile.migrations
-          context: backend
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
   docker-frontend:
     runs-on: ubuntu-latest
     permissions:

backend/Tiltfile

@@ -16,13 +16,15 @@ secret_settings(disable_scrub=True)
 namespace_create('neosync', allow_duplicates=True)
 
 local_resource(
-  'neosync-api-protos',
+  'neosync-api-gen',
   'hack/gen.sh',
   deps=[
     'protos',
     'buf.gen.yaml',
     'buf.work.yaml',
     'buf-es.gen.yaml',
+    'sql/postgresql/models',
+    'sql/postgresql/queries',
   ],
   labels=['build']
 )
@@ -43,21 +45,14 @@ local_resource(
     'charts',
     'Makefile',
     'Dockerfile',
-    'Dockerfile.migrations',
     'buf.gen.yaml',
     'buf.work.yaml',
     'buf-es.gen.yaml',
-    'migrations',
+    'sql/postgresql/schema',
   ],
   labels=['build']
 )
 
-docker_build(
-  ref='db-migration',
-  context='.',
-  dockerfile='../docker/Dockerfile.migrations',
-  only=['hack/', 'migrations/', 'sql'],
-)
 k8s_yaml(kustomize('./dev/yaml'))
 k8s_yaml(helmfile('./dev/helm/db', helmfileDir=base_dir+'/dev/helm/db'))
 k8s_resource('neosync-postgresql', labels=['db'], port_forwards=5432)
@@ -87,11 +82,12 @@ k8s_resource(
 docker_build_with_restart(
   ref='neosync-api',
   context='.',
-  entrypoint=['/app/mgmt', 'serve', 'connect'],
+  entrypoint=['/app/mgmt'],
   dockerfile='./dev/build/Dockerfile.dev',
-  only=['.env.dev', 'bin'],
+  only=['.env.dev', 'bin', 'sql'],
   live_update=[
     sync('bin', '/app'),
+    sync('sql/postgresql/schema', '/app/migrations'),
     sync('.env.dev', '/app'),
   ],
 )

backend/charts/api/templates/db-migrations-env-vars.yaml

@@ -5,8 +5,15 @@ metadata:
     namespace: {{ .Release.Namespace }}
 type: Opaque
 stringData:
-    SKIP_SCHEMA_CREATION: {{ .Values.migrations.skipSchemaCreation | quote }}
+    # SKIP_SCHEMA_CREATION: {{ .Values.migrations.skipSchemaCreation | quote }}
+
+    DB_HOST: {{ .Values.migrations.db.host }}
+    DB_PORT: {{ .Values.migrations.db.port | quote }}
+    DB_NAME: {{ .Values.migrations.db.name }}
+    DB_USER: {{ .Values.migrations.db.username }}
+    DB_PASS: {{ .Values.migrations.db.password }}
+    DB_SSL_DISABLE: {{ .Values.migrations.db.disableSsl | quote}}
+
+    DB_SCHEMA_DIR: {{ .Values.migrations.db.schemaDir }}
+
 
-    PG_USERNAME: {{ .Values.migrations.db.username }}
-    PG_PASSWORD: {{ .Values.migrations.db.password }}
-    PG_HOSTNAME: {{ .Values.migrations.db.host }}

backend/charts/api/templates/deployment.yaml

@@ -49,8 +49,11 @@ spec:
 
       initContainers:
         - name: db-migration
-          image: '{{ .Values.migrations.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion  }}'
+          image: '{{ .Values.image.repository | default "ghcr.io/nucleuscloud/neosync/backend" }}:{{ .Values.image.tag | default .Chart.AppVersion  }}'
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - migrate
+            - up
 
           envFrom:
             - secretRef:

backend/charts/api/values.yaml

@@ -25,15 +25,17 @@ db:
   port: 5432
   username:
   password:
+  disableSsl: false
 
 migrations:
-  skipSchemaCreation:
-  image:
-    repository:
   db:
     host:
+    name:
+    port: 5432
     username:
     password:
+    disableSsl: false
+    schemaDir: /app/migrations
 
 istio:
   enabled: false

backend/dev/build/Dockerfile.dev

@@ -6,6 +6,7 @@ RUN mkdir /app
 WORKDIR /app
 
 COPY /bin/mgmt /app
+COPY /sql/postgresql/schema/ /app/migrations/
 
 RUN echo 'NUCLEUS_ENV=dev' > /app/.env
 

backend/dev/helm/api/values.yaml

@@ -13,13 +13,14 @@ db:
   disableSsl: true
 
 migrations:
-  skipSchemaCreation: false
-  image:
-    repository: db-migration
   db:
-    host: neosync-postgresql:5432/nucleus
+    host: neosync-postgresql
+    name: nucleus
+    port: 5432
     username: postgres
     password: foofar
+    disableSsl: true
+    schemaDir: /app/migrations
 
 terminationGracePeriodSeconds: 300
 shutdownTimeoutSeconds: 5

backend/go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.17.0
 	github.com/stretchr/testify v1.8.4
+	github.com/zeebo/assert v1.3.0
 	go.temporal.io/api v1.25.0
 	go.temporal.io/sdk v1.25.1
 	golang.org/x/net v0.17.0

backend/go.sum

@@ -327,6 +327,7 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=
 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@@ -749,6 +750,7 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -1280,6 +1282,7 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

backend/hack/migrate.sh

@@ -42,7 +42,10 @@ if [ "$PG_HOSTNAME" = "db:5432/nucleus" ]; then
     PG_OPTIONS="${PG_OPTIONS}&sslmode=disable"
 fi
 
-SCRIPT_DIR=$(basename "$0")
+SCRIPT_DIR=$(dirname "$0")
+
+DEFAULT_SCHEMA_PATH="${SCRIPT_DIR}/../sql/postgresql/schema"
+SCHEMA_PATH="${MIGRATIONS_DIR:-$DEFAULT_SCHEMA_PATH}"
 
 if [ -z "$1" ]; then
     echo "Must specify up or down; you can optionally pass in number of steps as well, i.e., 'migrate up 1'"
@@ -84,4 +87,4 @@ debug "PG_CONNECT_STR: ${PG_CONNECT_STR}"
 
 # make stderr go through stdout so it doesn't end up an err in datadog
 # shellcheck disable=2086
-migrate -path "${SCRIPT_DIR}/../sql/postgresql/schema" -database "${PG_CONNECT_STR}" "$@" 2>&1
+migrate -path "$SCHEMA_PATH" -database "${PG_CONNECT_STR}" "$@" 2>&1

backend/internal/cmds/mgmt/mgmt.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	migrate_cmd "github.com/nucleuscloud/neosync/backend/internal/cmds/mgmt/migrate"
 	serve "github.com/nucleuscloud/neosync/backend/internal/cmds/mgmt/serve"
 )
 
@@ -25,6 +26,7 @@ func Execute() {
 
 	// Wire up subcommands here
 	rootCmd.AddCommand(serve.NewCmd())
+	rootCmd.AddCommand(migrate_cmd.NewCmd())
 
 	cobra.CheckErr(rootCmd.Execute())
 }

backend/internal/cmds/mgmt/migrate/migrate.go

@@ -0,0 +1,19 @@
+package migrate_cmd
+
+import (
+	up_cmd "github.com/nucleuscloud/neosync/backend/internal/cmds/mgmt/migrate/up"
+	"github.com/spf13/cobra"
+)
+
+func NewCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "migrate",
+		Short: "Parent command for migrations",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Help()
+		},
+	}
+
+	cmd.AddCommand(up_cmd.NewCmd())
+	return cmd
+}