NEOS-1360: enables cgo for worker to enable postgres query parsing (#…

…2523)
nucleuscloud · Aug 23, 2024 · a83e43b · a83e43b
1 parent 5e15935
commit a83e43b
Show file tree

Hide file tree

Showing 9 changed files with 90 additions and 40 deletions.
diff --git a/compose.dev.yml b/compose.dev.yml
@@ -113,8 +113,8 @@ services:
     container_name: neosync-worker
     image: neosync-worker
     build:
-      context: ./worker
-      dockerfile: ./dev/build/Dockerfile.dev
+      context: ./
+      dockerfile: ./worker/dev/build/Dockerfile.dev
     environment:
       - NUCLEUS_ENV=dev
       - TEMPORAL_URL=temporal:7233
@@ -127,11 +127,16 @@ services:
     networks:
       - neosync-network
       - temporal-network
+    volumes:
+      - neosync_worker_build_cache:/root/.cache/go-build
     develop:
       watch:
-        - path: worker/bin
-          action: rebuild
-          target: /app
+        - action: rebuild
+          path: internal/
+          target: /workspacse
+        - action: rebuild
+          path: worker/
+          target: /workspace
     restart: on-failure
 
   redis:
@@ -148,6 +153,7 @@ services:
 volumes:
   neosync_redis_cache:
   neosync_pg_data:
+  neosync_worker_build_cache:
 
 networks:
   neosync-network:

diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend
@@ -12,14 +12,6 @@ WORKDIR /workspace
 COPY worker/ worker/
 COPY internal/ internal/
 
-# COPY tools.go tools.go
-# COPY Makefile Makefile
-# RUN make install-go-tools
-
-# ENV PATH="$PATH:$(go env GOPATH)/bin"
-# ENV GOBIN="$PWD/bin"
-# ENV PATH="$PATH:$GOBIN"
-
 # Define build arguments
 ARG buildDate="1970-01-01T00:00:00Z"
 ARG gitCommit=""

diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker
@@ -1,5 +1,10 @@
 # Build the manager binary
-FROM golang:1.22 as builder
+FROM golang:1.22-bookworm AS builder
+
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y \
+    build-essential
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -22,12 +27,24 @@ COPY worker/pkg/ pkg/
 # Build
 RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
 
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
+# Use debian:bullseye-slim as the base image to include necessary C libraries
+FROM debian:bookworm-slim
 WORKDIR /
+
+# Install CA certificates for HTTPS connections
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
 COPY --from=builder /workspace/worker/bin/worker .
-USER 65532:65532
+
+# Install any runtime dependencies if needed
+# RUN apt-get update && apt-get install -y <your-dependencies> && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user
+RUN useradd -u 65532 -r nonroot
+USER nonroot
 
 ENTRYPOINT ["/worker"]
 

diff --git a/go.mod b/go.mod
@@ -57,7 +57,6 @@ require (
 	github.com/testcontainers/testcontainers-go/modules/redis v0.32.0
 	github.com/toqueteos/webbrowser v1.2.0
 	github.com/warpstreamlabs/bento v1.1.0
-	github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc
 	github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2
 	github.com/zeebo/assert v1.3.1
 	go.mongodb.org/mongo-driver v1.16.1
@@ -178,6 +177,7 @@ require (
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
 	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
 	github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect
+	github.com/cockroachdb/apd v1.1.1-0.20181017181144-bced77f817b4 // indirect
 	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
 	github.com/cockroachdb/cockroach-go/v2 v2.1.1 // indirect
 	github.com/containerd/containerd v1.7.18 // indirect
@@ -387,7 +387,6 @@ require (
 	github.com/urfave/cli/v2 v2.27.1 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/go-gitlab v0.15.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect

diff --git a/go.sum b/go.sum
@@ -974,8 +974,9 @@ github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=
 github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
-github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/cockroachdb/apd v1.1.1-0.20181017181144-bced77f817b4 h1:XWEdfNxDkZI3DXXlpo0hZJ1xdaH/f3CKuZpk93pS/Y0=
+github.com/cockroachdb/apd v1.1.1-0.20181017181144-bced77f817b4/go.mod h1:mdGz2CnkJrefFtlLevmE7JpL2zB9tKofya/6w7wWzNA=
 github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
 github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
 github.com/cockroachdb/cockroach-go/v2 v2.1.1 h1:3XzfSMuUT0wBe1a3o5C0eOTcArhmmFAg2Jzh/7hhKqo=
@@ -1744,6 +1745,7 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1939,10 +1941,6 @@ github.com/wI2L/jsondiff v0.4.0 h1:iP56F9tK83eiLttg3YdmEENtZnwlYd3ezEpNNnfZVyM=
 github.com/wI2L/jsondiff v0.4.0/go.mod h1:nR/vyy1efuDeAtMwc3AF6nZf/2LD1ID8GTyyJ+K8YB0=
 github.com/warpstreamlabs/bento v1.1.0 h1:7kc4sdtWjpv9glKG9WI1zM7ywP1LI+LIkL4rTk6PGJQ=
 github.com/warpstreamlabs/bento v1.1.0/go.mod h1:rWbBdeYUJ3aKawP6J99fGZAUrTndseKt39AwMFoh/vg=
-github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc h1:Hgim1Xgk1+viV7p0aZh9OOrMRfG+E4mGA+JsI2uB0+k=
-github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc/go.mod h1:ah6UfXIl/oA0K3SbourB/UHggVJOBXwPZ2XudDmmFac=
-github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38 h1:RBu75fhabyxyGJ2zhkoNuRyObBMhVeMoXqmeaPTg2CQ=
-github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38/go.mod h1:Z80JvMwvze8KUlVQIdw9L7OSskZJ1yxlpi4AQhoQe4s=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xanzy/go-gitlab v0.15.0 h1:rWtwKTgEnXyNUGrOArN7yyc3THRkpYcKXIXia9abywQ=

diff --git a/worker/Makefile b/worker/Makefile
@@ -17,8 +17,7 @@ vet: ## Runs go vet
 build: gen vet ## Generates code, runs go vet and builds the project
 	scripts/build.sh
 
-dbuild: gen vet ## Generates code, runs go vet, and builds the project specifically for Linux
-	GOOS=linux scripts/build.sh
+dbuild: build
 
 run: ## Runs the project
 	./bin/sync serve

diff --git a/worker/dev/build/Dockerfile.dev b/worker/dev/build/Dockerfile.dev
@@ -1,12 +1,52 @@
-FROM alpine:3.18
+# Build the manager binary
+FROM golang:1.22-bookworm as builder
 
-RUN mkdir -p .config/helm/registry
-RUN mkdir /app
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y \
+    build-essential
 
-WORKDIR /app
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
 
-RUN echo 'NUCLEUS_ENV=dev' > /app/.env
+RUN go mod download
 
-COPY /bin/worker /app
+WORKDIR /workspace
+COPY backend/ backend/
+COPY internal/ internal/
 
-ENTRYPOINT ["/app/worker", "serve"]
+WORKDIR /workspace/worker
+
+# Copy the go source
+COPY worker/cmd/ cmd/
+COPY worker/internal/ internal/
+COPY worker/pkg/ pkg/
+
+# Build
+RUN --mount=type=cache,target=/root/.cache/go-build \
+  go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
+
+# Use debian:bullseye-slim as the base image to include necessary C libraries
+FROM debian:bookworm-slim
+WORKDIR /
+
+# Install CA certificates for HTTPS connections
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /workspace/worker/bin/worker .
+
+# Install any runtime dependencies if needed
+# RUN apt-get update && apt-get install -y <your-dependencies> && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user
+RUN useradd -u 65532 -r nonroot
+USER nonroot
+
+ENTRYPOINT ["/worker"]
+
+CMD ["serve"]
diff --git a/worker/pkg/query-builder2/querybuilder.go b/worker/pkg/query-builder2/querybuilder.go
@@ -9,7 +9,6 @@ import (
 	"github.com/doug-martin/goqu/v9/sqlgen"
 	sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared"
 	pg_query "github.com/pganalyze/pg_query_go/v5"
-	pgquery "github.com/wasilibs/go-pgquery"
 	"github.com/xwb1989/sqlparser"
 )
 
@@ -261,7 +260,7 @@ func (qb *QueryBuilder) qualifyWhereCondition(schema *string, table, condition s
 }
 
 func qualifyPostgresWhereColumnNames(sql string, schema *string, table string) (string, error) {
-	tree, err := pgquery.Parse(sql)
+	tree, err := pg_query.Parse(sql)
 	if err != nil {
 		return "", err
 	}
@@ -273,7 +272,7 @@ func qualifyPostgresWhereColumnNames(sql string, schema *string, table string) (
 			updatePostgresExpr(schema, table, selectStmt.WhereClause)
 		}
 	}
-	updatedSql, err := pgquery.Deparse(tree)
+	updatedSql, err := pg_query.Deparse(tree)
 	if err != nil {
 		return "", err
 	}

diff --git a/worker/scripts/build.sh b/worker/scripts/build.sh
@@ -2,9 +2,9 @@
 
 if [ ! -z "${TILT_HOST}" ]; then
   # when invoked by tilt set the OS so that binary will run on linux container
-  CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
+  GOOS=linux go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
 elif [ ! -z "${GOOS}" ]; then
-  CGO_ENABLED=0 GOOS="${GOOS}" go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
+  GOOS="${GOOS}" go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
 else
-  CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
+  go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
 fi