Skip to content
This repository was archived by the owner on Aug 30, 2025. It is now read-only.

Commit 5151e88

Browse files
nickzeleinickzelei
authored
NEOS-1360: Fixes postgres query optimizations by enabling CGO in backend/worker (#2529)
1 parent 3177ff7 commit 5151e88

File tree

12 files changed

+178
-68
lines changed

12 files changed

+178
-68
lines changed

‎backend/Makefile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ vet: ## Runs go vet
1818
build: gen vet ## Generates code, runs go vet, and builds the project
1919
scripts/build.sh
2020

21-
dbuild: gen vet ## Generates code, runs go vet, and builds the project specifically for Linux
22-
GOOS=linux scripts/build.sh
21+
dbuild: build
2322

2423
run: ## Runs the project
2524
./bin/mgmt serve connect

‎backend/dev/build/Dockerfile.dev

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,54 @@
1-
FROM alpine:3.18
1+
# Build the manager binary
2+
FROM golang:1.22-bookworm AS builder
23

3-
RUN mkdir -p .config/helm/registry
4-
RUN mkdir /app
4+
RUN apt-get update \
5+
&& DEBIAN_FRONTEND=noninteractive \
6+
apt-get install --no-install-recommends -y \
7+
build-essential
58

6-
WORKDIR /app
9+
WORKDIR /workspace
10+
# Copy the Go Modules manifests
11+
COPY go.mod go.mod
12+
COPY go.sum go.sum
713

8-
COPY /sql/postgresql/schema/ /app/migrations/
14+
RUN go mod download
915

10-
RUN echo 'NUCLEUS_ENV=dev' > /app/.env
16+
WORKDIR /workspace
17+
COPY worker/ worker/
18+
COPY internal/ internal/
1119

12-
COPY /bin/mgmt /app
20+
WORKDIR /workspace/backend
1321

14-
ENTRYPOINT ["/app/mgmt"]
22+
# Copy in generated code
23+
COPY backend/gen/ gen/
24+
25+
# Copy the go source
26+
COPY backend/cmd/ cmd/
27+
COPY backend/internal/ internal/
28+
COPY backend/services/ services/
29+
COPY backend/pkg/ pkg/
30+
COPY backend/sql sql/
31+
32+
# Build
33+
RUN go build -ldflags="-s -w" -o bin/mgmt cmd/mgmt/*.go
34+
35+
# Use debian:bullseye-slim as the base image to include necessary C libraries
36+
FROM debian:bookworm-slim
37+
WORKDIR /
38+
39+
# Install CA certificates for HTTPS connections
40+
RUN apt-get update \
41+
&& DEBIAN_FRONTEND=noninteractive \
42+
apt-get install --no-install-recommends -y ca-certificates \
43+
&& rm -rf /var/lib/apt/lists/*
44+
45+
COPY --from=builder /workspace/backend/bin/mgmt .
46+
COPY /backend/sql/postgresql/schema/ /migrations/
47+
48+
# Create a non-root user
49+
RUN useradd -u 65532 -r nonroot
50+
USER nonroot
51+
52+
ENTRYPOINT ["/mgmt"]
53+
54+
CMD ["serve", "connect"]

‎backend/scripts/build.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ VERSION=$(echo $(git describe --tags --abbrev=0 2> /dev/null || echo "untagged"
77
if [ ! -z "${TILT_HOST}" ]; then
88
# when invoked by tilt set the OS so that binary will run on linux container
99
# Note the ldflags set here are used only for local builds.
10-
CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$VERSION' -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$BUILD_DATE' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$GIT_COMMIT'" -o bin/mgmt cmd/mgmt/*.go
10+
GOOS=linux go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$VERSION' -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$BUILD_DATE' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$GIT_COMMIT'" -o bin/mgmt cmd/mgmt/*.go
1111
#CGO_ENABLED=0 GOOS=linux go build -gcflags="all=-N -l" -o bin/mgmt cmd/mgmt/*.go
1212
elif [ ! -z "${GOOS}" ]; then
13-
CGO_ENABLED=0 GOOS="${GOOS}" go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$VERSION' -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$BUILD_DATE' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$GIT_COMMIT'" -o bin/mgmt cmd/mgmt/*.go
13+
GOOS="${GOOS}" go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$VERSION' -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$BUILD_DATE' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$GIT_COMMIT'" -o bin/mgmt cmd/mgmt/*.go
1414
else
15-
CGO_ENABLED=0 go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$VERSION' -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$BUILD_DATE' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$GIT_COMMIT'" -o bin/mgmt cmd/mgmt/*.go
15+
go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$VERSION' -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$BUILD_DATE' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$GIT_COMMIT'" -o bin/mgmt cmd/mgmt/*.go
1616
#CGO_ENABLED=0 $(GO) build -gcflags="all=-N -l" -o bin/mgmt cmd/mgmt/*.go
1717
fi

‎compose.dev.yml

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,8 @@ services:
6969
container_name: neosync-api
7070
image: neosync-api
7171
build:
72-
context: ./backend
73-
dockerfile: ./dev/build/Dockerfile.dev
72+
context: ./
73+
dockerfile: ./backend/dev/build/Dockerfile.dev
7474
ports:
7575
- 8080:8080
7676
command: serve connect
@@ -80,7 +80,7 @@ services:
8080
- NUCLEUS_ENV=dev
8181
- TEMPORAL_URL=temporal:7233
8282
- DB_AUTO_MIGRATE=true
83-
- DB_SCHEMA_DIR=/app/migrations
83+
- DB_SCHEMA_DIR=/migrations
8484
- DB_HOST=db
8585
- DB_PORT=5432
8686
- DB_NAME=nucleus
@@ -97,24 +97,31 @@ services:
9797
networks:
9898
- neosync-network
9999
- temporal-network
100-
101100
depends_on:
102101
db:
103102
condition: service_healthy
104103
restart: true
104+
volumes:
105+
- neosync_backend_build_cache:/root/.cache/go-build
105106
develop:
106107
watch:
107-
- path: backend/bin
108-
action: rebuild
109-
target: /app
108+
- action: rebuild
109+
path: internal/
110+
target: /workspacse
111+
- action: rebuild
112+
path: backend/
113+
target: /workspace
114+
- action: rebuild
115+
path: worker/
116+
target: /workspace
110117
restart: on-failure
111118

112119
worker:
113120
container_name: neosync-worker
114121
image: neosync-worker
115122
build:
116-
context: ./worker
117-
dockerfile: ./dev/build/Dockerfile.dev
123+
context: ./
124+
dockerfile: ./worker/dev/build/Dockerfile.dev
118125
environment:
119126
- NUCLEUS_ENV=dev
120127
- TEMPORAL_URL=temporal:7233
@@ -127,11 +134,19 @@ services:
127134
networks:
128135
- neosync-network
129136
- temporal-network
137+
volumes:
138+
- neosync_worker_build_cache:/root/.cache/go-build
130139
develop:
131140
watch:
132-
- path: worker/bin
133-
action: rebuild
134-
target: /app
141+
- action: rebuild
142+
path: internal/
143+
target: /workspacse
144+
- action: rebuild
145+
path: worker/
146+
target: /workspace
147+
- action: rebuild
148+
path: backend/
149+
target: /workspace
135150
restart: on-failure
136151

137152
redis:
@@ -148,6 +163,8 @@ services:
148163
volumes:
149164
neosync_redis_cache:
150165
neosync_pg_data:
166+
neosync_worker_build_cache:
167+
neosync_backend_build_cache:
151168

152169
networks:
153170
neosync-network:

‎docker/Dockerfile.backend

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Build the manager binary
2-
FROM golang:1.22 as builder
2+
FROM golang:1.22-bookworm AS builder
3+
4+
RUN apt-get update \
5+
&& DEBIAN_FRONTEND=noninteractive \
6+
apt-get install --no-install-recommends -y \
7+
build-essential
38

49
WORKDIR /workspace
510
# Copy the Go Modules manifests
@@ -12,14 +17,6 @@ WORKDIR /workspace
1217
COPY worker/ worker/
1318
COPY internal/ internal/
1419

15-
# COPY tools.go tools.go
16-
# COPY Makefile Makefile
17-
# RUN make install-go-tools
18-
19-
# ENV PATH="$PATH:$(go env GOPATH)/bin"
20-
# ENV GOBIN="$PWD/bin"
21-
# ENV PATH="$PATH:$GOBIN"
22-
2320
# Define build arguments
2421
ARG buildDate="1970-01-01T00:00:00Z"
2522
ARG gitCommit=""
@@ -43,16 +40,24 @@ COPY backend/pkg/ pkg/
4340
COPY backend/sql sql/
4441

4542
# Build
46-
RUN CGO_ENABLED=0 go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$buildDate' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$gitCommit' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$gitVersion'" -o bin/mgmt cmd/mgmt/*.go
47-
48-
# Use distroless as minimal base image to package the manager binary
49-
# Refer to https://github.com/GoogleContainerTools/distroless for more details
50-
FROM gcr.io/distroless/static:nonroot
43+
RUN go build -ldflags="-s -w -X 'github.com/nucleuscloud/neosync/backend/internal/version.buildDate=$buildDate' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitCommit=$gitCommit' -X 'github.com/nucleuscloud/neosync/backend/internal/version.gitVersion=$gitVersion'" -o bin/mgmt cmd/mgmt/*.go
5144

45+
# Use debian:bullseye-slim as the base image to include necessary C libraries
46+
FROM debian:bookworm-slim
5247
WORKDIR /
48+
49+
# Install CA certificates for HTTPS connections
50+
RUN apt-get update \
51+
&& DEBIAN_FRONTEND=noninteractive \
52+
apt-get install --no-install-recommends -y ca-certificates \
53+
&& rm -rf /var/lib/apt/lists/*
54+
5355
COPY --from=builder /workspace/backend/bin/mgmt .
5456
COPY /backend/sql/postgresql/schema/ /migrations/
55-
USER 65532:65532
57+
58+
# Create a non-root user
59+
RUN useradd -u 65532 -r nonroot
60+
USER nonroot
5661

5762
ENTRYPOINT ["/mgmt"]
5863

‎docker/Dockerfile.worker

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Build the manager binary
2-
FROM golang:1.22 as builder
2+
FROM golang:1.22-bookworm AS builder
3+
4+
RUN apt-get update \
5+
&& DEBIAN_FRONTEND=noninteractive \
6+
apt-get install --no-install-recommends -y \
7+
build-essential
38

49
WORKDIR /workspace
510
# Copy the Go Modules manifests
@@ -20,14 +25,23 @@ COPY worker/internal/ internal/
2025
COPY worker/pkg/ pkg/
2126

2227
# Build
23-
RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
28+
RUN go build -ldflags="-s -w" -o bin/worker cmd/worker/*.go
2429

25-
# Use distroless as minimal base image to package the manager binary
26-
# Refer to https://github.com/GoogleContainerTools/distroless for more details
27-
FROM gcr.io/distroless/static:nonroot
30+
# Use debian:bullseye-slim as the base image to include necessary C libraries
31+
FROM debian:bookworm-slim
2832
WORKDIR /
33+
34+
# Install CA certificates for HTTPS connections
35+
RUN apt-get update \
36+
&& DEBIAN_FRONTEND=noninteractive \
37+
apt-get install --no-install-recommends -y ca-certificates \
38+
&& rm -rf /var/lib/apt/lists/*
39+
2940
COPY --from=builder /workspace/worker/bin/worker .
30-
USER 65532:65532
41+
42+
# Create a non-root user
43+
RUN useradd -u 65532 -r nonroot
44+
USER nonroot
3145

3246
ENTRYPOINT ["/worker"]
3347

‎go.mod

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ require (
5757
github.com/testcontainers/testcontainers-go/modules/redis v0.32.0
5858
github.com/toqueteos/webbrowser v1.2.0
5959
github.com/warpstreamlabs/bento v1.1.0
60-
github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc
6160
github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2
6261
github.com/zeebo/assert v1.3.1
6362
go.mongodb.org/mongo-driver v1.16.1
@@ -178,6 +177,7 @@ require (
178177
github.com/clbanning/mxj/v2 v2.7.0 // indirect
179178
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
180179
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect
180+
github.com/cockroachdb/apd v1.1.1-0.20181017181144-bced77f817b4 // indirect
181181
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
182182
github.com/cockroachdb/cockroach-go/v2 v2.1.1 // indirect
183183
github.com/containerd/containerd v1.7.18 // indirect
@@ -387,7 +387,6 @@ require (
387387
github.com/urfave/cli/v2 v2.27.1 // indirect
388388
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
389389
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
390-
github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38 // indirect
391390
github.com/x448/float16 v0.8.4 // indirect
392391
github.com/xanzy/go-gitlab v0.15.0 // indirect
393392
github.com/xdg-go/pbkdf2 v1.0.0 // indirect

‎go.sum

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -974,8 +974,9 @@ github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWH
974974
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
975975
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=
976976
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
977-
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
978977
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
978+
github.com/cockroachdb/apd v1.1.1-0.20181017181144-bced77f817b4 h1:XWEdfNxDkZI3DXXlpo0hZJ1xdaH/f3CKuZpk93pS/Y0=
979+
github.com/cockroachdb/apd v1.1.1-0.20181017181144-bced77f817b4/go.mod h1:mdGz2CnkJrefFtlLevmE7JpL2zB9tKofya/6w7wWzNA=
979980
github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
980981
github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
981982
github.com/cockroachdb/cockroach-go/v2 v2.1.1 h1:3XzfSMuUT0wBe1a3o5C0eOTcArhmmFAg2Jzh/7hhKqo=
@@ -1744,6 +1745,7 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef
17441745
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
17451746
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
17461747
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
1748+
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
17471749
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
17481750
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
17491751
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1939,10 +1941,6 @@ github.com/wI2L/jsondiff v0.4.0 h1:iP56F9tK83eiLttg3YdmEENtZnwlYd3ezEpNNnfZVyM=
19391941
github.com/wI2L/jsondiff v0.4.0/go.mod h1:nR/vyy1efuDeAtMwc3AF6nZf/2LD1ID8GTyyJ+K8YB0=
19401942
github.com/warpstreamlabs/bento v1.1.0 h1:7kc4sdtWjpv9glKG9WI1zM7ywP1LI+LIkL4rTk6PGJQ=
19411943
github.com/warpstreamlabs/bento v1.1.0/go.mod h1:rWbBdeYUJ3aKawP6J99fGZAUrTndseKt39AwMFoh/vg=
1942-
github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc h1:Hgim1Xgk1+viV7p0aZh9OOrMRfG+E4mGA+JsI2uB0+k=
1943-
github.com/wasilibs/go-pgquery v0.0.0-20240606042535-c0843d6592cc/go.mod h1:ah6UfXIl/oA0K3SbourB/UHggVJOBXwPZ2XudDmmFac=
1944-
github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38 h1:RBu75fhabyxyGJ2zhkoNuRyObBMhVeMoXqmeaPTg2CQ=
1945-
github.com/wasilibs/wazero-helpers v0.0.0-20240604052452-61d7981e9a38/go.mod h1:Z80JvMwvze8KUlVQIdw9L7OSskZJ1yxlpi4AQhoQe4s=
19461944
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
19471945
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
19481946
github.com/xanzy/go-gitlab v0.15.0 h1:rWtwKTgEnXyNUGrOArN7yyc3THRkpYcKXIXia9abywQ=

‎worker/Makefile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ vet: ## Runs go vet
1717
build: gen vet ## Generates code, runs go vet and builds the project
1818
scripts/build.sh
1919

20-
dbuild: gen vet ## Generates code, runs go vet, and builds the project specifically for Linux
21-
GOOS=linux scripts/build.sh
20+
dbuild: build
2221

2322
run: ## Runs the project
2423
./bin/sync serve

0 commit comments

Comments
 (0)