Docker build frontend layer improvements (#1904)

* Improve frontend layer size and dep logging - test

* add temp build image for branch

* try to compress layers

* clean apt-get post-install

* Patch collector layer to preserve cache

* prep merge

.dockerignore

@@ -4,7 +4,6 @@
 **/server/storage/*.db
 **/server/storage/lancedb
 **/collector/hotdir/**
-**/collector/v-env/**
 **/collector/outputs/**
 **/node_modules/
 **/dist/
@@ -13,6 +12,7 @@
 **/.env
 **/.env.*
 **/bundleinspector.html
-!docker/.env.example
-!frontend/.env.production
 **/tmp/**
+**/.log
+!docker/.env.example
+!frontend/.env.production

docker/Dockerfile

@@ -27,7 +27,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     apt-get install -yq --no-install-recommends nodejs && \
     curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \
         && dpkg -i yarn_1.22.19_all.deb \
-        && rm yarn_1.22.19_all.deb
+        && rm yarn_1.22.19_all.deb && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Create a group and user with specific UID and GID
 RUN groupadd -g "$ARG_GID" anythingllm && \
@@ -85,7 +87,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     apt-get install -yq --no-install-recommends nodejs && \
     curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \
         && dpkg -i yarn_1.22.19_all.deb \
-        && rm yarn_1.22.19_all.deb
+        && rm yarn_1.22.19_all.deb && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Create a group and user with specific UID and GID
 RUN groupadd -g "$ARG_GID" anythingllm && \
@@ -112,55 +116,57 @@ RUN echo "Running common build flow of AnythingLLM image for all architectures"
 USER anythingllm
 WORKDIR /app
 
-# Install frontend dependencies
-FROM build AS frontend-deps
-
-COPY ./frontend/package.json ./frontend/yarn.lock ./frontend/
+# Install & Build frontend layer
+FROM build AS frontend-build
+COPY --chown=anythingllm:anythingllm ./frontend /app/frontend/
 WORKDIR /app/frontend
 RUN yarn install --network-timeout 100000 && yarn cache clean
+RUN yarn build && \
+    cp -r dist /tmp/frontend-build && \
+    rm -rf * && \
+    cp -r /tmp/frontend-build dist && \
+    rm -rf /tmp/frontend-build
 WORKDIR /app
 
-# Install server dependencies
-FROM build AS server-deps
-COPY ./server/package.json ./server/yarn.lock ./server/
+# Install server layer & build node-llama-cpp
+FROM build AS server-build
+COPY ./server /app/server/
 WORKDIR /app/server
 RUN yarn install --production --network-timeout 100000 && yarn cache clean
 WORKDIR /app
 
 # Compile Llama.cpp bindings for node-llama-cpp for this operating system.
+# Creates appropriate bindings for the OS
 USER root
 WORKDIR /app/server
 RUN npx --no node-llama-cpp download
 WORKDIR /app
 USER anythingllm
 
-# Build the frontend
-FROM frontend-deps AS build-stage
-COPY ./frontend/ ./frontend/
-WORKDIR /app/frontend
-RUN yarn build && yarn cache clean && rm -rf node_modules
-WORKDIR /app
-
-# Setup the server
-FROM server-deps AS production-stage
-COPY --chown=anythingllm:anythingllm ./server/ ./server/
-
-# Copy built static frontend files to the server public directory
-COPY --chown=anythingllm:anythingllm --from=build-stage /app/frontend/dist ./server/public
-
-# Copy the collector
-COPY --chown=anythingllm:anythingllm ./collector/ ./collector/
-
-# Install collector dependencies
+# Build collector deps (this also downloads proper chrome for collector in /app/.cache so that needs to be
+# transferred properly in prod-build stage.
+FROM build AS collector-build
+COPY ./collector /app/collector
 WORKDIR /app/collector
 ENV PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public 
 RUN yarn install --production --network-timeout 100000 && yarn cache clean
+WORKDIR /app
 
-# Migrate and Run Prisma against known schema
-WORKDIR /app/server
-RUN npx prisma generate --schema=./prisma/schema.prisma && \
-    npx prisma migrate deploy --schema=./prisma/schema.prisma
+FROM build AS production-build
 WORKDIR /app
+# Copy the server 
+COPY --chown=anythingllm:anythingllm --from=server-build /app/server/ /app/server/
+# Copy built static frontend files to the server public directory
+COPY --chown=anythingllm:anythingllm --from=frontend-build /app/frontend/dist /app/server/public
+# Copy the collector
+COPY --chown=anythingllm:anythingllm --from=collector-build /app/collector/ /app/collector/
+COPY --chown=anythingllm:anythingllm --from=collector-build /app/.cache/puppeteer /app/.cache/puppeteer
+
+# No longer needed? (deprecated)
+# WORKDIR /app/server
+# RUN npx prisma generate --schema=./prisma/schema.prisma && \
+#     npx prisma migrate deploy --schema=./prisma/schema.prisma
+# WORKDIR /app
 
 # Setup the environment
 ENV NODE_ENV=production
@@ -174,4 +180,5 @@ HEALTHCHECK --interval=1m --timeout=10s --start-period=1m \
   CMD /bin/bash /usr/local/bin/docker-healthcheck.sh || exit 1
 
 # Run the server
+# CMD ["sh", "-c", "tail -f /dev/null"] # For development: keep container open
 ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"]

docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 name: anythingllm
 
 networks:
@@ -8,8 +6,7 @@ networks:
 
 services:
   anything-llm:
-    container_name: anything-llm
-    image: anything-llm:latest
+    container_name: anythingllm
     build:
       context: ../.
       dockerfile: ./docker/Dockerfile