Skip to content

Commit 780f129

Browse files
ballPointPenguinballPointPenguin
authored
Update pip-tools and Delphi build (#2299)
* update pip-tools; remove pip version restriction; update requirements.lock * simplify Dockerfile; remove unused `IS_GITHUB_ACTION` conditional * update cypress config to not use `IS_GITHUB_ACTION` * conditionally use cpu-only torch libs in test builds
1 parent 3b09e77 commit 780f129

File tree

10 files changed

+138
-123
lines changed

10 files changed

+138
-123
lines changed

‎.github/workflows/cypress-tests.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,7 @@ jobs:
6767
- name: Build Docker images
6868
run: |
6969
# Build all services defined in test compose file
70-
# Pass IS_GITHUB_ACTION=true to build the smaller, CPU-only delphi image
71-
docker compose -f docker-compose.test.yml --env-file test.env build \
72-
--build-arg IS_GITHUB_ACTION=true
70+
docker compose -f docker-compose.test.yml --env-file test.env build
7371
7472
- name: Start services
7573
run: |

‎.github/workflows/python-ci.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,6 @@ jobs:
8080
bash -c " \
8181
echo '--- Setting up DynamoDB Tables ---'; \
8282
python create_dynamodb_tables.py --region us-east-1; \
83-
echo '--- Installing Pytest ---'; \
84-
pip install pytest; \
8583
echo '--- Running Pytest ---'; \
8684
pytest /app/tests \
8785
"

‎delphi/.pip-version‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎delphi/Dockerfile‎

Lines changed: 83 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# ---- Stage 1: Builder ----
22
FROM python:3.12-slim AS builder
33

4-
# Define a build argument to signal if running in GitHub Actions
5-
ARG IS_GITHUB_ACTION=false
4+
# Define a build argument to signal if we should install CPU-specific torch
5+
ARG USE_CPU_TORCH=false
66
ARG AWS_REGION
77

88
ENV PYTHONDONTWRITEBYTECODE=1
@@ -27,29 +27,23 @@
2727
# Copy only dependency files first - this layer is cached unless dependencies change
2828
COPY pyproject.toml requirements.lock ./
2929

30-
# Conditionally install PyTorch CPU versions if IS_GITHUB_ACTION is true
31-
RUN --mount=type=cache,target=/root/.cache/pip \
32-
if [ "$IS_GITHUB_ACTION" = "true" ]; then \
33-
echo "IS_GITHUB_ACTION is true. Installing PyTorch CPU versions..."; \
34-
pip install \
35-
--index-url https://download.pytorch.org/whl/cpu \
36-
torch==2.8.0+cpu \
37-
torchvision==0.23.0+cpu \
38-
torchaudio==2.8.0+cpu; \
39-
else \
40-
echo "IS_GITHUB_ACTION is false or not set. Skipping explicit PyTorch CPU-specific installation."; \
41-
echo "PyTorch (if required) will be installed from requirements.lock."; \
42-
fi
43-
44-
# Install dependencies from lock file (cached layer - reused unless requirements.lock changes)
45-
# BuildKit cache mount keeps pip cache between builds for faster rebuilds
46-
RUN --mount=type=cache,target=/root/.cache/pip \
47-
if [ "$IS_GITHUB_ACTION" = "true" ]; then \
48-
echo "IS_GITHUB_ACTION is true. Skipping requirements.lock (already installed CPU deps)."; \
49-
else \
50-
echo "IS_GITHUB_ACTION is false. Installing from requirements.lock..."; \
51-
pip install -r requirements.lock; \
52-
fi
30+
# Install dependencies from lock file (cached layer - reused unless requirements.lock changes)
31+
# BuildKit cache mount keeps pip cache between builds for faster rebuilds
32+
# If USE_CPU_TORCH is true, we install CPU-specific wheels and filter them out of the lockfile
33+
RUN --mount=type=cache,target=/root/.cache/pip \
34+
if [ "$USE_CPU_TORCH" = "true" ]; then \
35+
echo "USE_CPU_TORCH=true: Installing CPU-only PyTorch..." && \
36+
pip install --index-url https://download.pytorch.org/whl/cpu \
37+
torch==2.8.0+cpu \
38+
torchvision==0.23.0+cpu \
39+
torchaudio==2.8.0+cpu && \
40+
echo "Filtering standard torch packages from requirements.lock..." && \
41+
grep -vE "^(torch|torchvision|torchaudio)==" requirements.lock > requirements.filtered.lock && \
42+
pip install -r requirements.filtered.lock; \
43+
else \
44+
echo "USE_CPU_TORCH=false: Installing standard dependencies..." && \
45+
pip install -r requirements.lock; \
46+
fi
5347

5448
# ===== OPTIMIZATION: Copy source code LAST (busts cache on code changes) =====
5549
# Copy source code - this layer rebuilds when code changes but reuses dependency layers above
@@ -70,57 +64,71 @@
7064
(ls -lhR /usr/local/lib/python3.12/site-packages/nvidia || echo "NVIDIA directory not found.") && \
7165
(ls -lhR /usr/local/lib/python3.12/site-packages/torch/lib/*cuda* || echo "No CUDA libs in torch/lib.")
7266

73-
# ---- Stage 2: Final ----
74-
FROM python:3.12-slim AS final
67+
# ---- Stage 2: Final ----
68+
FROM python:3.12-slim AS final
69+
70+
ENV PYTHONDONTWRITEBYTECODE=1
71+
ENV PYTHONUNBUFFERED=1
72+
ENV AWS_REGION=${AWS_REGION:-us-east-1}
73+
74+
RUN apt-get update && \
75+
apt-get install -y --no-install-recommends \
76+
curl \
77+
&& apt-get clean && \
78+
rm -rf /var/lib/apt/lists/*
79+
80+
WORKDIR /app
81+
82+
# Copy installed packages from builder stage
83+
COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
84+
COPY --from=builder /usr/local/bin /usr/local/bin
85+
86+
# Copy source code from builder stage (already built into the package)
87+
COPY --from=builder /app/polismath/ ./polismath/
88+
COPY --from=builder /app/scripts/ ./scripts/
89+
COPY --from=builder /app/umap_narrative/ ./umap_narrative/
90+
COPY --from=builder /app/*.py ./
91+
92+
RUN mkdir -p data
93+
EXPOSE 8080
94+
# PYTHONPATH not needed since packages are properly installed via pip
95+
96+
# Make scripts executable
97+
RUN chmod +x run_delphi.py scripts/setup_ollama.sh
98+
99+
CMD ["bash", "-c", "\
100+
echo 'Ensuring DynamoDB tables are set up (runs in all environments)...'; \
101+
python create_dynamodb_tables.py --region ${AWS_REGION} && \
102+
echo 'DynamoDB table setup script finished.'; \
103+
\
104+
if [ -n \"${DYNAMODB_ENDPOINT}\" ]; then \
105+
echo 'DYNAMODB_ENDPOINT is set, assuming local/dev environment. Running additional local setup scripts...'; \
106+
echo 'Setting up MinIO bucket...' && python setup_minio.py && \
107+
echo 'Setting up Ollama model (local script)...' && ./scripts/setup_ollama.sh && \
108+
echo 'Starting job poller without ddtrace...' && \
109+
python scripts/job_poller.py --interval=2; \
110+
else \
111+
echo 'DYNAMODB_ENDPOINT is not set, assuming production-like environment. Skipping local setup scripts.'; \
112+
if [ \"${AWS_REGION}\" = \"us-east-1\" ]; then \
113+
echo 'Starting job poller WITH ddtrace in us-east-1...' && \
114+
ddtrace-run python scripts/job_poller.py --interval=2 --region ${AWS_REGION}; \
115+
else \
116+
echo 'Starting job poller WITHOUT ddtrace in other regions...' && \
117+
python scripts/job_poller.py --interval=2 --region ${AWS_REGION}; \
118+
fi; \
119+
fi \
120+
"]
75121

76-
ENV PYTHONDONTWRITEBYTECODE=1
77-
ENV PYTHONUNBUFFERED=1
78-
ENV AWS_REGION=${AWS_REGION:-us-east-1}
122+
# ---- Stage 3: Test ----
123+
# Extends final stage with dev dependencies for CI/testing
124+
FROM final AS test
79125

80-
RUN apt-get update && \
81-
apt-get install -y --no-install-recommends \
82-
curl \
83-
&& apt-get clean && \
84-
rm -rf /var/lib/apt/lists/*
126+
# Copy pyproject.toml to install dev dependencies
127+
COPY pyproject.toml .
85128

86-
WORKDIR /app
129+
# Install dev dependencies (pytest, etc.) using caching
130+
RUN --mount=type=cache,target=/root/.cache/pip \
131+
pip install --no-cache-dir ".[dev]"
87132

88-
# Copy installed packages from builder stage
89-
COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
90-
COPY --from=builder /usr/local/bin /usr/local/bin
91-
92-
# Copy source code from builder stage (already built into the package)
93-
COPY --from=builder /app/polismath/ ./polismath/
94-
COPY --from=builder /app/scripts/ ./scripts/
95-
COPY --from=builder /app/umap_narrative/ ./umap_narrative/
96-
COPY --from=builder /app/*.py ./
97-
98-
RUN mkdir -p data
99-
EXPOSE 8080
100-
# PYTHONPATH not needed since packages are properly installed via pip
101-
102-
# Make scripts executable
103-
RUN chmod +x run_delphi.py scripts/setup_ollama.sh
104-
105-
CMD ["bash", "-c", "\
106-
echo 'Ensuring DynamoDB tables are set up (runs in all environments)...'; \
107-
python create_dynamodb_tables.py --region ${AWS_REGION} && \
108-
echo 'DynamoDB table setup script finished.'; \
109-
\
110-
if [ -n \"${DYNAMODB_ENDPOINT}\" ]; then \
111-
echo 'DYNAMODB_ENDPOINT is set, assuming local/dev environment. Running additional local setup scripts...'; \
112-
echo 'Setting up MinIO bucket...' && python setup_minio.py && \
113-
echo 'Setting up Ollama model (local script)...' && ./scripts/setup_ollama.sh && \
114-
echo 'Starting job poller without ddtrace...' && \
115-
python scripts/job_poller.py --interval=2; \
116-
else \
117-
echo 'DYNAMODB_ENDPOINT is not set, assuming production-like environment. Skipping local setup scripts.'; \
118-
if [ \"${AWS_REGION}\" = \"us-east-1\" ]; then \
119-
echo 'Starting job poller WITH ddtrace in us-east-1...' && \
120-
ddtrace-run python scripts/job_poller.py --interval=2 --region ${AWS_REGION}; \
121-
else \
122-
echo 'Starting job poller WITHOUT ddtrace in other regions...' && \
123-
python scripts/job_poller.py --interval=2 --region ${AWS_REGION}; \
124-
fi; \
125-
fi \
126-
"]
133+
# Default command for test container (can be overridden)
134+
CMD ["tail", "-f", "/dev/null"]

‎delphi/Makefile‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@ venv: ## Create canonical virtual environment (delphi-env)
2828
else \
2929
echo "✓ Virtual environment already exists"; \
3030
fi
31-
@echo "Installing compatible pip version (< 25.3 for pip-tools 7.5.1 compatibility)..."
32-
@delphi-env/bin/pip install --upgrade 'pip<25.3' || echo "⚠️ Could not install pip version constraint"
3331
@echo "To activate: source delphi-env/bin/activate"
3432
@echo "Then run: make install-dev"
3533

@@ -41,15 +39,7 @@ venv-check: ## Check virtual environment status
4139
echo "Python location: $$(which python)"; \
4240
pip_version=$$(pip --version 2>/dev/null | cut -d' ' -f2); \
4341
if [ -n "$$pip_version" ]; then \
44-
major=$$(echo $$pip_version | cut -d'.' -f1); \
45-
minor=$$(echo $$pip_version | cut -d'.' -f2); \
46-
if [ $$major -eq 25 ] && [ $$minor -ge 3 ]; then \
47-
echo "⚠️ WARNING: pip version $$pip_version is >= 25.3"; \
48-
echo " pip-tools 7.5.1 supports pip up to 25.2"; \
49-
echo " Downgrade with: pip install 'pip<25.3'"; \
50-
else \
51-
echo "✓ pip version: $$pip_version (compatible)"; \
52-
fi; \
42+
echo "pip version: $$pip_version"; \
5343
fi; \
5444
else \
5545
echo "❌ No virtual environment active"; \

‎delphi/docs/DOCKER_BUILD_OPTIMIZATION.md‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -321,18 +321,43 @@ You can create custom Makefile targets for specific scenarios:
321321
```makefile
322322
# Fast dev build (reuse all caches)
323323
docker-build-dev:
324-
DOCKER_BUILDKIT=1 docker build \
325-
--target builder \
326-
-t polis/delphi:dev .
324+
DOCKER_BUILDKIT=1 docker build \
325+
--target builder \
326+
-t polis/delphi:dev .
327327
328328
# Production build (optimized final image)
329329
docker-build-prod:
330-
DOCKER_BUILDKIT=1 docker build \
331-
--target final \
332-
--no-cache \
333-
-t polis/delphi:prod .
330+
DOCKER_BUILDKIT=1 docker build \
331+
--target final \
332+
--no-cache \
333+
-t polis/delphi:prod .
334+
335+
# Test/CI build (includes dev dependencies and CPU-only torch)
336+
docker-build-test:
337+
DOCKER_BUILDKIT=1 docker build \
338+
--target test \
339+
--build-arg USE_CPU_TORCH=true \
340+
-t polis/delphi:test .
334341
```
335342

343+
## CI/CD Integration
344+
345+
The `Dockerfile` includes a `test` stage specifically for CI/CD pipelines:
346+
347+
1. **Base layers**: Can be optimized for CPU environments via `USE_CPU_TORCH=true`.
348+
2. **Test layer**: Installs additional dev dependencies from `pyproject.toml` (e.g., `pytest`).
349+
3. **Usage**:
350+
351+
```yaml
352+
services:
353+
delphi:
354+
build:
355+
context: ./delphi
356+
target: test
357+
args:
358+
USE_CPU_TORCH: "true" # Saves ~800MB by using CPU-only PyTorch wheels
359+
```
360+
336361
## References
337362

338363
- [Docker BuildKit Documentation](https://docs.docker.com/build/buildkit/)

‎delphi/mise.toml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,2 @@
11
[tools]
22
python = "3.12"
3-
# Note: pip version should be < 25.3 for compatibility with pip-tools 7.5.1
4-
# pip-tools 7.5.1 supports pip up to 25.2 (see changelog)
5-
# Install with: pip install 'pip<25.3'
6-
# Recommended: pip 25.2
7-
# See https://github.com/jazzband/pip-tools/issues/2252

‎delphi/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ dev = [
6969
# Code quality - streamlined modern stack
7070
"pre-commit>=4.3.0",
7171
# Dependency management
72-
"pip-tools>=7.5.1",
72+
"pip-tools>=7.5.2",
7373
# Type stubs
7474
"types-requests",
7575
"types-psycopg2",

0 commit comments

Comments
 (0)