Merge branch 'develop'

.github/workflows/backend_docker_image.yml

@@ -48,6 +48,7 @@ jobs:
         with:
           tags: ${{ env.TAG }}
           context: .
+          target: prod
           file: docker/Dockerfile.backend
           pull: true
           push: true

.github/workflows/hdfs-tests.yml

@@ -20,19 +20,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          large-packages: true
-          docker-images: true
-          swap-storage: true
-
       - name: Cache jars
         uses: actions/cache@v4
         with:
@@ -66,6 +53,13 @@ jobs:
         run: |
           docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and hdfs"
 
+      - name: Dump worker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/syncmaster-worker
+          dest: ./logs
+
       - name: Shutdown
         if: always()
         run: |
@@ -76,3 +70,10 @@ jobs:
         with:
           name: hdfs-tests
           path: reports/*
+
+      - name: Upload worker logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: hdfs-worker-logs
+          path: logs/*

.github/workflows/hive-tests.yml

@@ -20,19 +20,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          large-packages: true
-          docker-images: true
-          swap-storage: true
-
       - name: Cache jars
         uses: actions/cache@v4
         with:
@@ -66,6 +53,13 @@ jobs:
         run: |
           docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and hive"
 
+      - name: Dump worker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/syncmaster-worker
+          dest: ./logs
+
       - name: Shutdown
         if: always()
         run: |
@@ -76,3 +70,10 @@ jobs:
         with:
           name: hive-tests
           path: reports/*
+
+      - name: Upload worker logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: hive-worker-logs
+          path: logs/*

.github/workflows/oracle-tests.yml

@@ -20,19 +20,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          large-packages: true
-          docker-images: true
-          swap-storage: true
-
       - name: Cache jars
         uses: actions/cache@v4
         with:
@@ -66,6 +53,13 @@ jobs:
         run: |
           docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and oracle"
 
+      - name: Dump worker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/syncmaster-worker
+          dest: ./logs
+
       - name: Shutdown
         if: always()
         run: |
@@ -76,3 +70,10 @@ jobs:
         with:
           name: oracle-tests
           path: reports/*
+
+      - name: Upload worker logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: oracle-worker-logs
+          path: logs/*

.github/workflows/s3-tests.yml

@@ -20,19 +20,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          large-packages: true
-          docker-images: true
-          swap-storage: true
-
       - name: Cache jars
         uses: actions/cache@v4
         with:
@@ -65,6 +52,13 @@ jobs:
         run: |
           docker compose -f ./docker-compose.test.yml exec -T worker coverage run -m pytest -vvv -s -m "worker and s3"
 
+      - name: Dump worker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/syncmaster-worker
+          dest: ./logs
+
       # This is important, as coverage is exported after receiving SIGTERM
       - name: Shutdown
         if: always()
@@ -76,3 +70,10 @@ jobs:
         with:
           name: s3-tests
           path: reports/*
+
+      - name: Upload worker logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: s3-worker-logs
+          path: logs/*

.github/workflows/tests.yml

@@ -20,6 +20,10 @@ jobs:
     name: Oracle tests
     uses: ./.github/workflows/oracle-tests.yml
 
+  hdfs_tests:
+    name: HDFS tests
+    uses: ./.github/workflows/hdfs-tests.yml
+
   hive_tests:
     name: Hive tests
     uses: ./.github/workflows/hive-tests.yml

.pre-commit-config.yaml

@@ -73,7 +73,7 @@ repos:
           - --ignore-init-module-imports
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.3.0
+    rev: 24.4.0
     hooks:
       - id: black
         language_version: python3.11

docker-compose.test.yml

@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   db:
-    image: postgres:15
+    image: postgres
     restart: unless-stopped
     env_file: .env.docker
     ports:
@@ -26,6 +26,7 @@ services:
     command: --loglevel=info -Q test_queue
     env_file: .env.docker
     volumes:
+      - ./syncmaster:/app/syncmaster
       - ./cached_jars:/root/.ivy2
       - ./reports:/app/reports
       - ./tests:/app/tests
@@ -35,14 +36,6 @@ services:
         condition: service_healthy
       rabbitmq:
         condition: service_healthy
-      test-oracle:
-        condition: service_started  # Oracle image does not have healthcheck
-      test-postgres:
-        condition: service_healthy
-      test-hive:
-        condition: service_healthy
-      test-s3:
-        condition: service_healthy
 
   backend:
     image: mtsrus/syncmaster-backend::${BACKEND_IMAGE_TAG:-test}
@@ -56,6 +49,7 @@ services:
     ports:
       - 8000:8000
     volumes:
+      - ./syncmaster:/app/syncmaster
       - ./cached_jars:/root/.ivy2
       - ./reports:/app/reports
       - ./tests:/app/tests
@@ -96,7 +90,7 @@ services:
       retries: 3
 
   test-postgres:
-    image: postgres:15
+    image: postgres
     restart: unless-stopped
     ports:
       - 5433:5432
@@ -121,7 +115,7 @@ services:
       APP_USER_PASSWORD: test_password
 
   metastore-hive:
-    image: postgres:15
+    image: postgres
     restart: unless-stopped
     environment:
       POSTGRES_DB: metastore

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   db:
-    image: postgres:15
+    image: postgres
     restart: unless-stopped
     env_file: .env.docker
     ports:
@@ -16,7 +16,7 @@ services:
       retries: 3
 
   worker:
-    image: syncmaster_worker
+    image: mtsrus/syncmaster-worker
     restart: unless-stopped
     build:
       dockerfile: docker/Dockerfile.worker
@@ -25,7 +25,7 @@ services:
     env_file: .env.docker
 
   backend:
-    image: syncmaster_back
+    image: mtsrus/syncmaster-backend
     restart: unless-stopped
     build:
       dockerfile: docker/Dockerfile.backend

docker/Dockerfile.backend

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE=python:3.11-slim
-FROM $BASE_IMAGE AS prod
+FROM $BASE_IMAGE AS base
 
 RUN apt-get update && apt-get install -y \
         libssl-dev \
@@ -14,20 +14,23 @@ RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry \
     && poetry config virtualenvs.create false
 
 WORKDIR /app
+ENV PYTHONPATH=/app
 
 COPY ./pyproject.toml ./poetry.lock* /app/
 
 RUN pip install --upgrade pip setuptools wheel packaging
-RUN poetry install --no-root --extras "backend"
-
-COPY ./syncmaster/ /app/syncmaster/
-ENV PYTHONPATH=/app
+RUN poetry install --no-root --extras "backend" --without test,docs,dev
 
 COPY ./docker/entrypoint_backend.sh /app/entrypoint.sh
 ENTRYPOINT ["/app/entrypoint.sh"]
 
 
-FROM prod as test
+FROM base AS prod
+
+COPY ./syncmaster/ /app/syncmaster/
+
+
+FROM base as test
 
-RUN poetry install --no-root --extras "backend" --with test
+RUN poetry install --no-root --extras "backend" --with test --without docs,dev
 RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh

docker/Dockerfile.worker

@@ -1,9 +1,7 @@
 ARG BASE_IMAGE=python:3.11-slim
-FROM $BASE_IMAGE AS prod
+FROM $BASE_IMAGE AS base
 
 RUN apt-get update && apt-get install -y \
-        krb5-user \
-        libkrb5-dev \
         libsasl2-dev \
         libsasl2-modules-gssapi-mit \
         libsasl2-modules-ldap \
@@ -22,24 +20,27 @@ RUN apt-get update && apt-get install -y \
 RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry && poetry config virtualenvs.create false
 
 WORKDIR /app
+ENV PYTHONPATH=/app
 
 COPY ./pyproject.toml ./poetry.lock* /app/
 
 RUN pip install --upgrade pip setuptools wheel packaging
-RUN poetry install --no-root --extras "worker"
-
-COPY ./syncmaster/ /app/syncmaster/
-ENV PYTHONPATH=/app
+RUN poetry install --no-root --extras "worker" --without test,docs,dev
 
 COPY ./docker/entrypoint_worker.sh /app/entrypoint.sh
 ENTRYPOINT ["/app/entrypoint.sh"]
 CMD ["--loglevel=info"]
 
 
-FROM prod as test
+FROM base as prod
+
+COPY ./syncmaster/ /app/syncmaster/
+
+
+FROM base as test
 
-ENV CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session.get_worker_spark_session
+ENV CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
 
 # CI runs tests in the worker container, so we need backend dependencies too
-RUN poetry install --no-root --extras "worker backend" --with test
+RUN poetry install --no-root --extras "worker backend" --with test --without docs,dev
 RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh

docs/changelog/0.1.5.rst

@@ -0,0 +1,23 @@
+0.1.5 (2024-04-22)
+==================
+
+Breaking Changes
+----------------
+
+- Pass current ``Run`` to ``CREATE_SPARK_SESSION_FUNCTION``. This allows using run/transfer/group information for Spark session options,
+  like ``appName`` or custom ones. (:issue:`38`)
+
+
+Improvements
+------------
+
+- Reduce backend image size (:issue:`44`)
+
+
+Bug Fixes
+---------
+
+- Fix 500 error in case of ``PATCH v1/connections/:id`` request with passed ``auth_data.password`` field value (:issue:`39`)
+- Do not use ``asyncio.gather`` with SQLAlchemy requests (:issue:`40`)
+- Fix 500 error while creating HDFS connection (:issue:`41`)
+- Fix missing ``options`` field from Transfer params with ``hdfs`` and ``s3`` type (:issue:`39`)