build-docker-images.yml

name: Build docker images (scheduled)

on:
  push:
    branches:
      - build_ci_docker_image*
  repository_dispatch:
  workflow_call:
    inputs:
      image_postfix:
        required: true
        type: string
  schedule:
    - cron: "17 0 * * *"

concurrency:
  group: docker-images-builds
  cancel-in-progress: false

jobs:
  latest-docker:
    name: "Latest PyTorch + TensorFlow [dev]"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-all-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-all-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-all-latest-gpu-push-ci

  latest-torch-deepspeed-docker:
    name: "Latest PyTorch + DeepSpeed"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      - name: Cleanup disk
        run: |
          sudo ls -l /usr/local/lib/
          sudo ls -l /usr/share/
          sudo du -sh /usr/local/lib/
          sudo du -sh /usr/share/
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo du -sh /usr/local/lib/
          sudo du -sh /usr/share/
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }}

  # Can't build 2 images in a single job `latest-torch-deepspeed-docker` (for `nvcr.io/nvidia`)
  latest-torch-deepspeed-docker-for-push-ci-daily-build:
    name: "Latest PyTorch + DeepSpeed (Push CI - Daily Build)"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      - name: Cleanup disk
        run: |
          sudo ls -l /usr/local/lib/
          sudo ls -l /usr/share/
          sudo du -sh /usr/local/lib/
          sudo du -sh /usr/share/
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo du -sh /usr/local/lib/
          sudo du -sh /usr/share/
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci

  doc-builder:
    name: "Doc builder"
    # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-doc-builder
          push: true
          tags: huggingface/transformers-doc-builder

  latest-pytorch:
    name: "Latest PyTorch [dev]"
    # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      - name: Cleanup disk
        run: |
          sudo ls -l /usr/local/lib/
          sudo ls -l /usr/share/
          sudo du -sh /usr/local/lib/
          sudo du -sh /usr/share/
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo du -sh /usr/local/lib/
          sudo du -sh /usr/share/
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-gpu

  latest-pytorch-amd:
    name: "Latest PyTorch (AMD) [dev]"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      - 
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - 
        name: Check out code
        uses: actions/checkout@v4
      - 
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - 
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-amd-gpu${{ inputs.image_postfix }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-amd-gpu-push-ci

  latest-tensorflow:
    name: "Latest TensorFlow [dev]"
    # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-tensorflow-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-tensorflow-gpu

  latest-pytorch-deepspeed-amd:
    name: "PyTorch + DeepSpeed (AMD) [dev]"
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      - 
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - 
        name: Check out code
        uses: actions/checkout@v4
      - 
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - 
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-amd-gpu${{ inputs.image_postfix }}
      # Push CI images still need to be re-built daily
      -
        name: Build and push (for Push CI) in a daily basis
        # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
        # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
        if: inputs.image_postfix != '-push-ci'
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-pytorch-deepspeed-amd-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-pytorch-deepspeed-amd-gpu-push-ci

  latest-quantization-torch-docker:
    name: "Latest Pytorch + Quantization [dev]"
     # Push CI doesn't need this image
    if: inputs.image_postfix != '-push-ci'
    runs-on: [intel-cpu, 8-cpu, ci]
    steps:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Check out code
        uses: actions/checkout@v4
      -
        name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      -
        name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: ./docker/transformers-quantization-latest-gpu
          build-args: |
            REF=main
          push: true
          tags: huggingface/transformers-quantization-latest-gpu${{ inputs.image_postfix }}