From 967be48c4854590035e1658d0773e859acf549b8 Mon Sep 17 00:00:00 2001 From: "Jan \"yenda\" Trmal" Date: Wed, 11 Sep 2024 10:53:46 +0200 Subject: [PATCH 1/4] upgraded docker build and github pipeline --- .github/workflows/docker-images.yml | 12 +++--- .../{debian10-cpu => debian12-cpu}/Dockerfile | 23 ++++------ docker/debian9.8-cpu/Dockerfile | 43 ------------------- 3 files changed, 14 insertions(+), 64 deletions(-) rename docker/{debian10-cpu => debian12-cpu}/Dockerfile (52%) delete mode 100644 docker/debian9.8-cpu/Dockerfile diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index d0bb01c5bf6..bc04b7a3c1f 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -14,8 +14,8 @@ on: - info - warning - debug -# pull_request: #for debugging purposes -# branches: [ "master" ] + pull_request: #for debugging purposes + branches: [ "master" ] jobs: enable_build: @@ -94,10 +94,10 @@ jobs: echo "Platforms: ${{ steps.buildx.outputs.platforms }}" - name: Build and push run: | - cd docker/debian10-cpu/ - docker build --push \ + cd docker/debian12-cpu/ + docker build \ --tag kaldiasr/kaldi:latest \ --tag kaldiasr/kaldi:cpu-latest \ - --tag kaldiasr/kaldi:cpu-debian10 \ - --tag kaldiasr/kaldi:cpu-debian10-$(date +%F) . + --tag kaldiasr/kaldi:cpu-debian12 \ + --tag kaldiasr/kaldi:cpu-debian12-$(date +%F) . diff --git a/docker/debian10-cpu/Dockerfile b/docker/debian12-cpu/Dockerfile similarity index 52% rename from docker/debian10-cpu/Dockerfile rename to docker/debian12-cpu/Dockerfile index 05079922d03..89fb44f88c0 100644 --- a/docker/debian10-cpu/Dockerfile +++ b/docker/debian12-cpu/Dockerfile @@ -1,9 +1,10 @@ -FROM debian:10 -LABEL maintainer="rick@scriptix.io" +FROM debian:12 +LABEL maintainer="jtrmal@apptek.com" RUN apt-get update && \ apt-get install -y --no-install-recommends \ g++ \ + gfortran \ make \ automake \ autoconf \ @@ -13,29 +14,21 @@ RUN apt-get update && \ sox \ libtool \ git \ - subversion \ - python2.7 \ python3 \ zlib1g-dev \ ca-certificates \ - gfortran \ patch \ - ffmpeg \ - vim && \ + python-is-python3 && \ rm -rf /var/lib/apt/lists/* -RUN ln -s /usr/bin/python3 /usr/bin/python RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi #EOL RUN cd /opt/kaldi/tools && \ - ./extras/install_mkl.sh && \ - make -j $(nproc) && \ + ./extras/install_openblas.sh && \ + make -j 5 && \ cd /opt/kaldi/src && \ ./configure --shared && \ make depend -j $(nproc) && \ - make -j $(nproc) && \ - find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - find /opt/intel -type f -name "*.a" -exec rm {} \; && \ - find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \ - rm -rf /opt/kaldi/.git + make -j 5 + WORKDIR /opt/kaldi/ diff --git a/docker/debian9.8-cpu/Dockerfile b/docker/debian9.8-cpu/Dockerfile deleted file mode 100644 index ba694d1fb96..00000000000 --- a/docker/debian9.8-cpu/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ - -FROM debian:9.8 -LABEL maintainer="mdoulaty@gmail.com" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - g++ \ - make \ - automake \ - autoconf \ - bzip2 \ - unzip \ - wget \ - sox \ - libtool \ - git \ - subversion \ - python2.7 \ - python3 \ - zlib1g-dev \ - ca-certificates \ - gfortran \ - patch \ - ffmpeg \ - vim && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python2.7 /usr/bin/python - -RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ - cd /opt/kaldi/tools && \ - ./extras/install_mkl.sh && \ - make -j $(nproc) && \ - cd /opt/kaldi/src && \ - ./configure --shared && \ - make depend -j $(nproc) && \ - make -j $(nproc) && \ - find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - find /opt/intel -type f -name "*.a" -exec rm {} \; && \ - find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \ - rm -rf /opt/kaldi/.git -WORKDIR /opt/kaldi/ - From b5ed9330b005483a215409c8d4e80d5dedb974ee Mon Sep 17 00:00:00 2001 From: "Jan \"yenda\" Trmal" Date: Wed, 11 Sep 2024 11:57:27 +0200 Subject: [PATCH 2/4] modify the gh action triggers --- .github/workflows/docker-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index bc04b7a3c1f..0e206bd5d97 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -15,11 +15,11 @@ on: - warning - debug pull_request: #for debugging purposes - branches: [ "master" ] + #branches: [ "master" ] jobs: enable_build: - if: github.repository == 'kaldi-asr/kaldi' + #if: github.repository == 'kaldi-asr/kaldi' runs-on: ubuntu-latest outputs: enabled: ${{ steps.set-enabled.outputs.enabled }} From 42d40c6e7ddbd8ba51dec2134580970c5e1fe6de Mon Sep 17 00:00:00 2001 From: "Jan \"yenda\" Trmal" Date: Wed, 11 Sep 2024 12:07:33 +0200 Subject: [PATCH 3/4] modify the gh action triggers --- .github/workflows/docker-images.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 0e206bd5d97..c60b3c37561 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -20,11 +20,12 @@ on: jobs: enable_build: #if: github.repository == 'kaldi-asr/kaldi' + if: if: github.repository_owner == 'jtrmal' || github.repository_owner == 'kaldi-asr' runs-on: ubuntu-latest outputs: enabled: ${{ steps.set-enabled.outputs.enabled }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set enabled @@ -38,7 +39,7 @@ jobs: enabled=true fi echo "enabled: $enabled" - echo "::set-output name=enabled::${enabled}" + echo "enabled=${enabled}" >> $GITHUB_OUTPUT docker-buildx-gpu: @@ -46,7 +47,7 @@ jobs: if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 @@ -74,7 +75,7 @@ jobs: if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 From f43be335a8eb81f90ea2aca8230311ce2e6cd01a Mon Sep 17 00:00:00 2001 From: "Jan \"yenda\" Trmal" Date: Wed, 11 Sep 2024 12:25:40 +0200 Subject: [PATCH 4/4] test --- docker/ubuntu16.04-gpu/ubuntu18.04-cuda10.0 | 44 ------------------ docker/ubuntu18.04-cuda10.0/Dockerfile | 44 ------------------ docker/ubuntu22.04-cuda12.2.0/Dockerfile | 46 ------------------- .../Dockerfile | 24 ++++------ 4 files changed, 9 insertions(+), 149 deletions(-) delete mode 100644 docker/ubuntu16.04-gpu/ubuntu18.04-cuda10.0 delete mode 100644 docker/ubuntu18.04-cuda10.0/Dockerfile delete mode 100644 docker/ubuntu22.04-cuda12.2.0/Dockerfile rename docker/{ubuntu16.04-gpu => ubuntu22.04-cuda12}/Dockerfile (63%) diff --git a/docker/ubuntu16.04-gpu/ubuntu18.04-cuda10.0 b/docker/ubuntu16.04-gpu/ubuntu18.04-cuda10.0 deleted file mode 100644 index 41fc78beb83..00000000000 --- a/docker/ubuntu16.04-gpu/ubuntu18.04-cuda10.0 +++ /dev/null @@ -1,44 +0,0 @@ - -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 -LABEL maintainer="mdoulaty@gmail.com" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - g++ \ - make \ - automake \ - autoconf \ - bzip2 \ - unzip \ - wget \ - sox \ - libtool \ - git \ - subversion \ - python2.7 \ - python3 \ - zlib1g-dev \ - gfortran \ - ca-certificates \ - patch \ - ffmpeg \ - vim && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python2.7 /usr/bin/python - -RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ - cd /opt/kaldi/tools && \ - ./extras/install_mkl.sh && \ - make -j $(nproc) && \ - cd /opt/kaldi/src && \ - ./configure --shared --use-cuda && \ - make depend -j $(nproc) && \ - make -j $(nproc) && \ - find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - find /opt/intel -type f -name "*.a" -exec rm {} \; && \ - find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \ - rm -rf /opt/kaldi/.git - -WORKDIR /opt/kaldi/ - diff --git a/docker/ubuntu18.04-cuda10.0/Dockerfile b/docker/ubuntu18.04-cuda10.0/Dockerfile deleted file mode 100644 index 0c75863fedd..00000000000 --- a/docker/ubuntu18.04-cuda10.0/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ - -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 -LABEL maintainer="mdoulaty@gmail.com" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - g++ \ - make \ - automake \ - autoconf \ - bzip2 \ - unzip \ - wget \ - sox \ - libtool \ - git \ - subversion \ - python2.7 \ - python3 \ - zlib1g-dev \ - gfortran \ - ca-certificates \ - patch \ - ffmpeg \ - vim && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python2.7 /usr/bin/python - -RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ - cd /opt/kaldi/tools && \ - ./extras/install_mkl.sh && \ - make -j $(nproc) && \ - cd /opt/kaldi/src && \ - ./configure --shared --use-cuda && \ - make depend -j $(nproc) && \ - make -j $(nproc) && \ - find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - find /opt/intel -type f -name "*.a" -exec rm {} \; && \ - find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \ - rm -rf /opt/kaldi/.git - -WORKDIR /opt/kaldi/ - diff --git a/docker/ubuntu22.04-cuda12.2.0/Dockerfile b/docker/ubuntu22.04-cuda12.2.0/Dockerfile deleted file mode 100644 index ae413def077..00000000000 --- a/docker/ubuntu22.04-cuda12.2.0/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 -LABEL maintainer="williamhilton.works@gmail.com" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - g++ \ - make \ - automake \ - bzip2 \ - unzip \ - wget \ - sox \ - libtool \ - git \ - subversion \ - python2.7 \ - python3 \ - zlib1g-dev \ - ca-certificates \ - gfortran \ - patch \ - ffmpeg \ - vim && \ - apt-get update && \ - apt-get install -y --no-install-recommends\ - software-properties-common && \ - apt-add-repository multiverse && \ - apt-get update && \ - yes | DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends\ - intel-mkl && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python2.7 /usr/bin/python - -RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ - cd /opt/kaldi/tools && \ - make -j $(nproc) && \ - cd /opt/kaldi/src && \ - ./configure --shared --use-cuda && \ - make depend -j $(nproc) && \ - make -j $(nproc) && \ - find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - rm -rf /opt/kaldi/.git - -WORKDIR /opt/kaldi/ diff --git a/docker/ubuntu16.04-gpu/Dockerfile b/docker/ubuntu22.04-cuda12/Dockerfile similarity index 63% rename from docker/ubuntu16.04-gpu/Dockerfile rename to docker/ubuntu22.04-cuda12/Dockerfile index 41fc78beb83..eda6152a126 100644 --- a/docker/ubuntu16.04-gpu/Dockerfile +++ b/docker/ubuntu22.04-cuda12/Dockerfile @@ -1,44 +1,38 @@ - -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 -LABEL maintainer="mdoulaty@gmail.com" +FROM nvidia/cuda:12.6.1-cudnn-devel-ubuntu22.04 +LABEL maintainer="jtrmal@apptek.com" RUN apt-get update && \ apt-get install -y --no-install-recommends \ + build-essential \ g++ \ make \ automake \ - autoconf \ bzip2 \ unzip \ wget \ - sox \ libtool \ git \ - subversion \ - python2.7 \ python3 \ zlib1g-dev \ - gfortran \ ca-certificates \ + gfortran \ patch \ - ffmpeg \ - vim && \ + software-properties-common && \ + apt-add-repository multiverse && \ + apt-get update && \ + yes | DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends\ + intel-mkl && \ rm -rf /var/lib/apt/lists/* -RUN ln -s /usr/bin/python2.7 /usr/bin/python RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ cd /opt/kaldi/tools && \ - ./extras/install_mkl.sh && \ make -j $(nproc) && \ cd /opt/kaldi/src && \ ./configure --shared --use-cuda && \ make depend -j $(nproc) && \ make -j $(nproc) && \ find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - find /opt/intel -type f -name "*.a" -exec rm {} \; && \ - find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \ rm -rf /opt/kaldi/.git WORKDIR /opt/kaldi/ -