[CI] Automatically refresh docker images (#4942)

kaldi-asr · Sep 16, 2024 · d9ab046 · d9ab046
1 parent 22f36e5
commit d9ab046
Show file tree

Hide file tree

Showing 8 changed files with 168 additions and 194 deletions.
diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
@@ -3,6 +3,7 @@ name: Docker Image CI
 on:
   schedule:
   - cron: '37 2 * * 1'
+
   workflow_dispatch:
     inputs:
       logLevel:
@@ -14,17 +15,20 @@ on:
           - info
           - warning
           - debug
+
 #  pull_request: #for debugging purposes
 #    branches: [ "master" ]
 
 jobs:
+
   enable_build:
-    if: github.repository == 'kaldi-asr/kaldi'
+    #if: github.repository_owner == 'jtrmal' || github.repository_owner == 'kaldi-asr'
+    if: github.repository_owner == 'kaldi-asr'
     runs-on: ubuntu-latest
     outputs:
       enabled: ${{ steps.set-enabled.outputs.enabled }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set enabled
@@ -38,22 +42,34 @@ jobs:
             enabled=true
           fi
           echo "enabled: $enabled"
-          echo "::set-output name=enabled::${enabled}"
+          echo "enabled=${enabled}" >> $GITHUB_OUTPUT
 
 
-  docker-buildx-gpu:
+  docker-buildx-gpu-12:
     needs: enable_build
     if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Maximize build space
+        uses: AdityaGarg8/[email protected]
+        with:
+          remove-android: 'true'
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          remove-large-packages: 'true'
+          remove-cached-tools: 'true'
+          remove-swapfile: 'false'
+          verbose: 'true'
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -66,22 +82,34 @@ jobs:
           echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
       - name: Build and push
         run: |
-          cd docker/ubuntu18.04-cuda10.0/
-          docker build --push --tag kaldiasr/kaldi:gpu-latest --tag kaldiasr/kaldi:gpu-ubuntu18.04-cuda10.0 --tag kaldiasr/kaldi:gpu-ubuntu18.04-cuda10.0-$(date +%F) .
+          cd docker/ubuntu22.04-cuda12
+          docker build --push  --tag kaldiasr/kaldi:gpu-latest --tag kaldiasr/kaldi:gpu-ubuntu22.04-cuda12 --tag kaldiasr/kaldi:gpu-ubuntu22.04-cuda12-$(date +%F) .
 
-  docker-buildx-cpu:
+  docker-buildx-gpu-cuda11:
     needs: enable_build
     if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Maximize build space
+        uses: AdityaGarg8/[email protected]
+        with:
+          remove-android: 'true'
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          remove-large-packages: 'true'
+          remove-cached-tools: 'true'
+          remove-swapfile: 'false'
+          verbose: 'true'
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -94,10 +122,71 @@ jobs:
           echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
       - name: Build and push
         run: |
-          cd docker/debian10-cpu/
+          cd docker/ubuntu20.04-cuda11
+          docker build --push  --tag kaldiasr/kaldi:gpu-ubuntu20.04-cuda11 --tag kaldiasr/kaldi:gpu-ubuntu20.04-cuda11-$(date +%F) .
+
+  docker-buildx-cpu-openblas:
+    needs: enable_build
+    if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+      - name: Build and push
+        run: |
+          cd docker/debian12-cpu/
           docker build --push \
                        --tag kaldiasr/kaldi:latest \
                        --tag kaldiasr/kaldi:cpu-latest  \
-                       --tag kaldiasr/kaldi:cpu-debian10 \
-                       --tag kaldiasr/kaldi:cpu-debian10-$(date +%F) .
+                       --tag kaldiasr/kaldi:cpu-latest-openblas  \
+                       --tag kaldiasr/kaldi:cpu-debian12-openblas \
+                       --tag kaldiasr/kaldi:cpu-debian12-openblas-$(date +%F) .
 
+  docker-buildx-cpu-mkl:
+    needs: enable_build
+    if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+      - name: Build and push
+        run: |
+          cd docker/debian12-cpu-mkl/
+          docker build --push \
+                       --tag kaldiasr/kaldi:cpu-latest-mkl  \
+                       --tag kaldiasr/kaldi:cpu-debian12-mkl \
+                       --tag kaldiasr/kaldi:cpu-debian12-mkl-$(date +%F) .
+ 
+
diff --git a/docker/debian10-cpu/Dockerfile → docker/debian12-cpu-mkl/Dockerfile b/docker/debian10-cpu/Dockerfile → docker/debian12-cpu-mkl/Dockerfile
@@ -1,9 +1,10 @@
-FROM debian:10
-LABEL maintainer="[email protected]"
+FROM debian:12
+LABEL maintainer="[email protected]"
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         g++ \
+        gfortran \
         make \
         automake \
         autoconf \
@@ -13,29 +14,21 @@ RUN apt-get update && \
         sox \
         libtool \
         git \
-        subversion \
-        python2.7 \
         python3 \
         zlib1g-dev \
         ca-certificates \
-        gfortran \
         patch \
-        ffmpeg \
-	vim && \
+        python-is-python3 && \
     rm -rf /var/lib/apt/lists/*
 
-RUN ln -s /usr/bin/python3 /usr/bin/python
 
 RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi #EOL
 RUN    cd /opt/kaldi/tools && \
-       ./extras/install_mkl.sh && \
-       make -j $(nproc) && \
+       ./extras/install_mkl.sh  && \
+       make -j 5 && \
        cd /opt/kaldi/src && \
        ./configure --shared && \
        make depend -j $(nproc) && \
-       make -j $(nproc) && \
-       find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \
-       find /opt/intel -type f -name "*.a" -exec rm {} \; && \
-       find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \
-       rm -rf /opt/kaldi/.git
+       make -j 5 
+
 WORKDIR /opt/kaldi/
diff --git a/docker/debian12-cpu/Dockerfile b/docker/debian12-cpu/Dockerfile
@@ -0,0 +1,34 @@
+FROM debian:12
+LABEL maintainer="[email protected]"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        g++ \
+        gfortran \
+        make \
+        automake \
+        autoconf \
+        bzip2 \
+        unzip \
+        wget \
+        sox \
+        libtool \
+        git \
+        python3 \
+        zlib1g-dev \
+        ca-certificates \
+        patch \
+        python-is-python3 && \
+    rm -rf /var/lib/apt/lists/*
+
+
+RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi #EOL
+RUN    cd /opt/kaldi/tools && \
+       ./extras/install_openblas.sh  && \
+       make -j 5 && \
+       cd /opt/kaldi/src && \
+       ./configure --shared --mathlib=OPENBLAS && \
+       make depend -j $(nproc) && \
+       make -j 5 
+
+WORKDIR /opt/kaldi/
diff --git a/docker/debian9.8-cpu/Dockerfile b/docker/debian9.8-cpu/Dockerfile
diff --git a/docker/ubuntu16.04-gpu/Dockerfile b/docker/ubuntu16.04-gpu/Dockerfile