Add C++ runtime support for Whisper models (#698)

k2-fsa · Jan 14, 2025 · 0387685 · 0387685
1 parent cf0b558
commit 0387685
Show file tree

Hide file tree

Showing 51 changed files with 1,815 additions and 219 deletions.
diff --git a/.github/scripts/run-offline-whisper.sh b/.github/scripts/run-offline-whisper.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -ex
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+log "=========================================================================="
+model_list=(
+base
+base.en
+distil-large-v2
+distil-medium.en
+distil-small.en
+medium
+medium.en
+small
+small.en
+tiny
+tiny.en
+turbo
+)
+
+curl -SL -O https://hf-mirror.com/csukuangfj/sherpa-onnx-zh-wenet-aishell2/resolve/main/test_wavs/0.wav
+mv 0.wav zh.wav
+
+for m in ${model_list[@]}; do
+  d=sherpa-whisper-$m
+  log "----------testing $d----------"
+  curl -SL -O https://github.com/k2-fsa/sherpa/releases/download/asr-models/$d.tar.bz2
+  tar xvf $d.tar.bz2
+  rm $d.tar.bz2
+  ls -lh $d
+
+  if [[ $d == *en ]]; then
+    log "decode a single file"
+
+    ./build/bin/sherpa-offline \
+      --debug=1 \
+      --whisper-model=./$d/model.pt \
+      --tokens=./$d/tokens.txt \
+      ./$d/test_wavs/0.wav
+
+    log "decode two files"
+    ./build/bin/sherpa-offline \
+      --debug=1 \
+      --whisper-model=./$d/model.pt \
+      --tokens=./$d/tokens.txt \
+      ./$d/test_wavs/0.wav \
+      ./$d/test_wavs/1.wav
+  fi
+
+  if [[ $d != *en ]]; then
+
+    log "decode a single file"
+
+    ./build/bin/sherpa-offline \
+      --debug=1 \
+      --whisper-model=./$d/model.pt \
+      --tokens=./$d/tokens.txt \
+      ./$d/test_wavs/0.wav
+
+    log "decode two files"
+    ./build/bin/sherpa-offline \
+      --debug=1 \
+      --whisper-model=./$d/model.pt \
+      --tokens=./$d/tokens.txt \
+      ./$d/test_wavs/0.wav \
+      ./$d/test_wavs/1.wav
+
+    log "decode three files"
+    ./build/bin/sherpa-offline \
+      --debug=1 \
+      --whisper-model=./$d/model.pt \
+      --tokens=./$d/tokens.txt \
+      ./$d/test_wavs/0.wav \
+      ./$d/test_wavs/1.wav \
+      ./zh.wav
+  fi
+  rm -rf $d
+done
diff --git a/.github/workflows/build-doc.yml b/.github/workflows/build-doc.yml
@@ -49,7 +49,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8]
+        python-version: ["3.10"]
     steps:
       # refer to https://github.com/actions/checkout
       - uses: actions/checkout@v4

diff --git a/.github/workflows/macos-cpu-wheels.yml b/.github/workflows/macos-cpu-wheels.yml
@@ -17,7 +17,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Generating build matrix
@@ -38,12 +38,12 @@ jobs:
         ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -73,7 +73,7 @@ jobs:
           ls -lh ./wheelhouse/
 
       - name: Upload Wheel
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-macos-latest-cpu
           path: wheelhouse/*.whl

diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml
@@ -21,14 +21,14 @@ jobs:
   pypi:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: "3.10"
 
       - name: Install Python dependencies
         shell: bash

diff --git a/.github/workflows/python_style_check.yml b/.github/workflows/python_style_check.yml
@@ -44,16 +44,16 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8]
+        python-version: ["3.10"]
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/run-cpp-test.yaml b/.github/workflows/run-cpp-test.yaml
@@ -49,18 +49,17 @@ concurrency:
 
 jobs:
   run_cpp_tests:
-    if: github.event.label.name == 'ready' || github.event.label.name == 'cpp' || github.event_name == 'push'
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
         torch: ["2.1.2"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
         build_type: ["Release", "Debug"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -70,7 +69,7 @@ jobs:
           key: ${{ matrix.os }}-${{ matrix.torch }}-${{ matrix.python-version }}-${{ matrix.build_type }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -174,6 +173,11 @@ jobs:
 
           ./bin/sherpa-offline --help
 
+      - name: Run offline whisper
+        shell: bash
+        run: |
+          .github/scripts/run-offline-whisper.sh
+
       - name: Run offline sense-voice
         shell: bash
         run: |

diff --git a/.github/workflows/run-cpp-websocket-test.yaml b/.github/workflows/run-cpp-websocket-test.yaml
@@ -33,11 +33,11 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         torch: ["1.13.1"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
         build_type: ["Release", "Debug"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -47,7 +47,7 @@ jobs:
           key: ${{ matrix.os }}-${{ matrix.torch }}-${{ matrix.python-version }}-${{ matrix.build_type }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml
@@ -38,11 +38,11 @@ jobs:
         os: [ubuntu-latest]
         torch: ["1.13.1"]
         torchaudio: ["0.13.1"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
         build_type: ["Release"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -52,7 +52,7 @@ jobs:
           key: ${{ matrix.os }}-${{ matrix.torch }}-${{ matrix.python-version }}-${{ matrix.build_type }}
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml
@@ -42,16 +42,16 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ["3.10"]
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/test-offline-websocket-rtf-wer.yaml b/.github/workflows/test-offline-websocket-rtf-wer.yaml
@@ -36,12 +36,12 @@ jobs:
         os: [ubuntu-latest]
         torch: ["1.13.1"]
         torchaudio: ["0.13.1"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
         decoding_method: ["greedy_search", "modified_beam_search"]
         num_connections: [50, 100, 200]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -55,7 +55,7 @@ jobs:
           echo "Number of CPU cores:  ${{ steps.cpu-cores.outputs.count }}"
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -115,7 +115,7 @@ jobs:
           cat ./log.txt
 
       - name: Upload decoding results
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: os-${{ matrix.os }}-decoding-method-${{ matrix.decoding_method }}
           path: ./*.txt
diff --git a/.github/workflows/test-online-websocket-rtf-wer.yaml b/.github/workflows/test-online-websocket-rtf-wer.yaml
@@ -36,12 +36,12 @@ jobs:
         os: [ubuntu-latest]
         torch: ["1.13.1"]
         torchaudio: ["0.13.1"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
         decoding_method: ["greedy_search", "modified_beam_search"]
         num_connections: [50, 100, 200]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -55,7 +55,7 @@ jobs:
           echo "Number of CPU cores:  ${{ steps.cpu-cores.outputs.count }}"
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -114,7 +114,7 @@ jobs:
           cat ./log.txt
 
       - name: Upload decoding results
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: os-${{ matrix.os }}-decoding-method-${{ matrix.decoding_method }}
           path: ./*.txt
diff --git a/.github/workflows/ubuntu-cpu-wheels.yml b/.github/workflows/ubuntu-cpu-wheels.yml
@@ -17,7 +17,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Generating build matrix
@@ -38,12 +38,12 @@ jobs:
         ${{ fromJson(needs.generate_build_matrix.outputs.matrix) }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -88,7 +88,7 @@ jobs:
           ls -lh ./wheelhouse/
 
       - name: Upload Wheel
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cpu
           path: wheelhouse/*.whl

diff --git a/.github/workflows/ubuntu-cuda-wheels.yml b/.github/workflows/ubuntu-cuda-wheels.yml
@@ -17,7 +17,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Generating build matrix
@@ -52,7 +52,7 @@ jobs:
           echo "github.workspace ${{ github.workspace }}"
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -122,7 +122,7 @@ jobs:
           ls -lh ./wheelhouse/
 
       - name: Upload Wheel
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: torch-${{ matrix.torch }}-python-${{ matrix.python-version }}-ubuntu-latest-cuda
           path: wheelhouse/*.whl