infra: finalizing some steps before releasing (#42)

* infra: finalizing some steps before releasing Signed-off-by: Aaron <[email protected]> * chore: setup python for cibuildwheel Signed-off-by: Aaron <[email protected]> * chore: no need to setup clang since hermetic chain Signed-off-by: Aaron <[email protected]> * chore: remove maven setup Signed-off-by: Aaron <[email protected]> * chore: ok I just now learn you can install clang with pip Signed-off-by: Aaron <[email protected]> * chore: use g++ on CI Signed-off-by: Aaron <[email protected]> * chore: ok env should be set fine now Signed-off-by: Aaron <[email protected]> * chore: ok we will get them next time Signed-off-by: Aaron <[email protected]> * finals: add docstring and run yamllint Signed-off-by: Aaron <[email protected]> * fix: sanity check Signed-off-by: Aaron <[email protected]> --------- Signed-off-by: Aaron <[email protected]>
aarnphm · Mar 6, 2023 · d7afe53 · d7afe53
1 parent b37a8a0
commit d7afe53
Show file tree

Hide file tree

Showing 30 changed files with 403 additions and 189 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,5 +1,4 @@
 name: CI
-
 on:
   push:
     branches:
@@ -9,20 +8,16 @@ on:
       - main
   schedule:
     - cron: "0 6 * * 1/2" # Every other day 6AM UTC
-
 concurrency:
   group: ci-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
-
 env:
   LINES: 200
   COLUMNS: 200
-
 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
 defaults:
   run:
     shell: bash --noprofile --norc -exo pipefail {0}
-
 jobs:
   unit-tests:
     strategy:
@@ -32,12 +27,10 @@ jobs:
         python-version: ["3.8"]
         exclude:
           - os: "windows-latest"
-
     if: "!github.event.repository.fork" # Don't run on fork repository
     name: python${{ matrix.python-version }} integration tests (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 90
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/scheduled-jobs.yml b/.github/workflows/scheduled-jobs.yml
@@ -39,7 +39,6 @@ jobs:
           git submodule update --remote
           git add extern && git commit -S -m "cron: update submodules to latest commits [generated]" || true
           git push
-
   run-formatter:
     if: github.repository_owner == 'aarnphm'
     runs-on: ubuntu-latest

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -1,9 +1,7 @@
 name: style-check
-
 concurrency:
   group: style-check-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
-
 on:
   push:
     branches:
@@ -13,16 +11,13 @@ on:
       - main
   schedule:
     - cron: "0 0 * * 1/2" # Every other day 12AM UTC
-
 env:
   LINES: 200
   COLUMNS: 200
-
 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
 defaults:
   run:
     shell: bash --noprofile --norc -exo pipefail {0}
-
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -57,4 +52,5 @@ jobs:
         run: npm install -g npm@^7 pyright
       - name: Type check
         if: ${{ github.event_name == 'pull_request' }}
-        run: git diff --name-only --diff-filter=AM "origin/$GITHUB_BASE_REF" -z -- '*.py{,i}' | xargs -0 --no-run-if-empty pyright
+        run: git diff --name-only --diff-filter=AM "origin/$GITHUB_BASE_REF" -z --
+          '*.py{,i}' | xargs -0 --no-run-if-empty pyright
diff --git a/.github/workflows/update-nixpkgs.yml b/.github/workflows/update-nixpkgs.yml
@@ -4,7 +4,6 @@ on:
   schedule:
     # run this every day at 12:00AM UTC
     - cron: "0 0 1/7 * *"
-
 jobs:
   niv-updater:
     name: "Create PR for niv-managed dependencies"

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -1,5 +1,4 @@
 name: wheels
-
 on:
   workflow_dispatch:
   push:
@@ -8,20 +7,16 @@ on:
   pull_request:
     branches:
       - main
-
 concurrency:
   group: wheels-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
-
 env:
   LINES: 200
   COLUMNS: 200
-
 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
 defaults:
   run:
     shell: bash --noprofile --norc -exo pipefail {0}
-
 jobs:
   build-sdist:
     name: Build source distribution
@@ -43,19 +38,63 @@ jobs:
           pip install build
 
           python -m build --sdist
+      - name: Test built sdist
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install dist/*.tar.gz && python -c "import whispercpp as w;print(dir(w.api)); print(dir(w.audio));"
       - name: Upload to PyPI
-        if:  github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
         run: |
           pip install twine
           twine upload dist/* --repository pypi --verbose
-
+  build-arm-wheels:
+    name: Build ARM wheels for ${{ matrix.python[1] }}-${{ matrix.platform[0] }}
+    runs-on: ${{ matrix.platform[1] }}
+    if: ${{ failure() }} # Disable this for now. We will tackle this later.
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        python:
+          - ["cp38", "3.8"]
+          - ["cp39", "3.9"]
+          - ["cp310", "3.10"]
+          - ["cp311", "3.11"]
+        platform:
+          - [manylinux_aarch64, ubuntu-latest]
+          - [macosx_arm64, macos-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # fetch all tags and branches
+      - name: Setup CI
+        uses: ./.github/actions/setup-repo
+        with:
+          python-version: ${{ matrix.python[1] }}
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.platform[0] }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.python[0] }}-${{ startsWith(matrix.platform[0], 'macosx')
+            && 'macosx' || matrix.platform[0] }}
+          path: ./wheelhouse/*.whl
   build-wheel:
     name: Build wheels for python${{ matrix.python-version }} (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 20
+    timeout-minutes: 90
     if: github.repository_owner == 'aarnphm' # Don't run on fork repository
     strategy:
       fail-fast: false
@@ -65,7 +104,6 @@ jobs:
         exclude:
           - os: "windows-latest"
           - python-version: "3.11.2"
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -80,14 +118,17 @@ jobs:
         uses: docker/setup-qemu-action@v2
         with:
           platforms: all
-      - name: Set up Clang [Linux]
-        if: runner.os == 'Linux'
-        uses: egor-tensin/setup-clang@v1
       - name: Running update requirements
         run: bazel run pypi_update
       - name: Building wheels
         if: github.event_name == 'pull_request'
         run: bazel build whispercpp_wheel
+      - name: Test built wheel
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install $(bazel info bazel-bin)/*.whl && python -c "import whispercpp as w;print(dir(w.api)); print(dir(w.audio));"
       - name: Retrieving versions
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         id: get-info

diff --git a/.gitignore b/.gitignore
@@ -130,3 +130,6 @@ dmypy.json
 
 # bazel generated files
 bazel-*
+
+# bun lock file
+bun.lockb
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1,5 +1,5 @@
 load("@rules_python//python:versions.bzl", "gen_python_config_settings")
-load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")
+load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier", "buildifier_test")
 load("@pybind11_bazel//:build_defs.bzl", "pybind_extension")
 load("@rules_cc//cc:defs.bzl", "cc_library")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
@@ -19,10 +19,24 @@ exports_files([
     "LICENSE",
     "README.md",
     "README.rst",
+    "yarn.lock",
 ])
 
 buildifier(
-    name = "buildifier",
+    name = "buildfmt",
+)
+
+buildifier_test(
+    name = "buildcheck",
+    srcs = glob([
+        "**/*.bzl",
+        "**/*.bazel",
+    ]),
+)
+
+alias(
+    name = "pyright",
+    actual = "@npm//:node_modules/pyright/index.js",
 )
 
 alias(
@@ -49,7 +63,7 @@ COPTS = [
 })
 
 cc_library(
-    name = "audio",
+    name = "audio_lib",
     srcs = ["//src/whispercpp:audio.cc"],
     hdrs = [
         "//src/whispercpp:audio.h",
@@ -67,7 +81,7 @@ cc_library(
         ],
     }),
     deps = [
-        ":context",
+        ":context_lib",
         "@com_github_ggerganov_whisper//:common",
         "@com_github_libsdl_sdl2//:SDL",
         "@com_github_libsdl_sdl2//:include",
@@ -76,7 +90,7 @@ cc_library(
 )
 
 cc_library(
-    name = "context",
+    name = "context_lib",
     srcs = [
         "//src/whispercpp:context.cc",
         "//src/whispercpp:context.h",
@@ -88,27 +102,40 @@ cc_library(
     copts = COPTS,
     defines = ["BAZEL_BUILD"],
     deps = [
+        "@com_github_ggerganov_whisper//:common",
         "@com_github_ggerganov_whisper//:whisper",
         "@pybind11",
     ],
 )
 
 pybind_extension(
-    name = "api",
+    name = "audio_cpp2py_export",
     srcs = [
-        "//src/whispercpp:api_export.cc",
-        "//src/whispercpp:api_export.h",
+        "//src/whispercpp:audio.cc",
         "//src/whispercpp:audio.h",
         "//src/whispercpp:context.h",
-        "@com_github_ggerganov_whisper//:examples/common.h",
         "@com_github_ggerganov_whisper//:whisper.h",
     ],
     copts = COPTS,
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":audio",
-        ":context",
+        ":audio_lib",
+        ":context_lib",
+    ],
+)
+
+pybind_extension(
+    name = "api_cpp2py_export",
+    srcs = [
+        "//src/whispercpp:api_cpp2py_export.cc",
+        "//src/whispercpp:api_cpp2py_export.h",
+        "//src/whispercpp:context.h",
+        "@com_github_ggerganov_whisper//:examples/common.h",
+        "@com_github_ggerganov_whisper//:whisper.h",
     ],
+    copts = COPTS,
+    defines = ["BAZEL_BUILD"],
+    deps = [":context_lib"],
 )
 
 write_file(
@@ -117,14 +144,18 @@ write_file(
     content = [
         "#!/usr/bin/env bash",
         "cd $BUILD_WORKSPACE_DIRECTORY",
-        "cp -fv bazel-bin/api.so src/whispercpp/api.so",
+        "cp -fv bazel-bin/api_cpp2py_export.so src/whispercpp/api_cpp2py_export.so",
+        "cp -fv bazel-bin/audio_cpp2py_export.so src/whispercpp/audio_cpp2py_export.so",
     ],
 )
 
 sh_binary(
     name = "extensions",
     srcs = [":gen_extensions"],
-    data = [":api.so"],
+    data = [
+        ":api_cpp2py_export.so",
+        ":audio_cpp2py_export.so",
+    ],
 )
 
 # public exports
@@ -195,8 +226,5 @@ py_wheel(
         ":ci": "{BUILD_EMBED_LABEL}",
     }),
     visibility = ["//:__subpackages__"],
-    deps = [
-        ":api.so",
-        ":whispercpp_pkg",
-    ],
+    deps = [":whispercpp_pkg"],
 )
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -9,20 +9,37 @@ Start new shell:
 nix-shell
 ```
 
-We are using [bazel](https://bazel.build/) as a build system. I also provided a `tools/bazel` script to make it easier to use.
+We are using [bazel](https://bazel.build/) as a build system. I also provided a
+`tools/bazel` script to make it easier to use.
 
 Build the extension:
 
 ```bash
-./tools/bazel run //:extensions
+./tools/bazel run extensions
+```
+
+To run all format, it is most convenient to use treefmt:
+
+```bash
+nix-shell --command treefmt
+```
+
+Otherwise run `black`, `isort`, and `ruff`
+
+Whispercpp also use `pyright` for Python type check. To run it do:
+
+```bash
+./tools/bazel run //:pyright
 ```
 
 ### Testing
 
 Run tests:
 
 ```bash
-./tools/bazel test //tests/...
+./tools/bazel test tests/... examples/...
 ```
 
-> NOTE: Make sure to include the `extern/whispercpp`, `extern/pybind11/include`, and ```$(python3-config --prefix)/include/python3```  in your `CPLUS_INCLUDE_PATH` so that `clangd` can find the headers in your editor.
+> NOTE: Make sure to include the `extern/whispercpp`, `extern/pybind11/include`,
+> and `$(python3-config --prefix)/include/python3` in your `CPLUS_INCLUDE_PATH`
+> so that `clangd` can find the headers in your editor.