More binary builds v2 (#582)

almarklein · web-flow · commit 64ede94428dd · 2024-09-14T19:34:51.000+02:00
* More binary builds v2

* fix version

* fix archs

* lets see what happens

* fix cd script

* typo

* wut

* ok, need true platform

* comment and fix

* fix builds

* more fix

* better test

* fix test

* fix win

* try again

* safer

* add pushd

* cannot install deps with no-index

* maybe like this

* had an extta popd

* sod it

* comments

* try windows test
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -1,3 +1,16 @@
+# Github Actions script to produce binary wheels.
+#
+# Note that a lot of the cibuildwheel config is in pyproject.toml.
+#
+# We perform one build for each wheel that we generate, i.e. one per architecture.
+# In download_wgpu_native.py, we detect CIBW_PLATFORM and CIBW_ARCHS to determine
+# the required binary from wgpu-native.
+#
+# If https://github.com/pypa/cibuildwheel/issues/944 gets implemented, we can build more wheels per build.
+#
+# Also includes the sdist build that does not include a binary.
+
+
 name: CD
 
 on:
@@ -11,70 +24,75 @@ on:
 
 jobs:
 
-  # The release builds are done for the platforms that we want to build wheels for.
-  # We build wheels, test them, and then upload the wheel as an artifact.
   release-builds:
-    name: Build wheels on ${{ matrix.os }}
+    name: Build wheel for ${{ matrix.platform }} ${{ matrix.arch }}
     timeout-minutes: 10
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+          - platform: windows
+            arch: AMD64
+            os: windows-latest
+            testable: true
+          - platform: windows
+            arch: ARM64
+            os: windows-latest
+          - platform: windows
+            arch: x86
+            os: windows-latest
+          - platform: macos
+            arch: arm64
+            os: macos-latest
+            testable: true
+          - platform: macos
+            arch: x86_64
+            os: macos-13  # last Intel MacOS
+            cibw_version: '==2.16'  # delocation does not work for later versions
+          - platform: linux
+            arch: x86_64
+            os: ubuntu-latest
+            testable: true
+          - platform: linux
+            arch: aarch64
+            os: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - name: Set up QEMU
+      if: matrix.platform == 'linux' && matrix.arch == 'aarch64'
+      uses: docker/setup-qemu-action@v3
       with:
-        python-version: '3.9'
+        platforms: arm64
     - name: Install dev dependencies
       run: |
-          python -m pip install --upgrade pip wheel setuptools twine
+          python -m pip install --upgrade pip wheel setuptools twine cibuildwheel${{ matrix.cibw_version}}
     - name: Build wheels
-      # Use v2.16, v2.20 fails the osx builds
-      uses: pypa/cibuildwheel@v2.16
+      run: python -m cibuildwheel --output-dir dist
       env:
-        CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64
-        CIBW_ARCHS_LINUX: x86_64
-        CIBW_SKIP: cp39-musllinux_x86_64
-      with:
-        output-dir: dist
+        CIBW_PLATFORM: ${{ matrix.platform }}
+        CIBW_ARCHS: ${{ matrix.arch }}
     - name: Twine check
       run: |
           twine check dist/*
+    - name: Test wheel
+      if: matrix.testable
+      shell: bash
+      run: |
+          rm -rf ./wgpu
+          filename=$(ls dist/*.whl)
+          pip install $filename
+          pushd $HOME
+          python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)'
+          popd
+          pip uninstall -y wgpu
+          git reset --hard HEAD
     - name: Upload distributions
       uses: actions/upload-artifact@v4
       with:
         path: dist
-        name: ${{ matrix.os }}-build
+        name: ${{ matrix.platform }}-${{ matrix.arch }}-build
 
-  # These release builds uses QEMU so that we can build wheels for arm64.
-  # We build wheels and upload the wheel as an artifact, but we don't test them here.
-  qemu-release-builds:
-    name: Build wheels on ubuntu-latest with QEMU
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-      with:
-        platforms: arm64
-    - name: Build wheels
-      uses: pypa/cibuildwheel@v2.20
-      env:
-        CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64
-        CIBW_ARCHS_LINUX: aarch64
-        CIBW_SKIP: cp39-musllinux_aarch64
-      with:
-        output-dir: dist
-    - name: Upload distributions
-      uses: actions/upload-artifact@v4
-      with:
-        path: dist
-        name: qemu-build
 
   sdist-build:
     name: Build sdist
@@ -99,9 +117,8 @@ jobs:
       shell: bash
       run: |
           rm -rf ./wgpu
-          pushd $HOME
-          pip install $GITHUB_WORKSPACE/dist/*.tar.gz
-          popd
+          filename=$(ls dist/*.tar.gz)
+          pip install $filename
           # don't run tests, we just want to know if the sdist can be installed
           pip uninstall -y wgpu
           git reset --hard HEAD
@@ -114,10 +131,11 @@ jobs:
         path: dist
         name: sdist-build
 
+
   publish:
     name: Publish to Github and Pypi
     runs-on: ubuntu-latest
-    needs: [release-builds, qemu-release-builds, sdist-build]
+    needs: [release-builds, sdist-build]
     if: success() && startsWith(github.ref, 'refs/tags/v')
     steps:
     - uses: actions/checkout@v4
diff --git a/download-wgpu-native.py b/download-wgpu-native.py
@@ -66,7 +66,9 @@ def extract_file(zip_filename, member, path):
 
 
 def get_os_string():
-    if sys.platform.startswith("win"):
+    if os.environ.get("CIBUILDWHEEL") == "1" and os.getenv("CIBW_PLATFORM"):
+        return os.getenv("CIBW_PLATFORM")
+    elif sys.platform.startswith("win"):
         return "windows"
     elif sys.platform.startswith("darwin"):
         return "macos"
@@ -84,24 +86,46 @@ def get_arch():
     is_64_bit = sys.maxsize > 2**32
     machine = platform.machine()
 
-    # See if this is run by cibuildwheel and check to see if ARCHFLAGS is
-    # specified (only done on macOS). This allows to select the proper binaries.
-    # For specifics of CIBUILDWHEEL and macOS build envs, see:
-    # https://github.com/pypa/cibuildwheel/blob/4307b52ff28b631519d38bfa0dd09d6a9b39a81e/cibuildwheel/macos.py#L277
-    if os.environ.get("CIBUILDWHEEL") == "1" and "ARCHFLAGS" in os.environ:
-        archflags = os.environ["ARCHFLAGS"]
-        return "aarch64" if "arm64" in archflags else "x86_64"
-
     if machine == "armv7l":
         # Raspberry pi
-        return "armv7"
+        detected_arch = "armv7"
     elif is_64_bit and machine.startswith(("arm", "aarch64")):
         # Includes MacOS M1, arm linux, ...
-        return "aarch64"
+        detected_arch = "aarch64"
     elif is_64_bit:
-        return "x86_64"
+        detected_arch = "x86_64"
     else:
-        return "i686"
+        detected_arch = "i686"
+
+    if os.environ.get("CIBUILDWHEEL") == "1":
+        # When running in cibuildwheel, we derive the intended arch from
+        # an env var (the same one that cibuildwheel uses) that we set in cd.yml.
+        cibw_arch = os.getenv("CIBW_ARCHS")  # must be singular
+        if not cibw_arch:
+            # Linux builds run on Docker, so env is not visible
+            cibw_arch = detected_arch
+        elif "," in cibw_arch:
+            raise RuntimeError("CIBW_ARCHS must have a single arch")
+        arch_map = {
+            "windows": {
+                "AMD64": "x86_64",
+                "ARM64": "aarch64",
+                "x86": "i686",
+            },
+            "macos": {
+                "arm64": "aarch64",
+                "x86_64": "x86_64",
+            },
+            "linux": {
+                "x86_64": "x86_64",
+                "aarch64": "aarch64",
+                "i868": "i686",
+            },
+        }
+        maps_for_os = arch_map[get_os_string()]
+        return maps_for_os[cibw_arch]
+
+    return detected_arch
 
 
 def main(version, os_string, arch, upstream):
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,33 +1,31 @@
 [build-system]
-requires = [
-    "setuptools>=42",
-]
+requires = ["setuptools>=42"]
 build-backend = "setuptools.build_meta"
 
+
 [tool.cibuildwheel]
 # we only build on one python version since the wheels are not bound to it
-build = "cp39-*"
-
-# we can't list requests under build-system.requires because
-# that step happens _after_ the before-build command
+build = "cp312-*"
+# Print system info before build
+before-all = "uname -a"
+# Can't list requests under build-system.requires because that step happens _after_ the before-build command
 before-build = "pip install requests && python download-wgpu-native.py"
-
-# this is sufficient to trigger an install of the built wheel
+# This is sufficient to trigger an install of the built wheel
 test-command = "echo Wheel installed"
 
-# this is the minimum supported manylinux version
-manylinux-x86_64-image = "manylinux_2_24"
-manylinux-i686-image = "manylinux_2_24"
-manylinux-aarch64-image = "manylinux_2_24"
-manylinux-ppc64le-image = "manylinux_2_24"
-manylinux-s390x-image = "manylinux_2_24"
-manylinux-pypy_x86_64-image = "manylinux_2_24"
-manylinux-pypy_i686-image = "manylinux_2_24"
-manylinux-pypy_aarch64-image = "manylinux_2_24"
+[tool.cibuildwheel.windows]
+# Only for local use, overridden in cd.yml
+archs = ["amd64"]
 
 [tool.cibuildwheel.macos]
-# also create apple silicon wheels
-archs = ["x86_64", "arm64"]
+# Only for local use, overridden in cd.yml
+archs = ["arm64"]
 
-# the upstream binaries are not universal yet
-# archs = ["x86_64", "universal2", "arm64"]
+[tool.cibuildwheel.linux]
+# wgpu-native does not build for musllinux yet
+skip = "*musllinux*"
+# Use custom images, with minimal version that matches wgpu-native
+manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64"
+manylinux-aarch64-image = "quay.io/pypa/manylinux_2_28_aarch64"
+manylinux-i686-image = "quay.io/pypa/manylinux_2_28_i686"
+manylinux-ppc64le-image = "quay.io/pypa/manylinux_2_28_ppc64le"
