Run examples on ci (#23)

Author: almarklein

.github/workflows/ci.yml

@@ -95,6 +95,45 @@ jobs:
       run: |
         pytest -v tests
 
+  test-examples-build:
+    name: Test examples ${{ matrix.pyversion }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            pyversion: '3.10'
+          - os: ubuntu-latest
+            pyversion: '3.12'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Install llvmpipe and lavapipe for offscreen canvas
+      run: |
+        sudo apt-get update -y -qq
+        sudo apt-get install --no-install-recommends -y libegl1-mesa-dev libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
+    - name: Install dev dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[examples]
+    - name: Show wgpu backend
+      run: |
+        python -c "from examples.tests.test_examples import adapter_summary; print(adapter_summary)"
+    - name: Test examples
+      env:
+        PYGFX_EXPECT_LAVAPIPE: true
+      run: |
+        pytest -v examples
+    - uses: actions/upload-artifact@v4
+      if: ${{ failure() }}
+      with:
+        name: screenshots{{ matrix.pyversion }}
+        path: examples/screenshots
+
   test-pyinstaller:
     name: Test pyinstaller
     runs-on: ubuntu-latest

.gitignore

@@ -2,6 +2,7 @@
 nogit/
 docs/gallery/
 docs/sg_execution_times.rst
+examples/screenshots/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -120,3 +120,5 @@ This code is distributed under the 2-clause BSD license.
 * Use `ruff check` to check for linting errors.
 * Optionally, if you install [pre-commit](https://github.com/pre-commit/pre-commit/) hooks with `pre-commit install`, lint fixes and formatting will be automatically applied on `git commit`.
 * Use `pytest tests` to run the tests.
+* Use `pytest examples` to run a subset of the examples.
+

examples/cube_auto.py

@@ -5,6 +5,8 @@
 Run a wgpu example on an automatically selected backend.
 """
 
+# run_example = true
+
 from rendercanvas.auto import RenderCanvas, run
 
 from rendercanvas.utils.cube import setup_drawing_sync

examples/noise.py

@@ -5,6 +5,8 @@
 Simple example that uses the bitmap-context to show images of noise.
 """
 
+# run_example = true
+
 import numpy as np
 from rendercanvas.auto import RenderCanvas, loop
 

examples/tests/test_examples.py

@@ -0,0 +1,119 @@
+"""
+Test that the examples run without error.
+"""
+
+import os
+import sys
+import importlib
+from pathlib import Path
+
+import imageio.v2 as iio
+import numpy as np
+import pytest
+import wgpu
+
+
+ROOT = Path(__file__).parent.parent.parent  # repo root
+examples_dir = ROOT / "examples"
+screenshots_dir = examples_dir / "screenshots"
+
+
+def find_examples(query=None, negative_query=None, return_stems=False):
+    result = []
+    for example_path in examples_dir.glob("*.py"):
+        example_code = example_path.read_text()
+        query_match = query is None or query in example_code
+        negative_query_match = (
+            negative_query is None or negative_query not in example_code
+        )
+        if query_match and negative_query_match:
+            result.append(example_path)
+    result = list(sorted(result))
+    if return_stems:
+        result = [r for r in result]
+    return result
+
+
+def get_default_adapter_summary():
+    """Get description of adapter, or None when no adapter is available."""
+    try:
+        adapter = wgpu.gpu.request_adapter_sync()
+    except RuntimeError:
+        return None  # lib not available, or no adapter on this system
+    return adapter.summary
+
+
+adapter_summary = get_default_adapter_summary()
+can_use_wgpu_lib = bool(adapter_summary)
+is_ci = bool(os.getenv("CI", None))
+
+
+is_lavapipe = adapter_summary and all(
+    x in adapter_summary.lower() for x in ("llvmpipe", "vulkan")
+)
+
+if not can_use_wgpu_lib:
+    pytest.skip("Skipping tests that need the wgpu lib", allow_module_level=True)
+
+
+# run all tests unless they opt out
+examples_to_run = find_examples(query="# run_example = true", return_stems=True)
+
+
+def import_from_path(module_name, filename):
+    spec = importlib.util.spec_from_file_location(module_name, filename)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    # With this approach the module is not added to sys.modules, which
+    # is great, because that way the gc can simply clean up when we lose
+    # the reference to the module
+    assert module.__name__ == module_name
+    assert module_name not in sys.modules
+
+    return module
+
+
+@pytest.fixture
+def force_offscreen():
+    """Force the offscreen canvas to be selected by the auto gui module."""
+    os.environ["RENDERCANVAS_FORCE_OFFSCREEN"] = "true"
+    try:
+        yield
+    finally:
+        del os.environ["RENDERCANVAS_FORCE_OFFSCREEN"]
+
+
+@pytest.mark.skipif(not os.getenv("CI"), reason="Not on CI")
+def test_that_we_are_on_lavapipe():
+    print(adapter_summary)
+    assert is_lavapipe
+
+
+@pytest.mark.parametrize("filename", examples_to_run, ids=lambda x: x.stem)
+def test_examples_compare(filename, force_offscreen):
+    """Run every example marked to compare its result against a reference screenshot."""
+    check_example(filename)
+
+
+def check_example(filename):
+    # import the example module
+    module = import_from_path(filename.stem, filename)
+
+    # render a frame
+    img = np.asarray(module.canvas.draw())
+
+    # check if _something_ was rendered
+    assert img is not None and img.size > 0
+
+    # store screenshot
+    screenshots_dir.mkdir(exist_ok=True)
+    screenshot_path = screenshots_dir / f"{filename.stem}.png"
+    iio.imsave(screenshot_path, img)
+
+
+if __name__ == "__main__":
+    # Enable tweaking in an IDE by running in an interactive session.
+    os.environ["RENDERCANVAS_FORCE_OFFSCREEN"] = "true"
+    for name in examples_to_run:
+        check_example(name)

pyproject.toml

@@ -25,7 +25,7 @@ jupyter = ["jupyter_rfb>=0.4.2"]
 glfw = ["glfw>=1.9"]
 # For devs / ci
 lint = ["ruff", "pre-commit"]
-examples = ["numpy", "wgpu", "glfw", "pyside6"]
+examples = ["numpy", "wgpu", "glfw", "pyside6", "imageio", "pytest"]
 docs = ["sphinx>7.2", "sphinx_rtd_theme", "sphinx-gallery", "numpy", "wgpu"]
 tests = ["pytest", "numpy", "wgpu", "glfw"]
 dev = ["rendercanvas[lint,tests,examples,docs]"]