CI: Split CI tests to run them in parallel (huggingface#1691)

* Split CI tests to run them in parallel

The idea of this PR is to split tests into multiple sets that can be run
in parallel by GH. For this, all tests in test_models.py that would run
on GH get a pytest marker. The GH workflow matrix is factorized to run
only a single marker. That way, only a subset of tests should run per
worker, leading to quicker results.

There is also a worker that runs all the tests that are not inside
test_models.py.

* [skip ci] empty commit to abort ci

* Fix typo in marker name

* Split fx into forward and backward

* Comment out test coverage for now

Checking if it's responsible for regression in CI runtime.

* Remove pytest cov completely from requirements

* Remove cov call in pyproject.toml

Missed that one.

Author: BenjaminBossan

.github/workflows/tests.yml

@@ -19,6 +19,7 @@ jobs:
         python: ['3.10']
         torch: ['1.13.0']
         torchvision: ['0.14.0']
+        testmarker: ['-k "not test_models"', '-m base', '-m cfg', '-m torchscript', '-m features', '-m fxforward', '-m fxbackward']
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -54,10 +55,10 @@ jobs:
         PYTHONDONTWRITEBYTECODE: 1
       run: |
         pytest -vv tests
-    - name: Run tests on Linux / Mac
+    - name: Run '${{ matrix.testmarker }}' tests on Linux / Mac
       if: ${{ !startsWith(matrix.os, 'windows') }}
       env:
         LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
         PYTHONDONTWRITEBYTECODE: 1
       run: |
-        pytest -vv --forked --durations=0 tests
+        pytest -vv --forked --durations=0 ${{ matrix.testmarker }} tests

pyproject.toml

@@ -1,9 +1,11 @@
 [tool.pytest.ini_options]
-addopts = "--cov=timm --cov-report=term-missing"
-
-[tool.coverage.run]
-omit = [
-    "tests/test_*.py",
+markers = [
+    "base: marker for model tests using the basic setup",
+    "cfg: marker for model tests checking the config",
+    "torchscript: marker for model tests using torchscript",
+    "features: marker for model tests checking feature extraction",
+    "fxforward: marker for model tests using torch fx (only forward)",
+    "fxbackward: marker for model tests using torch fx (only backward)",
 ]
 
 [tool.black]

requirements-dev.txt

@@ -3,4 +3,3 @@ pytest-timeout
 pytest-xdist
 pytest-forked
 expecttest
-pytest-cov

tests/test_models.py

@@ -1,3 +1,16 @@
+"""Run tests for all models
+
+Tests that run on CI should have a specific marker, e.g. @pytest.mark.base. This
+marker is used to parallelize the CI runs, with one runner for each marker.
+
+If new tests are added, ensure that they use one of the existing markers
+(documented in pyproject.toml > pytest > markers) or that a new marker is added
+for this set of tests. If using a new marker, adjust the test matrix in
+.github/workflows/tests.yml to run tests with this new marker, otherwise the
+tests will be skipped on CI.
+
+"""
+
 import pytest
 import torch
 import platform
@@ -83,6 +96,7 @@ def _get_input_size(model=None, model_name='', target=None):
     return input_size
 
 
+@pytest.mark.base
 @pytest.mark.timeout(120)
 @pytest.mark.parametrize('model_name', list_models(exclude_filters=EXCLUDE_FILTERS))
 @pytest.mark.parametrize('batch_size', [1])
@@ -101,6 +115,7 @@ def test_model_forward(model_name, batch_size):
     assert not torch.isnan(outputs).any(), 'Output included NaNs'
 
 
+@pytest.mark.base
 @pytest.mark.timeout(120)
 @pytest.mark.parametrize('model_name', list_models(exclude_filters=EXCLUDE_FILTERS, name_matches_cfg=True))
 @pytest.mark.parametrize('batch_size', [2])
@@ -128,6 +143,7 @@ def test_model_backward(model_name, batch_size):
     assert not torch.isnan(outputs).any(), 'Output included NaNs'
 
 
+@pytest.mark.cfg
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize('model_name', list_models(exclude_filters=NON_STD_FILTERS, include_tags=True))
 @pytest.mark.parametrize('batch_size', [1])
@@ -190,6 +206,7 @@ def test_model_default_cfgs(model_name, batch_size):
         assert fc + ".weight" in state_dict.keys(), f'{fc} not in model params'
 
 
+@pytest.mark.cfg
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize('model_name', list_models(filter=NON_STD_FILTERS, exclude_filters=NON_STD_EXCLUDE_FILTERS, include_tags=True))
 @pytest.mark.parametrize('batch_size', [1])
@@ -274,6 +291,7 @@ def test_model_features_pretrained(model_name, batch_size):
 ]
 
 
+@pytest.mark.torchscript
 @pytest.mark.timeout(120)
 @pytest.mark.parametrize(
     'model_name', list_models(exclude_filters=EXCLUDE_FILTERS + EXCLUDE_JIT_FILTERS, name_matches_cfg=True))
@@ -303,6 +321,7 @@ def test_model_forward_torchscript(model_name, batch_size):
     EXCLUDE_FEAT_FILTERS += ['*resnext101_32x32d', '*resnext101_32x16d']
 
 
+@pytest.mark.features
 @pytest.mark.timeout(120)
 @pytest.mark.parametrize('model_name', list_models(exclude_filters=EXCLUDE_FILTERS + EXCLUDE_FEAT_FILTERS, include_tags=True))
 @pytest.mark.parametrize('batch_size', [1])
@@ -379,6 +398,7 @@ def _create_fx_model(model, train=False):
         ]
 
 
+    @pytest.mark.fxforward
     @pytest.mark.timeout(120)
     @pytest.mark.parametrize('model_name', list_models(exclude_filters=EXCLUDE_FILTERS + EXCLUDE_FX_FILTERS))
     @pytest.mark.parametrize('batch_size', [1])
@@ -412,6 +432,7 @@ def test_model_forward_fx(model_name, batch_size):
         assert not torch.isnan(outputs).any(), 'Output included NaNs'
 
 
+    @pytest.mark.fxbackward
     @pytest.mark.timeout(120)
     @pytest.mark.parametrize('model_name', list_models(
         exclude_filters=EXCLUDE_FILTERS + EXCLUDE_FX_FILTERS, name_matches_cfg=True))