fix: incorrect time when checking concurrency (#32914) #12593
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs all of our dagster tests. | |
| name: Dagster CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| env: | |
| SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| REDIS_URL: 'redis://localhost' | |
| CLICKHOUSE_HOST: 'localhost' | |
| CLICKHOUSE_SECURE: 'False' | |
| CLICKHOUSE_VERIFY: 'False' | |
| TEST: 1 | |
| OBJECT_STORAGE_ENABLED: 'True' | |
| OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000' | |
| OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user' | |
| OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password' | |
| # tests would intermittently fail in GH actions | |
| # with exit code 134 _after passing_ all tests | |
| # this appears to fix it | |
| # absolute wild tbh https://stackoverflow.com/a/75503402 | |
| DISPLAY: ':99.0' | |
| jobs: | |
| # Job to decide if we should run dagster ci | |
| # See https://github.com/dorny/paths-filter#conditional-execution for more details | |
| changes: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| name: Determine need to run dagster checks | |
| # Set job outputs to values from filter step | |
| outputs: | |
| dagster: ${{ steps.filter.outputs.dagster }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code, but we | |
| # also want this to run on master so we need to checkout | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3 | |
| - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2 | |
| id: filter | |
| with: | |
| filters: | | |
| dagster: | |
| - 'dags/**' | |
| # Dagster depends on code from the posthog package, so ensure we run on changes to it | |
| - 'posthog/**/*' | |
| # Make sure we run if someone is explicitly change the workflow | |
| - .github/workflows/ci-dagster.yml | |
| # We use docker compose for tests, make sure we rerun on | |
| # changes to docker-compose.dev.yml e.g. dependency | |
| # version changes | |
| - docker-compose.dev.yml | |
| - frontend/public/email/* | |
| # These scripts are used in the CI | |
| - bin/check_kafka_clickhouse_up | |
| code-quality: | |
| needs: changes | |
| timeout-minutes: 30 | |
| if: needs.changes.outputs.dagster == 'true' | |
| name: Python code quality checks | |
| runs-on: depot-ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/python-code-quality | |
| with: | |
| python-version: 3.11.9 | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} | |
| dagster: | |
| name: Dagster tests | |
| needs: changes | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| clickhouse-server-image: ['clickhouse/clickhouse-server:24.8.7.41'] | |
| if: needs.changes.outputs.dagster == 'true' | |
| runs-on: depot-ubuntu-latest | |
| steps: | |
| - name: 'Checkout repo' | |
| uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Start stack with Docker Compose | |
| run: | | |
| export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ matrix.clickhouse-server-image }} | |
| docker compose -f docker-compose.dev.yml down | |
| docker compose -f docker-compose.dev.yml up -d | |
| - name: Set up Python | |
| uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182 # v5.4.1 | |
| with: | |
| enable-cache: true | |
| pyproject-file: 'pyproject.toml' | |
| - name: Install SAML (python3-saml) dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install python dependencies | |
| shell: bash | |
| run: | | |
| UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Add Kafka and ClickHouse to /etc/hosts | |
| run: sudo echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts | |
| - name: Wait for Clickhouse & Kafka | |
| run: bin/check_kafka_clickhouse_up | |
| - name: Run Dagster tests | |
| run: | | |
| pytest dags |