From bc68adf8946e9d610020dee6cc8cf1d4b34b2f86 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 10:08:28 +0100 Subject: [PATCH] feat: CI speedup --- .github/workflows/pytest.yml | 2 +- tests/conftest.py | 49 ++++++++++++++++++++++++++---------- tests/requirements.txt | 1 + 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d7a66bdc..8ce1a21c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -26,7 +26,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Test with pytest and generate coverage report run: | - python -m pytest --cov=application --cov-report=xml --cov-report=term-missing + python -m pytest -n auto --cov=application --cov-report=xml --cov-report=term-missing - name: Upload coverage reports to Codecov if: github.event_name == 'pull_request' && matrix.python-version == '3.12' uses: codecov/codecov-action@v5 diff --git a/tests/conftest.py b/tests/conftest.py index e1c35531..091f08b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,13 @@ ephemeral ``pg_ctl``-managed cluster in a temp directory and tears it down at the end of the session, so CI only needs Postgres *binaries* installed, not a running service. +Alembic migrations run ONCE per session (per xdist worker) into the +cluster's template database via the ``load=`` hook on ``postgresql_proc``. +Every per-test database handed out by the ``postgresql`` fixture is then +cloned from that template (``CREATE DATABASE .. TEMPLATE ..`` — a cheap +file-level copy), so each test still starts from a pristine schema at +head without replaying the migration chain. + Tests under ``tests/storage/db/`` intentionally override ``pg_conn`` in their own conftest to point at a real, long-running Postgres instance (DBngin locally, a service container in CI). Those are integration/e2e @@ -51,10 +58,31 @@ from sqlalchemy import create_engine # Postgres fixtures (ephemeral cluster via pytest-postgresql) # --------------------------------------------------------------------------- -# ``postgresql_proc`` starts a fresh ``pg_ctl`` cluster once per session. -# ``postgresql`` hands out a per-test DB on top of it. We layer our own +_ALEMBIC_INI = Path(__file__).resolve().parent.parent / "application" / "alembic.ini" + + +def _migrate_template_db(host, port, user, dbname, password) -> None: + """Run alembic ``upgrade head`` into the session's template database. + + Called once per session by pytest-postgresql's ``load=`` hook (against + the template DB, before any test runs). Runs in a subprocess so the + parent process never imports application settings with this URI cached. + """ + url = ( + f"postgresql+psycopg://{user}:{password or ''}@{host}:{port}/{dbname}" + ) + subprocess.check_call( + [sys.executable, "-m", "alembic", "-c", str(_ALEMBIC_INI), "upgrade", "head"], + timeout=120, + env={**os.environ, "POSTGRES_URI": url}, + ) + + +# ``postgresql_proc`` starts a fresh ``pg_ctl`` cluster once per session and +# migrates its template database (see ``_migrate_template_db``). ``postgresql`` +# hands out a per-test DB cloned from that template. We layer our own # SQLAlchemy engine + rolled-back transaction on top for test isolation. -postgresql_proc = factories.postgresql_proc() +postgresql_proc = factories.postgresql_proc(load=[_migrate_template_db]) postgresql = factories.postgresql("postgresql_proc") @@ -72,12 +100,13 @@ def _alembic_ini_path() -> Path: @pytest.fixture() -def pg_engine(postgresql, _alembic_ini_path, monkeypatch): +def pg_engine(postgresql, monkeypatch): """Per-test SQLAlchemy engine against a fresh ephemeral Postgres DB. - Alembic is run from scratch against the per-test database so the full - schema is present. ``POSTGRES_URI`` is patched in the environment for - the duration of the test so any code that reads it via + The database is a clone of the session's already-migrated template + (see ``_migrate_template_db``), so the full schema is present without + running alembic here. ``POSTGRES_URI`` is patched in the environment + for the duration of the test so any code that reads it via ``application.core.settings`` sees the ephemeral DB. """ url = _sqlalchemy_url(postgresql.info) @@ -89,12 +118,6 @@ def pg_engine(postgresql, _alembic_ini_path, monkeypatch): monkeypatch.setattr(settings_module.settings, "POSTGRES_URI", url, raising=False) - subprocess.check_call( - [sys.executable, "-m", "alembic", "-c", str(_alembic_ini_path), "upgrade", "head"], - timeout=60, - env={**__import__("os").environ, "POSTGRES_URI": url}, - ) - engine = create_engine(url) yield engine engine.dispose() diff --git a/tests/requirements.txt b/tests/requirements.txt index 65610d93..72ddee03 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ pytest>=8.0.0 pytest-asyncio>=0.23 pytest-cov>=4.1.0 +pytest-xdist>=3.5 coverage>=7.4.0 pytest-postgresql>=6.0.0 psycopg[binary]>=3.1