Add artifacts data model (tables, migration, repository)

Introduce artifacts and artifact_versions tables: an append-only,
pass-by-reference store with a stable identity row and immutable
versions that atomically bump current_version. Access is parent-derived
(conversation or workflow run); a CHECK enforces at least one parent and
a nullable team_id is reserved for future team scoping.

Adds the 0025_artifacts migration, ArtifactsRepository with a
parent-scoped safe getter alongside the unscoped one, and repo tests.
This commit is contained in:
Alex committed 2026-06-24 10:21:48 +01:00
1 parent 1acb836332
commit 31525621e0
4 files changed
+690

No files matched your search

@@ -0,0 +1,111 @@
"""0025 artifacts — append-only artifact identity + version store.
Adds the long-anticipated ``artifacts`` entity referenced by the schema header
in ``models.py`` but never built. Two tables:
* ``artifacts`` — one identity row per logical artifact. The stable ``id`` is the
handle passed around (chat/workflow ``state``/message bodies carry only this
reference, never bytes). ``current_version`` mirrors
``workflows.current_graph_version`` (atomic increment on each new version).
Authz is parent-derived: whoever can reach the ``conversation_id`` (chat) or
``workflow_run_id`` (run) can reach its artifacts, so a CHECK requires at least
one parent. ``user_id`` is kept for ownership/quota only; ``team_id`` is a
nullable forward-compat hook for the in-flight Teams work (no FK, matching how
``teams`` itself leaves owner/member columns FK-free).
* ``artifact_versions`` — append-only, never mutated. Each edit appends a row and
bumps ``artifacts.current_version``. ``UNIQUE(artifact_id, version)`` enforces
monotonic, gap-checkable versions. ``storage_path`` is the ``BaseStorage`` key
(NULL when the version is spec-only). Pass-by-reference: bytes live in storage,
only metadata + the key live here.
Revision ID: 0025_artifacts
Revises: 0024_wiki_pages_updated_via
"""
from typing import Sequence, Union
from alembic import op
revision: str = "0025_artifacts"
down_revision: Union[str, None] = "0024_wiki_pages_updated_via"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# --- artifacts -----------------------------------------------------------
op.execute(
"""
CREATE TABLE IF NOT EXISTS artifacts (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
conversation_id UUID,
workflow_run_id UUID,
team_id UUID,
message_id UUID,
kind TEXT NOT NULL,
title TEXT,
metadata JSONB,
current_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT artifacts_parent_present_check
CHECK (conversation_id IS NOT NULL OR workflow_run_id IS NOT NULL)
);
"""
)
op.execute("CREATE INDEX IF NOT EXISTS artifacts_user_idx ON artifacts (user_id);")
op.execute(
"CREATE INDEX IF NOT EXISTS artifacts_conversation_idx "
"ON artifacts (conversation_id) WHERE conversation_id IS NOT NULL;"
)
op.execute(
"CREATE INDEX IF NOT EXISTS artifacts_workflow_run_idx "
"ON artifacts (workflow_run_id) WHERE workflow_run_id IS NOT NULL;"
)
# Reuse the shared set_updated_at() trigger fn defined in 0001.
op.execute("DROP TRIGGER IF EXISTS artifacts_set_updated_at ON artifacts;")
op.execute(
"""
CREATE TRIGGER artifacts_set_updated_at
BEFORE UPDATE ON artifacts
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
"""
)
# --- artifact_versions ---------------------------------------------------
op.execute(
"""
CREATE TABLE IF NOT EXISTS artifact_versions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
artifact_id UUID NOT NULL REFERENCES artifacts(id) ON DELETE CASCADE,
version INTEGER NOT NULL,
mime_type TEXT,
filename TEXT,
storage_path TEXT,
size BIGINT,
sha256 TEXT,
spec JSONB,
preview_text TEXT,
produced_by JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT artifact_versions_artifact_version_uidx
UNIQUE (artifact_id, version)
);
"""
)
op.execute(
"CREATE INDEX IF NOT EXISTS artifact_versions_artifact_idx "
"ON artifact_versions (artifact_id);"
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS artifact_versions_artifact_idx;")
op.execute("DROP TABLE IF EXISTS artifact_versions;")
op.execute("DROP TRIGGER IF EXISTS artifacts_set_updated_at ON artifacts;")
op.execute("DROP INDEX IF EXISTS artifacts_workflow_run_idx;")
op.execute("DROP INDEX IF EXISTS artifacts_conversation_idx;")
op.execute("DROP INDEX IF EXISTS artifacts_user_idx;")
op.execute("DROP TABLE IF EXISTS artifacts;")
+68
View File
@@ -401,6 +401,74 @@ attachments_table = Table(
Column("legacy_mongo_id", Text),
)
# Identity row, one per logical artifact. The stable ``id`` is the handle passed
# around (chat/workflow state/message bodies carry only this reference, never
# bytes). Authz is parent-derived: whoever can reach ``conversation_id`` (chat)
# or ``workflow_run_id`` (run) can reach its artifacts, so a CHECK requires at
# least one parent. ``user_id`` is ownership/quota only; ``team_id`` is a
# nullable forward-compat hook for Teams (no FK, matching ``teams`` itself).
artifacts_table = Table(
"artifacts",
metadata,
Column("id", UUID(as_uuid=True), primary_key=True, server_default=func.gen_random_uuid()),
Column("user_id", Text, nullable=False),
Column("conversation_id", UUID(as_uuid=True)),
Column("workflow_run_id", UUID(as_uuid=True)),
Column("team_id", UUID(as_uuid=True)),
Column("message_id", UUID(as_uuid=True)),
Column("kind", Text, nullable=False),
Column("title", Text),
Column("metadata", JSONB),
Column("current_version", Integer, nullable=False, server_default="1"),
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
CheckConstraint(
"conversation_id IS NOT NULL OR workflow_run_id IS NOT NULL",
name="artifacts_parent_present_check",
),
)
Index(
"artifacts_conversation_idx",
artifacts_table.c.conversation_id,
postgresql_where=artifacts_table.c.conversation_id.isnot(None),
)
Index(
"artifacts_workflow_run_idx",
artifacts_table.c.workflow_run_id,
postgresql_where=artifacts_table.c.workflow_run_id.isnot(None),
)
Index("artifacts_user_idx", artifacts_table.c.user_id)
# Append-only version history; never mutated. Each edit appends a row and bumps
# ``artifacts.current_version``. ``UNIQUE(artifact_id, version)`` keeps versions
# monotonic. ``storage_path`` is the ``BaseStorage`` key (NULL when spec-only);
# bytes live in storage, only metadata + the key live here (pass-by-reference).
artifact_versions_table = Table(
"artifact_versions",
metadata,
Column("id", UUID(as_uuid=True), primary_key=True, server_default=func.gen_random_uuid()),
Column(
"artifact_id",
UUID(as_uuid=True),
ForeignKey("artifacts.id", ondelete="CASCADE"),
nullable=False,
),
Column("version", Integer, nullable=False),
Column("mime_type", Text),
Column("filename", Text),
Column("storage_path", Text),
Column("size", BigInteger),
Column("sha256", Text),
Column("spec", JSONB),
Column("preview_text", Text),
Column("produced_by", JSONB),
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
UniqueConstraint("artifact_id", "version", name="artifact_versions_artifact_version_uidx"),
)
Index("artifact_versions_artifact_idx", artifact_versions_table.c.artifact_id)
memories_table = Table(
"memories",
metadata,
@@ -0,0 +1,270 @@
"""Repository for the ``artifacts`` / ``artifact_versions`` tables.
Append-only artifact store. An ``artifacts`` row is the stable identity; each
edit appends an immutable ``artifact_versions`` row and atomically bumps
``artifacts.current_version`` (mirroring ``workflows.increment_graph_version``).
Authz is parent-derived: reads are not gated on ``user_id`` — callers resolve
access via the artifact's ``conversation_id`` / ``workflow_run_id`` parent.
``user_id`` is carried for ownership / quota accounting only. Pass-by-reference:
only metadata + the ``BaseStorage`` key are stored here, never binary bytes.
"""
from __future__ import annotations
import json
from typing import Any, Optional
from sqlalchemy import Connection, text
from application.storage.db.base_repository import row_to_dict
def _artifact_to_dict(row: Any) -> dict:
"""row_to_dict for an ``artifacts`` row (``metadata`` key preserved)."""
return row_to_dict(row)
def _version_to_dict(row: Any) -> dict:
"""row_to_dict for an ``artifact_versions`` row (``spec``/``produced_by`` keys preserved)."""
return row_to_dict(row)
class ArtifactsRepository:
"""CRUD for artifact identities and their append-only versions."""
def __init__(self, conn: Connection) -> None:
"""Bind the repository to an open SQLAlchemy connection."""
self._conn = conn
def create_artifact(
self,
user_id: str,
kind: str,
*,
conversation_id: Optional[str] = None,
workflow_run_id: Optional[str] = None,
team_id: Optional[str] = None,
message_id: Optional[str] = None,
title: Optional[str] = None,
metadata: Any = None,
mime_type: Optional[str] = None,
filename: Optional[str] = None,
storage_path: Optional[str] = None,
size: Optional[int] = None,
sha256: Optional[str] = None,
spec: Any = None,
preview_text: Optional[str] = None,
produced_by: Any = None,
) -> dict:
"""Create the identity row and its version 1 atomically; return the artifact dict."""
artifact = self._conn.execute(
text(
"""
INSERT INTO artifacts (
user_id, conversation_id, workflow_run_id, team_id,
message_id, kind, title, metadata, current_version
)
VALUES (
:user_id,
CAST(:conversation_id AS uuid),
CAST(:workflow_run_id AS uuid),
CAST(:team_id AS uuid),
CAST(:message_id AS uuid),
:kind, :title, CAST(:metadata AS jsonb), 1
)
RETURNING *
"""
),
{
"user_id": user_id,
"conversation_id": conversation_id,
"workflow_run_id": workflow_run_id,
"team_id": team_id,
"message_id": message_id,
"kind": kind,
"title": title,
"metadata": json.dumps(metadata) if metadata is not None else None,
},
).fetchone()
artifact_dict = _artifact_to_dict(artifact)
self._insert_version(
artifact_id=artifact_dict["id"],
version=1,
mime_type=mime_type,
filename=filename,
storage_path=storage_path,
size=size,
sha256=sha256,
spec=spec,
preview_text=preview_text,
produced_by=produced_by,
)
return artifact_dict
def get_artifact(self, artifact_id: str) -> Optional[dict]:
"""Fetch one artifact by id, unscoped (prefer ``get_artifact_in_parent`` at request boundaries)."""
result = self._conn.execute(
text("SELECT * FROM artifacts WHERE id = CAST(:id AS uuid)"),
{"id": artifact_id},
)
row = result.fetchone()
return _artifact_to_dict(row) if row is not None else None
def get_artifact_in_parent(
self,
artifact_id: str,
*,
conversation_id: Optional[str] = None,
workflow_run_id: Optional[str] = None,
) -> Optional[dict]:
"""Fetch an artifact only if it belongs to the given parent; the safe gate for request handlers."""
if conversation_id is None and workflow_run_id is None:
raise ValueError("get_artifact_in_parent requires conversation_id or workflow_run_id")
clauses = ["id = CAST(:id AS uuid)"]
params: dict[str, Any] = {"id": artifact_id}
if conversation_id is not None:
clauses.append("conversation_id = CAST(:conversation_id AS uuid)")
params["conversation_id"] = conversation_id
if workflow_run_id is not None:
clauses.append("workflow_run_id = CAST(:workflow_run_id AS uuid)")
params["workflow_run_id"] = workflow_run_id
result = self._conn.execute(
text(f"SELECT * FROM artifacts WHERE {' AND '.join(clauses)}"),
params,
)
row = result.fetchone()
return _artifact_to_dict(row) if row is not None else None
def list_artifacts(
self,
conversation_id: Optional[str] = None,
workflow_run_id: Optional[str] = None,
user_id: Optional[str] = None,
) -> list[dict]:
"""List artifacts filtered by parent / owner (newest first); at least one filter is required."""
if conversation_id is None and workflow_run_id is None and user_id is None:
raise ValueError("list_artifacts requires at least one of conversation_id, workflow_run_id, user_id")
clauses: list[str] = []
params: dict[str, Any] = {}
if conversation_id is not None:
clauses.append("conversation_id = CAST(:conversation_id AS uuid)")
params["conversation_id"] = conversation_id
if workflow_run_id is not None:
clauses.append("workflow_run_id = CAST(:workflow_run_id AS uuid)")
params["workflow_run_id"] = workflow_run_id
if user_id is not None:
clauses.append("user_id = :user_id")
params["user_id"] = user_id
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
result = self._conn.execute(
text(f"SELECT * FROM artifacts {where} ORDER BY created_at DESC"),
params,
)
return [_artifact_to_dict(r) for r in result.fetchall()]
def get_version(self, artifact_id: str, version: int) -> Optional[dict]:
"""Fetch a single version row by artifact id and version number."""
result = self._conn.execute(
text(
"SELECT * FROM artifact_versions "
"WHERE artifact_id = CAST(:artifact_id AS uuid) AND version = :version"
),
{"artifact_id": artifact_id, "version": version},
)
row = result.fetchone()
return _version_to_dict(row) if row is not None else None
def list_versions(self, artifact_id: str) -> list[dict]:
"""List every version of an artifact, oldest first."""
result = self._conn.execute(
text(
"SELECT * FROM artifact_versions "
"WHERE artifact_id = CAST(:artifact_id AS uuid) ORDER BY version ASC"
),
{"artifact_id": artifact_id},
)
return [_version_to_dict(r) for r in result.fetchall()]
def append_version(
self,
artifact_id: str,
*,
mime_type: Optional[str] = None,
filename: Optional[str] = None,
storage_path: Optional[str] = None,
size: Optional[int] = None,
sha256: Optional[str] = None,
spec: Any = None,
preview_text: Optional[str] = None,
produced_by: Any = None,
) -> dict:
"""Append a new version and atomically bump ``current_version``; return the version dict."""
new_version = self._conn.execute(
text(
"UPDATE artifacts "
"SET current_version = current_version + 1, updated_at = now() "
"WHERE id = CAST(:id AS uuid) "
"RETURNING current_version"
),
{"id": artifact_id},
).fetchone()
if new_version is None:
raise ValueError(f"artifact {artifact_id} not found")
return self._insert_version(
artifact_id=artifact_id,
version=new_version[0],
mime_type=mime_type,
filename=filename,
storage_path=storage_path,
size=size,
sha256=sha256,
spec=spec,
preview_text=preview_text,
produced_by=produced_by,
)
def _insert_version(
self,
*,
artifact_id: str,
version: int,
mime_type: Optional[str],
filename: Optional[str],
storage_path: Optional[str],
size: Optional[int],
sha256: Optional[str],
spec: Any,
preview_text: Optional[str],
produced_by: Any,
) -> dict:
"""Insert one append-only version row; UNIQUE(artifact_id, version) guards duplicates."""
result = self._conn.execute(
text(
"""
INSERT INTO artifact_versions (
artifact_id, version, mime_type, filename, storage_path,
size, sha256, spec, preview_text, produced_by
)
VALUES (
CAST(:artifact_id AS uuid), :version, :mime_type, :filename,
:storage_path, :size, :sha256, CAST(:spec AS jsonb),
:preview_text, CAST(:produced_by AS jsonb)
)
RETURNING *
"""
),
{
"artifact_id": artifact_id,
"version": version,
"mime_type": mime_type,
"filename": filename,
"storage_path": storage_path,
"size": size,
"sha256": sha256,
"spec": json.dumps(spec) if spec is not None else None,
"preview_text": preview_text,
"produced_by": json.dumps(produced_by) if produced_by is not None else None,
},
)
return _version_to_dict(result.fetchone())
@@ -0,0 +1,241 @@
"""Tests for ArtifactsRepository against a real Postgres instance."""
from __future__ import annotations
import uuid
import pytest
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from application.storage.db.repositories.artifacts import ArtifactsRepository
def _repo(conn) -> ArtifactsRepository:
return ArtifactsRepository(conn)
def _conversation_id() -> str:
return str(uuid.uuid4())
class TestCreateArtifact:
def test_creates_identity_and_auto_v1(self, pg_conn):
repo = _repo(pg_conn)
conv = _conversation_id()
artifact = repo.create_artifact(
"user-1",
"presentation",
conversation_id=conv,
title="Q3 deck",
metadata={"source": "chat"},
mime_type="application/vnd.ms-powerpoint",
filename="deck.pptx",
storage_path="inputs/user-1/artifacts/x/v1/deck.pptx",
size=1234,
sha256="abc",
spec={"slides": []},
)
assert artifact["user_id"] == "user-1"
assert artifact["kind"] == "presentation"
assert artifact["title"] == "Q3 deck"
assert artifact["metadata"] == {"source": "chat"}
assert artifact["current_version"] == 1
assert artifact["id"] is not None
assert artifact["_id"] == artifact["id"]
v1 = repo.get_version(artifact["id"], 1)
assert v1 is not None
assert v1["version"] == 1
assert v1["filename"] == "deck.pptx"
assert v1["storage_path"] == "inputs/user-1/artifacts/x/v1/deck.pptx"
assert v1["size"] == 1234
assert v1["sha256"] == "abc"
assert v1["spec"] == {"slides": []}
def test_spec_only_version_allows_null_storage_path(self, pg_conn):
repo = _repo(pg_conn)
artifact = repo.create_artifact(
"user-1",
"document",
workflow_run_id=_conversation_id(),
spec={"body": "draft"},
)
v1 = repo.get_version(artifact["id"], 1)
assert v1["storage_path"] is None
assert v1["spec"] == {"body": "draft"}
def test_produced_by_and_preview_round_trip(self, pg_conn):
repo = _repo(pg_conn)
artifact = repo.create_artifact(
"user-1",
"code",
conversation_id=_conversation_id(),
preview_text="head of output",
produced_by={"tool_id": "code_executor", "session_id": "s1"},
)
v1 = repo.get_version(artifact["id"], 1)
assert v1["preview_text"] == "head of output"
assert v1["produced_by"] == {"tool_id": "code_executor", "session_id": "s1"}
def test_missing_both_parents_rejected(self, pg_conn):
repo = _repo(pg_conn)
try:
with pg_conn.begin_nested():
repo.create_artifact("user-1", "document")
except IntegrityError:
pass
else:
pytest.fail("expected IntegrityError when no parent is present")
class TestGetArtifact:
def test_get_existing(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
fetched = repo.get_artifact(created["id"])
assert fetched["id"] == created["id"]
def test_get_missing_returns_none(self, pg_conn):
repo = _repo(pg_conn)
assert repo.get_artifact(str(uuid.uuid4())) is None
def test_get_not_gated_on_user(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"owner", "document", conversation_id=_conversation_id()
)
# Parent-derived authz: the repository never filters reads by user_id.
assert repo.get_artifact(created["id"])["user_id"] == "owner"
class TestGetArtifactInParent:
def test_returns_when_parent_matches(self, pg_conn):
repo = _repo(pg_conn)
conv = _conversation_id()
created = repo.create_artifact("user-1", "document", conversation_id=conv)
fetched = repo.get_artifact_in_parent(created["id"], conversation_id=conv)
assert fetched is not None
assert fetched["id"] == created["id"]
def test_returns_none_on_parent_mismatch(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
# Correct id but a different conversation -> denied (no cross-parent leak).
assert repo.get_artifact_in_parent(created["id"], conversation_id=_conversation_id()) is None
def test_requires_a_parent(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
with pytest.raises(ValueError):
repo.get_artifact_in_parent(created["id"])
class TestAppendVersion:
def test_increments_current_version(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
assert created["current_version"] == 1
v2 = repo.append_version(created["id"], spec={"body": "edit"}, filename="d.md")
assert v2["version"] == 2
refreshed = repo.get_artifact(created["id"])
assert refreshed["current_version"] == 2
assert refreshed["updated_at"] >= created["updated_at"]
def test_two_appends_yield_v2_then_v3(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
assert repo.append_version(created["id"])["version"] == 2
assert repo.append_version(created["id"])["version"] == 3
assert repo.get_artifact(created["id"])["current_version"] == 3
def test_append_to_missing_artifact_raises(self, pg_conn):
repo = _repo(pg_conn)
with pytest.raises(ValueError):
repo.append_version(str(uuid.uuid4()))
def test_duplicate_version_raises(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
# Force a collision on (artifact_id, version) by re-inserting v1.
try:
with pg_conn.begin_nested():
repo._insert_version(
artifact_id=created["id"],
version=1,
mime_type=None,
filename=None,
storage_path=None,
size=None,
sha256=None,
spec=None,
preview_text=None,
produced_by=None,
)
except IntegrityError:
pass
else:
pytest.fail("expected IntegrityError on duplicate (artifact_id, version)")
class TestListArtifacts:
def test_filters_by_conversation(self, pg_conn):
repo = _repo(pg_conn)
conv_a = _conversation_id()
conv_b = _conversation_id()
repo.create_artifact("user-1", "document", conversation_id=conv_a)
repo.create_artifact("user-1", "document", conversation_id=conv_a)
repo.create_artifact("user-1", "document", conversation_id=conv_b)
assert len(repo.list_artifacts(conversation_id=conv_a)) == 2
assert len(repo.list_artifacts(conversation_id=conv_b)) == 1
def test_filters_by_workflow_run(self, pg_conn):
repo = _repo(pg_conn)
run = _conversation_id()
repo.create_artifact("user-1", "document", workflow_run_id=run)
repo.create_artifact("user-1", "document", conversation_id=_conversation_id())
assert len(repo.list_artifacts(workflow_run_id=run)) == 1
def test_filters_by_user(self, pg_conn):
repo = _repo(pg_conn)
conv = _conversation_id()
repo.create_artifact("alice", "document", conversation_id=conv)
repo.create_artifact("bob", "document", conversation_id=conv)
assert len(repo.list_artifacts(conversation_id=conv, user_id="alice")) == 1
def test_requires_at_least_one_filter(self, pg_conn):
repo = _repo(pg_conn)
with pytest.raises(ValueError):
repo.list_artifacts()
class TestCascadeDelete:
def test_deleting_artifact_removes_versions(self, pg_conn):
repo = _repo(pg_conn)
created = repo.create_artifact(
"user-1", "document", conversation_id=_conversation_id()
)
repo.append_version(created["id"])
assert len(repo.list_versions(created["id"])) == 2
pg_conn.execute(
text("DELETE FROM artifacts WHERE id = CAST(:id AS uuid)"),
{"id": created["id"]},
)
assert repo.get_artifact(created["id"]) is None
assert repo.list_versions(created["id"]) == []