§22 S2: registry mirror reads .collection.yaml manifests
Discover named collections by walking each project's content-repo root for <subdir>/.collection.yaml; parse + upsert with immutable-type enforcement (§22.4a) and project-visibility inheritance. The default collection still flows from projects.yaml. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ben Stull
@@ -55,6 +55,17 @@ class ProjectEntry:
|
||||
config: dict = field(default_factory=dict) # theme, enabled_models
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollectionEntry:
|
||||
"""A named collection declared by a `.collection.yaml` manifest inside a
|
||||
project's content repo (S2). `visibility=None` means "inherit the project's
|
||||
visibility"."""
|
||||
type: str
|
||||
visibility: str | None
|
||||
initial_state: str
|
||||
name: str | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RegistryDoc:
|
||||
deployment_name: str
|
||||
@@ -114,6 +125,33 @@ def parse_registry(text: str) -> RegistryDoc:
|
||||
)
|
||||
|
||||
|
||||
def parse_collection_manifest(text: str) -> CollectionEntry:
|
||||
"""Parse + validate a `.collection.yaml`. Pure (no I/O). Raises RegistryError.
|
||||
|
||||
`type` is required and immutable (§22.4a, enforced at upsert). `visibility`
|
||||
is optional — omitted means inherit the project's. `initial_state` defaults
|
||||
per type (§22.4b)."""
|
||||
raw = yaml.safe_load(text) or {}
|
||||
if not isinstance(raw, dict):
|
||||
raise RegistryError("collection manifest must be a mapping")
|
||||
ctype = str(raw.get("type") or "").strip()
|
||||
if ctype not in VALID_TYPES:
|
||||
raise RegistryError(f"collection has invalid type {ctype!r}")
|
||||
vis = raw.get("visibility")
|
||||
if vis is not None:
|
||||
vis = str(vis).strip()
|
||||
if vis not in VALID_VISIBILITY:
|
||||
raise RegistryError(f"collection has invalid visibility {vis!r}")
|
||||
initial_state = str(
|
||||
raw.get("initial_state") or _TYPE_DEFAULT_INITIAL_STATE[ctype]
|
||||
).strip()
|
||||
if initial_state not in VALID_INITIAL_STATE:
|
||||
raise RegistryError(f"collection has invalid initial_state {initial_state!r}")
|
||||
name = raw.get("name")
|
||||
name = str(name).strip() if name else None
|
||||
return CollectionEntry(ctype, vis, initial_state, name)
|
||||
|
||||
|
||||
def _default_collection_id(project_id: str, default_id: str) -> str:
|
||||
"""The id of a project's default collection. The deployment's primary
|
||||
project (== `default_id`, the §22.13 resolved default) gets the stable
|
||||
@@ -193,6 +231,71 @@ def apply_registry(doc: RegistryDoc, registry_sha: str, default_id: str) -> None
|
||||
)
|
||||
|
||||
|
||||
def _upsert_named_collection(
|
||||
proj: ProjectEntry, subdir: str, ce: CollectionEntry, sha: str
|
||||
) -> None:
|
||||
"""Upsert one named collection (S2). Type is immutable (§22.4a): a type
|
||||
change against an existing row is refused (logged, not applied). A None
|
||||
manifest visibility inherits the project's visibility."""
|
||||
visibility = ce.visibility or proj.visibility
|
||||
with db.tx() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT type FROM collections WHERE id = ?", (subdir,)
|
||||
).fetchone()
|
||||
if existing is not None and existing["type"] != ce.type:
|
||||
log.error(
|
||||
"registry: refusing immutable type change on collection %s (%s -> %s)",
|
||||
subdir, existing["type"], ce.type,
|
||||
)
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO collections
|
||||
(id, project_id, type, subfolder, initial_state, visibility, name, registry_sha, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
project_id = excluded.project_id,
|
||||
initial_state = excluded.initial_state,
|
||||
visibility = excluded.visibility,
|
||||
name = excluded.name,
|
||||
registry_sha = excluded.registry_sha,
|
||||
updated_at = datetime('now')
|
||||
""",
|
||||
(subdir, proj.id, ce.type, subdir, ce.initial_state, visibility, ce.name, sha),
|
||||
)
|
||||
|
||||
|
||||
async def _mirror_named_collections(config: Config, gitea: Gitea, doc: RegistryDoc, sha: str) -> None:
|
||||
"""§22 S2: named collections are declared by `.collection.yaml` manifests
|
||||
inside each project's content repo (the default collection comes from
|
||||
projects.yaml). Walk each content repo root; a subdir carrying a manifest
|
||||
becomes a collection keyed by the subdir name. Tolerant: a transport or
|
||||
parse failure on one project/collection logs and is skipped, never aborts
|
||||
the wider mirror (keep last-good)."""
|
||||
for proj in doc.projects:
|
||||
try:
|
||||
items = await gitea.list_dir(config.gitea_org, proj.content_repo, "", ref="main")
|
||||
except Exception as e: # noqa: BLE001 — GiteaError/transport: tolerate
|
||||
log.warning("registry: cannot list %s root: %s", proj.content_repo, e)
|
||||
continue
|
||||
for it in items:
|
||||
if it.get("type") != "dir":
|
||||
continue
|
||||
subdir = it["name"]
|
||||
manifest = await gitea.get_contents(
|
||||
config.gitea_org, proj.content_repo, f"{subdir}/.collection.yaml", ref="main"
|
||||
)
|
||||
if not manifest or manifest.get("type") != "file":
|
||||
continue
|
||||
mtext = base64.b64decode(manifest["content"]).decode("utf-8")
|
||||
try:
|
||||
ce = parse_collection_manifest(mtext)
|
||||
except RegistryError as e:
|
||||
log.error("registry: bad manifest %s/%s: %s", proj.content_repo, subdir, e)
|
||||
continue
|
||||
_upsert_named_collection(proj, subdir, ce, sha)
|
||||
|
||||
|
||||
async def refresh_registry(config: Config, gitea: Gitea) -> None:
|
||||
"""Mirror REGISTRY_REPO/projects.yaml into projects + deployment.
|
||||
|
||||
@@ -213,4 +316,6 @@ async def refresh_registry(config: Config, gitea: Gitea) -> None:
|
||||
doc = parse_registry(text)
|
||||
from . import projects as projects_mod
|
||||
apply_registry(doc, sha, projects_mod.resolved_default_id(config))
|
||||
# §22 S2: discover + upsert named collections from each content repo.
|
||||
await _mirror_named_collections(config, gitea, doc, sha)
|
||||
log.info("registry: mirrored %d project(s) at %s", len(doc.projects), sha)
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""§22 S2 — the registry mirror reads `.collection.yaml` manifests inside each
|
||||
project's content repo and upserts a named collection per manifest. The default
|
||||
collection still flows from projects.yaml (test_registry.py)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import db, registry
|
||||
from app.config import Config
|
||||
|
||||
|
||||
def _db() -> Config:
|
||||
cfg = Config(
|
||||
gitea_url="x", gitea_bot_user="x", gitea_bot_token="x", gitea_org="wiggleverse",
|
||||
registry_repo="registry", oauth_client_id="x",
|
||||
oauth_client_secret="x", app_url="x", secret_key="x",
|
||||
database_path=Path(tempfile.mkdtemp(prefix="colreg-")) / "t.db",
|
||||
owner_gitea_login="x", webhook_secret="x",
|
||||
)
|
||||
db.run_migrations(cfg)
|
||||
if db._CONN is not None:
|
||||
db._CONN.close()
|
||||
db._CONN = None
|
||||
db.init(cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
# --- pure parser --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_collection_manifest_minimal():
|
||||
doc = registry.parse_collection_manifest("type: bdd\n")
|
||||
assert doc.type == "bdd"
|
||||
# §22.4b: bdd defaults to 'active'; visibility inherits (None == inherit).
|
||||
assert doc.initial_state == "active"
|
||||
assert doc.visibility is None
|
||||
assert doc.name is None
|
||||
|
||||
|
||||
def test_parse_collection_manifest_full():
|
||||
doc = registry.parse_collection_manifest(
|
||||
"type: document\nvisibility: public\ninitial_state: active\nname: Model\n"
|
||||
)
|
||||
assert (doc.type, doc.visibility, doc.initial_state, doc.name) == (
|
||||
"document", "public", "active", "Model",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_collection_manifest_rejects_bad_type():
|
||||
with pytest.raises(registry.RegistryError):
|
||||
registry.parse_collection_manifest("type: nonsense\n")
|
||||
|
||||
|
||||
def test_parse_collection_manifest_rejects_bad_visibility():
|
||||
with pytest.raises(registry.RegistryError):
|
||||
registry.parse_collection_manifest("type: bdd\nvisibility: nope\n")
|
||||
|
||||
|
||||
# --- mirror discovery ---------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeGitea:
|
||||
"""Minimal Gitea stub: projects.yaml in the registry repo + a content repo
|
||||
whose root holds a `features/` subdir carrying a `.collection.yaml`."""
|
||||
|
||||
def __init__(self, projects_yaml: str, repo_tree: dict[str, dict[str, str]]):
|
||||
self._projects_yaml = projects_yaml
|
||||
self._repo_tree = repo_tree # {repo: {path: text}}
|
||||
|
||||
async def get_contents(self, org, repo, path, ref="main"):
|
||||
if path == "projects.yaml":
|
||||
return {"type": "file",
|
||||
"content": base64.b64encode(self._projects_yaml.encode()).decode(),
|
||||
"sha": "regsha-test"}
|
||||
text = self._repo_tree.get(repo, {}).get(path)
|
||||
if text is None:
|
||||
return None
|
||||
return {"type": "file",
|
||||
"content": base64.b64encode(text.encode()).decode(), "sha": "c0ffee"}
|
||||
|
||||
async def list_dir(self, org, repo, path, ref="main"):
|
||||
# Root listing: surface each top-level segment as a 'dir' entry.
|
||||
prefix = (path.rstrip("/") + "/") if path else ""
|
||||
dirs = set()
|
||||
for p in self._repo_tree.get(repo, {}):
|
||||
if not p.startswith(prefix):
|
||||
continue
|
||||
rest = p[len(prefix):]
|
||||
if "/" in rest:
|
||||
dirs.add(rest.split("/", 1)[0])
|
||||
return [{"type": "dir", "name": n, "path": prefix + n} for n in sorted(dirs)]
|
||||
|
||||
|
||||
_PROJECTS = (
|
||||
"deployment:\n name: Ohm\n tagline: t\n"
|
||||
"projects:\n - id: ohm\n name: Ohm\n type: document\n"
|
||||
" content_repo: ohm-rfc\n visibility: public\n"
|
||||
)
|
||||
|
||||
|
||||
def test_refresh_registry_mirrors_named_collection():
|
||||
cfg = _db()
|
||||
gitea = _FakeGitea(
|
||||
projects_yaml=_PROJECTS,
|
||||
repo_tree={"ohm-rfc": {"features/.collection.yaml": "type: bdd\nname: Features\n"}},
|
||||
)
|
||||
asyncio.run(registry.refresh_registry(cfg, gitea))
|
||||
row = db.conn().execute(
|
||||
"SELECT type, subfolder, name, project_id, visibility FROM collections WHERE id='features'"
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
assert (row["type"], row["subfolder"], row["project_id"]) == ("bdd", "features", "ohm")
|
||||
assert row["name"] == "Features"
|
||||
# visibility inherits the project's (public) when the manifest omits it.
|
||||
assert row["visibility"] == "public"
|
||||
|
||||
|
||||
def test_refresh_registry_leaves_default_collection_intact():
|
||||
cfg = _db()
|
||||
gitea = _FakeGitea(
|
||||
projects_yaml=_PROJECTS,
|
||||
repo_tree={"ohm-rfc": {"features/.collection.yaml": "type: bdd\n"}},
|
||||
)
|
||||
asyncio.run(registry.refresh_registry(cfg, gitea))
|
||||
# The default collection (from projects.yaml) and the named one coexist.
|
||||
ids = {r["id"] for r in db.conn().execute("SELECT id FROM collections")}
|
||||
assert {"default", "features"} <= ids
|
||||
|
||||
|
||||
def test_refresh_registry_immutable_type_on_named_collection():
|
||||
cfg = _db()
|
||||
gitea = _FakeGitea(
|
||||
projects_yaml=_PROJECTS,
|
||||
repo_tree={"ohm-rfc": {"features/.collection.yaml": "type: bdd\n"}},
|
||||
)
|
||||
asyncio.run(registry.refresh_registry(cfg, gitea))
|
||||
# A later manifest that flips the type is refused (§22.4a immutable type).
|
||||
gitea._repo_tree["ohm-rfc"]["features/.collection.yaml"] = "type: document\n"
|
||||
asyncio.run(registry.refresh_registry(cfg, gitea))
|
||||
t = db.conn().execute("SELECT type FROM collections WHERE id='features'").fetchone()["type"]
|
||||
assert t == "bdd"
|
||||
Reference in New Issue
Block a user