§22 S2: corpus mirror reads each collection's <subfolder>/rfcs/
refresh_meta_repo now iterates a project's collections and keys cached_rfcs by collection_id; the default collection (subfolder '') keeps the shipped rfcs/ root path. N=1 default path unchanged (469 green). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ben Stull
+26
-11
@@ -54,15 +54,27 @@ async def refresh_meta_repo(config: Config, gitea: Gitea) -> None:
|
||||
|
||||
|
||||
async def _refresh_project_corpus(org: str, project_id: str, repo: str, gitea: Gitea) -> None:
|
||||
# §22 S1: the corpus grain is the collection. The mirror is project-grained
|
||||
# (reads rfcs/ at the repo root = the project's default collection); resolve
|
||||
# that collection once and key cached_rfcs by it.
|
||||
# §22 S2: the corpus grain is the collection. Mirror every collection of the
|
||||
# project from its `<subfolder>/rfcs/` directory, keying cached_rfcs by the
|
||||
# collection id. The default collection (subfolder '') reads `rfcs/` — the
|
||||
# shipped path, unchanged. include_unlisted: the mirror serves every
|
||||
# collection's content regardless of enumeration visibility.
|
||||
from . import collections as collections_mod
|
||||
collection_id = collections_mod.default_collection_id(project_id)
|
||||
for col in collections_mod.list_collections(project_id, include_unlisted=True):
|
||||
await _refresh_collection_corpus(
|
||||
org, project_id, repo, col["id"], col["subfolder"] or "", gitea
|
||||
)
|
||||
|
||||
|
||||
async def _refresh_collection_corpus(
|
||||
org: str, project_id: str, repo: str, collection_id: str, subfolder: str, gitea: Gitea
|
||||
) -> None:
|
||||
rfcs_dir = f"{subfolder}/rfcs" if subfolder else "rfcs"
|
||||
try:
|
||||
files = await gitea.list_dir(org, repo, "rfcs", ref="main")
|
||||
files = await gitea.list_dir(org, repo, rfcs_dir, ref="main")
|
||||
except GiteaError as e:
|
||||
log.warning("refresh_meta_repo: project %s: cannot list rfcs/: %s", project_id, e)
|
||||
log.warning("refresh_meta_repo: %s/%s: cannot list %s: %s",
|
||||
project_id, collection_id, rfcs_dir, e)
|
||||
return
|
||||
|
||||
seen_slugs: set[str] = set()
|
||||
@@ -76,17 +88,19 @@ async def _refresh_project_corpus(org: str, project_id: str, repo: str, gitea: G
|
||||
try:
|
||||
entry = entry_mod.parse(text)
|
||||
except Exception as parse_err:
|
||||
log.warning("refresh_meta_repo: %s: skipping %s: %s", project_id, f["path"], parse_err)
|
||||
log.warning("refresh_meta_repo: %s/%s: skipping %s: %s",
|
||||
project_id, collection_id, f["path"], parse_err)
|
||||
continue
|
||||
if not entry.slug:
|
||||
log.warning("refresh_meta_repo: %s: skipping %s: missing slug", project_id, f["path"])
|
||||
log.warning("refresh_meta_repo: %s/%s: skipping %s: missing slug",
|
||||
project_id, collection_id, f["path"])
|
||||
continue
|
||||
seen_slugs.add(entry.slug)
|
||||
_upsert_cached_rfc(entry, body_sha=sha, collection_id=collection_id)
|
||||
|
||||
# Entries removed from a project's rfcs/ — the spec keeps withdrawn entries
|
||||
# Entries removed from a collection's rfcs/ — the spec keeps withdrawn entries
|
||||
# as historical record (§3), so this fires only for out-of-band deletes;
|
||||
# leave the row, scoped to this project, for reconciler attention.
|
||||
# leave the row, scoped to this collection, for reconciler attention.
|
||||
existing = {
|
||||
row["slug"]
|
||||
for row in db.conn().execute(
|
||||
@@ -94,7 +108,8 @@ async def _refresh_project_corpus(org: str, project_id: str, repo: str, gitea: G
|
||||
)
|
||||
}
|
||||
for missing in existing - seen_slugs:
|
||||
log.info("refresh_meta_repo: %s/%s no longer in rfcs/ — leaving cache row", project_id, missing)
|
||||
log.info("refresh_meta_repo: %s/%s/%s no longer present — leaving cache row",
|
||||
project_id, collection_id, missing)
|
||||
|
||||
|
||||
def _upsert_cached_rfc(entry: entry_mod.Entry, body_sha: str, collection_id: str = "default") -> None:
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""§22 S2 — collection-grained corpus mirror + collection-scoped serve/propose.
|
||||
|
||||
The mirror test drives cache.refresh_meta_repo against an in-memory content repo
|
||||
holding entries under both the default `rfcs/` and a named collection's
|
||||
`features/rfcs/`, and asserts cached_rfcs is keyed by the right collection_id."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from app import cache, db
|
||||
from app.config import Config
|
||||
|
||||
|
||||
def _db() -> Config:
|
||||
cfg = Config(
|
||||
gitea_url="x", gitea_bot_user="x", gitea_bot_token="x", gitea_org="wiggleverse",
|
||||
registry_repo="registry", oauth_client_id="x",
|
||||
oauth_client_secret="x", app_url="x", secret_key="x",
|
||||
database_path=Path(tempfile.mkdtemp(prefix="colserve-")) / "t.db",
|
||||
owner_gitea_login="x", webhook_secret="x",
|
||||
)
|
||||
db.run_migrations(cfg)
|
||||
if db._CONN is not None:
|
||||
db._CONN.close()
|
||||
db._CONN = None
|
||||
db.init(cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
class _CorpusGitea:
|
||||
"""A content repo modelled as a flat {path: text} map, listing files under a
|
||||
directory prefix and reading them back."""
|
||||
|
||||
def __init__(self, tree: dict[str, str]):
|
||||
self._tree = tree
|
||||
|
||||
async def list_dir(self, org, repo, path, ref="main"):
|
||||
out = []
|
||||
prefix = (path.rstrip("/") + "/") if path else ""
|
||||
for p in self._tree:
|
||||
if p.startswith(prefix) and "/" not in p[len(prefix):]:
|
||||
out.append({"type": "file", "name": p.split("/")[-1], "path": p})
|
||||
return out
|
||||
|
||||
async def read_file(self, org, repo, path, ref="main"):
|
||||
t = self._tree.get(path)
|
||||
return (t, "sha-" + path) if t is not None else None
|
||||
|
||||
|
||||
def _entry_md(slug, title):
|
||||
return f"---\nslug: {slug}\ntitle: {title}\nstate: active\n---\nbody\n"
|
||||
|
||||
|
||||
def _seed_project_with_two_collections():
|
||||
db.conn().execute(
|
||||
"INSERT OR REPLACE INTO projects (id, name, content_repo, visibility, updated_at) "
|
||||
"VALUES ('ohm','Ohm','ohm-rfc','public', datetime('now'))")
|
||||
for cid, sub in [("default", ""), ("features", "features")]:
|
||||
db.conn().execute(
|
||||
"INSERT OR REPLACE INTO collections (id, project_id, type, subfolder, initial_state, "
|
||||
"visibility, created_at, updated_at) VALUES (?, 'ohm','document',?, "
|
||||
"'super-draft','public', datetime('now'), datetime('now'))", (cid, sub))
|
||||
|
||||
|
||||
def test_mirror_keys_entries_by_collection():
|
||||
cfg = _db()
|
||||
_seed_project_with_two_collections()
|
||||
gitea = _CorpusGitea({
|
||||
"rfcs/a.md": _entry_md("a", "Default A"),
|
||||
"features/rfcs/b.md": _entry_md("b", "Feature B"),
|
||||
})
|
||||
asyncio.run(cache.refresh_meta_repo(cfg, gitea))
|
||||
got = {(r["collection_id"], r["slug"]) for r in
|
||||
db.conn().execute("SELECT collection_id, slug FROM cached_rfcs")}
|
||||
assert got == {("default", "a"), ("features", "b")}
|
||||
Reference in New Issue
Block a user