feat: catalog JSONL load/save/append
This commit is contained in:
@@ -5,6 +5,9 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, asdict, fields
|
||||
from typing import Optional
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
MODES = frozenset({"audio", "video", "av"})
|
||||
LICENSES = frozenset({"public_domain", "cc0", "cc_by", "cc_by_nc"})
|
||||
REVIEW_STATUSES = frozenset({"proposed", "approved"})
|
||||
@@ -87,3 +90,39 @@ def record_from_dict(data: dict) -> Record:
|
||||
return Record(**data)
|
||||
except TypeError as exc:
|
||||
raise CatalogError(str(exc)) from exc
|
||||
|
||||
|
||||
def load_catalog(path) -> list[Record]:
|
||||
"""Load and validate every record from a JSONL file. Blank lines skipped."""
|
||||
path = Path(path)
|
||||
records: list[Record] = []
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
for lineno, raw in enumerate(fh, start=1):
|
||||
line = raw.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise CatalogError(f"line {lineno}: invalid JSON: {exc}") from exc
|
||||
record = record_from_dict(data)
|
||||
validate(record)
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
|
||||
def save_catalog(records, path) -> None:
|
||||
"""Validate and write all records to a JSONL file (overwrites)."""
|
||||
path = Path(path)
|
||||
with path.open("w", encoding="utf-8") as fh:
|
||||
for record in records:
|
||||
validate(record)
|
||||
fh.write(json.dumps(record_to_dict(record), ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def append_record(record, path) -> None:
|
||||
"""Validate and append a single record to a JSONL file."""
|
||||
validate(record)
|
||||
path = Path(path)
|
||||
with path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(record_to_dict(record), ensure_ascii=False) + "\n")
|
||||
|
||||
Reference in New Issue
Block a user