diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2e51d15..dea5dac 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -170,21 +170,34 @@ every transition, so fast navigation stays responsive on placeholder media. contract** with the firmware (the keyboard/serial stand-in already works via a `Controls` stream). - **White-noise generation** — runtime white/pink noise for that content position. -- **Offline v2v variant pipeline** — author the pre-baked Right restyle variants - via the local flow-stabilized SD pipeline (now ~free, not a paid API — see the - scales-library/right-axis design §1/§4); a real multi-strength flow-stabilized - re-bake per base clip + the multilingual label/string tables + TTS. The - **scale-ring transitions** (forward/reverse per-edge morphs) are part of this - offline pipeline too. - - **Vertical slice done (session 0012):** the Right-variant half is - productionized and run for the **forest** scale — `simulator/bake_right_variants.py` - (the POC flow-restyle algorithm, repo-resident) bakes real strengths 1–4 on a - by-eye keyframe-strength ramp; forest now carries real Right variants in the - sim, so the dreamlike-axis look is judgeable on real footage. **Still - deferred:** real Right variants for cosmos/abyss (need their real strict-PD - bases sourced first) and the **real i2v ring transitions** (need a 2nd real - base + a generative-video model not in the POC). Plan: - [`2026-06-08-v2v-vertical-slice-forest-right-variants.md`](./superpowers/plans/2026-06-08-v2v-vertical-slice-forest-right-variants.md). +- **Content pipeline + label track** — source strict-PD neutral bases, run them + through the offline pipeline, and author the annotation tracks that drive the + Left-HUD overlay. The Right dream is now **real-time** (Kuwahara WebGL shader, + session 0013 — no per-clip ML bake), so the pipeline is + **source → crossfade-loop → master+proxy transcode → hand-authored / + motion-tracked labels → manifest**, not ML baking. Historical note: the + flow-stabilized SD-turbo v2v bake (session 0012, `simulator/bake_right_variants.py`) + was superseded when the operator found the baked-SD Right "too fluid" and the + design pivoted to the deterministic real-time dream; `bake_right_variants.py` + is now parked. Spec: + [`2026-06-24-content-pipeline-design.md`](./superpowers/specs/2026-06-24-content-pipeline-design.md). + + - **Increment 1 — deterministic tooling + 5-scale ring (session 0014) ✅ Done.** + `tools/pipeline/` (frame/loop/transcode/manifest, pure + unit-tested + an + opt-in integration test on real footage); the backward-compatible keyframed + annotation-track schema with client interpolation; and the **full 5-scale ring** + (cosmos → orbit → forest → reef → abyss) with orbit + reef as placeholder + scales alongside the existing cosmos/abyss/forest. Plan: + [`2026-06-24-content-pipeline-increment-1.md`](./superpowers/plans/2026-06-24-content-pipeline-increment-1.md). + + - **Increment 2 — hybrid motion-track pass + author mode (pending).** The + `tools/pipeline/track.py` classical OpenCV tracker + optional ML detect+track, + plus a simulator **author mode** for reviewing/editing annotation tracks in the + browser. Real strict-PD bases for all five scales sourced per + [`docs/content-sourcing.md`](./content-sourcing.md). + + - **Still deferred:** real **i2v ring transitions** (need a generative-video + model + adjacent real bases); Pi renderer + serial/firmware. - **Catalog model changes** — audio *source* + "neutral base" vs "altered variant" flag (sub-project 2 territory, design §13). diff --git a/docs/content-sourcing.md b/docs/content-sourcing.md new file mode 100644 index 0000000..1ad7966 --- /dev/null +++ b/docs/content-sourcing.md @@ -0,0 +1,29 @@ +# Content sourcing — strict-PD neutral bases + +For each scale, acquire a calm, neutral source clip from a **strict public-domain** +US-government source, verify the license, then run it through the pipeline. + +| Scale | Source | Where | +|--------|--------------------------------|------------------------------------| +| cosmos | NASA / Hubble / JWST | https://images.nasa.gov/ | +| orbit | NASA / ISS | https://images.nasa.gov/ | +| forest | NPS / USGS | https://www.nps.gov/ , https://www.usgs.gov/ | +| reef | NOAA Ocean Exploration | https://oceanexplorer.noaa.gov/ | +| abyss | NOAA Ocean Exploration | https://oceanexplorer.noaa.gov/ | + +**License check (mandatory):** confirm the asset is US-gov PD (17 U.S.C. §105) or +explicitly CC0. "Free stock" (Pexels/Pixabay/etc.) is royalty-free but NOT public +domain — do not use. Record a `Provenance` (tools/pipeline/provenance.py) per clip. + +**Run the pipeline** (replaces the placeholder bytes for a scale): + +```bash +python -c "from tools.pipeline.run import process_clip; \ + process_clip('downloads/.mp4', 'simulator/sample_media/', \ + start=, duration=, overlap=<0.5..1.0>)" +``` + +This writes `simulator/sample_media//base.mp4` (proxy) + `master.mp4`. Then +author the labels (Increment 2: the hybrid track pass + simulator author mode) or +hand-edit the annotation track for now (spec §4). The ring + manifest entry already +exist; `process_clip` only replaces the media bytes. diff --git a/docs/superpowers/plans/2026-06-24-content-pipeline-increment-1.md b/docs/superpowers/plans/2026-06-24-content-pipeline-increment-1.md new file mode 100644 index 0000000..b7164ee --- /dev/null +++ b/docs/superpowers/plans/2026-06-24-content-pipeline-increment-1.md @@ -0,0 +1,929 @@ +# Content Pipeline — Increment 1 (real footage, no ML) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the deterministic content-pipeline tooling (frame → crossfade-loop → master+proxy transcode → manifest assembly), add the backward-compatible keyframed annotation-track schema with client interpolation, and extend the scale ring to 5 scales — proving the pipeline end-to-end on real footage without any ML. + +**Architecture:** A new pure-Python `tools/pipeline/` package whose stage functions return ffmpeg argument lists (unit-tested without running ffmpeg); a thin `run.py` executes them and is covered by an opt-in integration test gated on `ffmpeg` + the real forest POC base. Manifest assembly is pure dict transforms. The annotation `track` is optional data that rides through the existing pass-through `annotations` list (no Python model change); interpolation is client-side JS, consistent with the existing "alteration math in Python, label placement in the client" split. This implements §3 (stages 1–4, 6), §4 (track schema), §6 (format), §7 (tooling), and Increment 1 of §8 in [`docs/superpowers/specs/2026-06-24-content-pipeline-design.md`](../specs/2026-06-24-content-pipeline-design.md). + +**Tech Stack:** Python 3.13, pytest, ffmpeg (libx264), vanilla JS (SVG overlay). Tests run with `pytest -q` from the repo root in the project `.venv`. + +--- + +### Task 1: Scaffold `tools/pipeline/` + the frame stage + +**Files:** +- Create: `tools/pipeline/__init__.py` +- Create: `tools/pipeline/ffmpeg_ops.py` +- Test: `tests/test_pipeline_ffmpeg_ops.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_pipeline_ffmpeg_ops.py +from tools.pipeline.ffmpeg_ops import frame_args + + +def test_frame_args_trims_scales_pads_and_strips_audio(): + args = frame_args( + "src.mp4", "out.mp4", + start=2.0, duration=8.0, width=1920, height=1080, ff="ffmpeg", + ) + assert args[0] == "ffmpeg" + assert "-ss" in args and args[args.index("-ss") + 1] == "2.0" + assert "-t" in args and args[args.index("-t") + 1] == "8.0" + assert "-an" in args # audio stripped (bases are video-only) + vf = args[args.index("-vf") + 1] + assert "scale=1920:1080:force_original_aspect_ratio=decrease" in vf + assert "pad=1920:1080" in vf + assert "format=yuv420p" in vf + assert args[-1] == "out.mp4" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_pipeline_ffmpeg_ops.py -q` +Expected: FAIL with `ModuleNotFoundError: No module named 'tools.pipeline'` + +- [ ] **Step 3: Write minimal implementation** + +```python +# tools/pipeline/__init__.py +"""Content pipeline: source -> frame -> loop -> transcode -> manifest. + +Pure ffmpeg-argument builders (ffmpeg_ops), a thin runner (run), provenance +records, and manifest assembly. See docs/superpowers/specs/2026-06-24-content-pipeline-design.md. +""" +``` + +```python +# tools/pipeline/ffmpeg_ops.py +"""Pure ffmpeg argument builders (no I/O) for the content pipeline. + +Each function returns a list[str] ready for ffmpeg, so command construction is +unit-testable without running ffmpeg. tools/pipeline/run.py executes them. The +ffmpeg binary name is injected (default "ffmpeg"). +""" + +from __future__ import annotations + + +def _fit(width: int, height: int) -> str: + """A scale+pad filter chain fitting any input into width×height (letterbox), + fixing SAR and pixel format so downstream concat/xfade get identical geometry.""" + return ( + f"scale={width}:{height}:force_original_aspect_ratio=decrease," + f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2," + "setsar=1,format=yuv420p" + ) + + +def frame_args(src, dst, *, start: float, duration: float, + width: int, height: int, ff: str = "ffmpeg") -> list[str]: + """Stage 2 — trim [start, start+duration], fit to width×height, strip audio. + Produces a high-quality mezzanine for the loop stage.""" + return [ + ff, "-y", "-ss", str(start), "-t", str(duration), + "-i", str(src), "-vf", _fit(width, height), "-an", + "-c:v", "libx264", "-preset", "medium", "-crf", "16", + "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(dst), + ] +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_pipeline_ffmpeg_ops.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add tools/pipeline/__init__.py tools/pipeline/ffmpeg_ops.py tests/test_pipeline_ffmpeg_ops.py +git commit -m "feat(pipeline): frame stage ffmpeg arg builder (content pipeline §3.2)" +``` + +--- + +### Task 2: Crossfade seamless-loop stage + +**Files:** +- Modify: `tools/pipeline/ffmpeg_ops.py` +- Test: `tests/test_pipeline_ffmpeg_ops.py` + +The seamless loop (spec §3.3) crossfades the clip's tail over its head. For a clip +of duration `D` with overlap `O`, the output (length `D − O`) is +`concat( xfade(tail, head, O), mid )` where `head`=`[0,O]`, `mid`=`[O, D−O]`, +`tail`=`[D−O, D]`. Both internal joins and the loop seam are then continuous, and +the tail→head jump is blended away over `O`. + +- [ ] **Step 1: Write the failing test** + +```python +# add to tests/test_pipeline_ffmpeg_ops.py +from tools.pipeline.ffmpeg_ops import crossfade_loop_args + + +def test_crossfade_loop_builds_head_mid_tail_xfade_concat(): + args = crossfade_loop_args("in.mp4", "loop.mp4", duration=8.0, overlap=1.0) + fc = args[args.index("-filter_complex") + 1] + assert "trim=0:1.0" in fc # head = first O secs + assert "trim=1.0:7.0" in fc # mid = [O, D-O] + assert "trim=7.0:8.0" in fc # tail = last O secs + assert "xfade=transition=fade:duration=1.0:offset=0" in fc + assert "concat=n=2:v=1" in fc + assert "-an" in args + assert args[-1] == "loop.mp4" + + +def test_crossfade_loop_rejects_overlap_too_large(): + import pytest + with pytest.raises(ValueError): + crossfade_loop_args("in.mp4", "loop.mp4", duration=4.0, overlap=2.0) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_pipeline_ffmpeg_ops.py -q` +Expected: FAIL with `ImportError: cannot import name 'crossfade_loop_args'` + +- [ ] **Step 3: Write minimal implementation** + +```python +# add to tools/pipeline/ffmpeg_ops.py +def crossfade_loop_args(src, dst, *, duration: float, overlap: float, + ff: str = "ffmpeg") -> list[str]: + """Stage 3 — make `src` loop seamlessly by crossfading its tail over its head. + Output length = duration − overlap. Requires overlap < duration/2 so a non-empty + middle remains.""" + if overlap <= 0 or overlap >= duration / 2: + raise ValueError(f"overlap {overlap} must be in (0, duration/2={duration/2})") + d, o = duration, overlap + fc = ( + f"[0:v]trim=0:{o},setpts=PTS-STARTPTS[head];" + f"[0:v]trim={o}:{d - o},setpts=PTS-STARTPTS[mid];" + f"[0:v]trim={d - o}:{d},setpts=PTS-STARTPTS[tail];" + f"[tail][head]xfade=transition=fade:duration={o}:offset=0[xf];" + f"[xf][mid]concat=n=2:v=1,format=yuv420p[out]" + ) + return [ + ff, "-y", "-i", str(src), "-filter_complex", fc, + "-map", "[out]", "-an", + "-c:v", "libx264", "-preset", "medium", "-crf", "16", + "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(dst), + ] +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_pipeline_ffmpeg_ops.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add tools/pipeline/ffmpeg_ops.py tests/test_pipeline_ffmpeg_ops.py +git commit -m "feat(pipeline): crossfade seamless-loop arg builder (content pipeline §3.3)" +``` + +--- + +### Task 3: Master + proxy transcode stage + +**Files:** +- Modify: `tools/pipeline/ffmpeg_ops.py` +- Test: `tests/test_pipeline_ffmpeg_ops.py` + +- [ ] **Step 1: Write the failing test** + +```python +# add to tests/test_pipeline_ffmpeg_ops.py +from tools.pipeline.ffmpeg_ops import transcode_args + + +def test_transcode_args_high_profile_even_dims_faststart(): + args = transcode_args("loop.mp4", "proxy.mp4", width=1920, height=1080, + crf=20, fps=30) + assert "-profile:v" in args and args[args.index("-profile:v") + 1] == "high" + assert "-crf" in args and args[args.index("-crf") + 1] == "20" + vf = args[args.index("-vf") + 1] + assert "scale=1920:1080:force_original_aspect_ratio=decrease" in vf + assert "fps=30" in vf + assert "+faststart" in args + assert "-an" in args + assert args[-1] == "proxy.mp4" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_pipeline_ffmpeg_ops.py -q` +Expected: FAIL with `ImportError: cannot import name 'transcode_args'` + +- [ ] **Step 3: Write minimal implementation** + +```python +# add to tools/pipeline/ffmpeg_ops.py +def transcode_args(src, dst, *, width: int, height: int, crf: int, + fps: int = 30, ff: str = "ffmpeg") -> list[str]: + """Stage 4 — encode a final deliverable (master or proxy). Master: source res + capped ~3840 wide, crf 18. Proxy: 1920×1080, crf 20. Both H.264 High, yuv420p, + even dims, +faststart, video-only (spec §6).""" + vf = ( + f"scale={width}:{height}:force_original_aspect_ratio=decrease," + f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2," + f"setsar=1,fps={fps},format=yuv420p" + ) + return [ + ff, "-y", "-i", str(src), "-vf", vf, "-an", + "-c:v", "libx264", "-profile:v", "high", "-preset", "slow", + "-crf", str(crf), "-pix_fmt", "yuv420p", + "-movflags", "+faststart", str(dst), + ] +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_pipeline_ffmpeg_ops.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add tools/pipeline/ffmpeg_ops.py tests/test_pipeline_ffmpeg_ops.py +git commit -m "feat(pipeline): master/proxy transcode arg builder (content pipeline §3.4/§6)" +``` + +--- + +### Task 4: Provenance record (stage 1 output) + +**Files:** +- Create: `tools/pipeline/provenance.py` +- Test: `tests/test_pipeline_provenance.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_pipeline_provenance.py +from tools.pipeline.provenance import Provenance + + +def test_provenance_to_dict_roundtrip(): + p = Provenance( + scale="abyss", + license="public-domain (US Gov, 17 U.S.C. §105)", + source="NOAA Ocean Exploration", + url="https://oceanexplorer.noaa.gov/", + fetched_at="2026-06-24", + ) + d = p.to_dict() + assert d == { + "scale": "abyss", + "license": "public-domain (US Gov, 17 U.S.C. §105)", + "source": "NOAA Ocean Exploration", + "url": "https://oceanexplorer.noaa.gov/", + "fetched_at": "2026-06-24", + } + assert Provenance.from_dict(d) == p +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_pipeline_provenance.py -q` +Expected: FAIL with `ModuleNotFoundError: No module named 'tools.pipeline.provenance'` + +- [ ] **Step 3: Write minimal implementation** + +```python +# tools/pipeline/provenance.py +"""Stage 1 — the per-clip strict-PD provenance record (spec §3.1). + +Captured at source time so license + source + URL travel with the media. The +'no explicit license -> assume PD, verify' trap (scales design §2.1) applies: +"free stock" (Pexels/Pixabay) is NOT public domain. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class Provenance: + scale: str + license: str + source: str + url: str + fetched_at: str # ISO date; passed in (no clock here -> deterministic) + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> "Provenance": + return cls( + scale=d["scale"], license=d["license"], source=d["source"], + url=d["url"], fetched_at=d["fetched_at"], + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_pipeline_provenance.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add tools/pipeline/provenance.py tests/test_pipeline_provenance.py +git commit -m "feat(pipeline): strict-PD provenance record (content pipeline §3.1)" +``` + +--- + +### Task 5: Manifest assembly (stage 6) + +**Files:** +- Create: `tools/pipeline/manifest.py` +- Test: `tests/test_pipeline_manifest.py` + +Pure dict transforms over a loaded manifest: upsert a clip entry, place a scale in +the ring order, and rebuild the per-edge transition list so it always has exactly +N entries for N scales (one transition per ring edge, including the wrap). + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_pipeline_manifest.py +from tools.pipeline.manifest import upsert_clip, add_ring_scale, rebuild_ring_edges + + +def _base(): + return {"clips": [{"id": "cosmos", "title": "Cosmos"}], + "ring": {"scales": [{"id": "cosmos", "clip_id": "cosmos"}], + "transitions": []}} + + +def test_upsert_clip_replaces_by_id_else_appends(): + m = _base() + m = upsert_clip(m, {"id": "reef", "title": "Reef"}) + assert [c["id"] for c in m["clips"]] == ["cosmos", "reef"] + m = upsert_clip(m, {"id": "reef", "title": "Coral Reef"}) # replace, not dup + assert [c["id"] for c in m["clips"]] == ["cosmos", "reef"] + assert m["clips"][1]["title"] == "Coral Reef" + + +def test_add_ring_scale_inserts_after_named_scale(): + m = _base() + m = add_ring_scale(m, "orbit", "orbit", after="cosmos") + assert [s["id"] for s in m["ring"]["scales"]] == ["cosmos", "orbit"] + + +def test_rebuild_ring_edges_one_transition_per_edge_with_wrap(): + m = _base() + m = add_ring_scale(m, "orbit", "orbit", after="cosmos") + m = add_ring_scale(m, "abyss", "abyss", after="orbit") + m = rebuild_ring_edges(m) + files = [t["file"] for t in m["ring"]["transitions"]] + assert files == [ + "transitions/cosmos-orbit.mp4", + "transitions/orbit-abyss.mp4", + "transitions/abyss-cosmos.mp4", # wrap edge + ] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_pipeline_manifest.py -q` +Expected: FAIL with `ModuleNotFoundError: No module named 'tools.pipeline.manifest'` + +- [ ] **Step 3: Write minimal implementation** + +```python +# tools/pipeline/manifest.py +"""Stage 6 — assemble manifest entries (spec §3.6). Pure dict transforms over a +loaded manifest dict; the caller does the json read/write. Idempotent by id so +re-running earlier stages never clobbers authored labels.""" + +from __future__ import annotations + +import copy + + +def upsert_clip(manifest: dict, clip: dict) -> dict: + """Replace the clip with the same id, or append it. Returns a new manifest.""" + m = copy.deepcopy(manifest) + clips = m.setdefault("clips", []) + for i, c in enumerate(clips): + if c["id"] == clip["id"]: + clips[i] = clip + return m + clips.append(clip) + return m + + +def add_ring_scale(manifest: dict, scale_id: str, clip_id: str, + after: str | None = None) -> dict: + """Insert a scale into ring.scales after the named scale (or append if `after` + is None / not found). No-op if scale_id already present. Returns a new manifest.""" + m = copy.deepcopy(manifest) + ring = m.setdefault("ring", {"scales": [], "transitions": []}) + scales = ring.setdefault("scales", []) + if any(s["id"] == scale_id for s in scales): + return m + entry = {"id": scale_id, "clip_id": clip_id} + idx = next((i for i, s in enumerate(scales) if s["id"] == after), None) + if idx is None: + scales.append(entry) + else: + scales.insert(idx + 1, entry) + return m + + +def rebuild_ring_edges(manifest: dict) -> dict: + """Rebuild ring.transitions to one placeholder entry per ring edge (N scales -> + N edges incl. the wrap), preserving any existing model string for an edge. + Returns a new manifest.""" + m = copy.deepcopy(manifest) + ring = m.setdefault("ring", {"scales": [], "transitions": []}) + scales = ring.get("scales", []) + existing = {t["file"]: t.get("model", "") for t in ring.get("transitions", [])} + edges = [] + n = len(scales) + for i in range(n): + a, b = scales[i]["id"], scales[(i + 1) % n]["id"] + file = f"transitions/{a}-{b}.mp4" + edges.append({"file": file, "model": existing.get(file, "placeholder-zoom")}) + ring["transitions"] = edges + return m +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_pipeline_manifest.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add tools/pipeline/manifest.py tests/test_pipeline_manifest.py +git commit -m "feat(pipeline): manifest assembly — upsert clip + ring scale/edges (content pipeline §3.6)" +``` + +--- + +### Task 6: Runner + opt-in integration test on the real forest POC base + +**Files:** +- Create: `tools/pipeline/run.py` +- Test: `tests/test_pipeline_integration.py` + +The runner shells out ffmpeg with the builders from Tasks 1–3, ffprobing the source +duration so the loop stage gets real `D`. The integration test runs the **real** +forest POC base (`simulator/sample_media/forest/base.mp4`, populated by +`simulator/setup_sample_media.py`) through frame → loop → transcode and ffprobes the +proxy — gated on `ffmpeg`/`ffprobe` and on the base existing (it's gitignored media), +matching `tests/test_tools_integration.py`'s gating. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_pipeline_integration.py +import shutil +import subprocess +from pathlib import Path + +import pytest + +from tools.pipeline.run import process_clip + +_HAS_FF = shutil.which("ffmpeg") is not None and shutil.which("ffprobe") is not None +_FOREST = Path("simulator/sample_media/forest/base.mp4") + + +@pytest.mark.skipif(not _HAS_FF, reason="ffmpeg/ffprobe not installed") +@pytest.mark.skipif(not _FOREST.exists(), + reason="forest POC base absent (run simulator/setup_sample_media.py)") +def test_process_clip_emits_proxy_and_master(tmp_path): + out = process_clip(_FOREST, tmp_path, overlap=0.5, start=0.0, duration=4.0) + assert out["proxy"].exists() and out["master"].exists() + # Proxy is exactly 1920x1080. + probe = subprocess.run( + ["ffprobe", "-v", "quiet", "-select_streams", "v:0", + "-show_entries", "stream=width,height", "-of", "csv=p=0", str(out["proxy"])], + capture_output=True, text=True, check=True, + ).stdout.strip() + assert probe == "1920,1080" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_pipeline_integration.py -q` +Expected: FAIL with `ModuleNotFoundError: No module named 'tools.pipeline.run'` +(or SKIP if ffmpeg/the base are absent — then verify by importing in `python -c`.) + +- [ ] **Step 3: Write minimal implementation** + +```python +# tools/pipeline/run.py +"""Thin runner: ffprobe the source, then execute the frame -> loop -> transcode +stages with tools.pipeline.ffmpeg_ops. The pure arg builders are unit-tested; +this glue is covered by the opt-in integration test.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from .ffmpeg_ops import crossfade_loop_args, frame_args, transcode_args + +MASTER_MAX_W = 3840 + + +def probe_duration(src, *, ff: str = "ffprobe") -> float: + out = subprocess.run( + [ff, "-v", "quiet", "-show_entries", "format=duration", + "-of", "csv=p=0", str(src)], + capture_output=True, text=True, check=True, + ).stdout.strip() + return float(out) + + +def _run(args: list[str]) -> None: + subprocess.run(args, check=True, capture_output=True) + + +def process_clip(src, out_dir, *, start: float = 0.0, duration: float | None = None, + overlap: float = 1.0, master_w: int = 3840, master_h: int = 2160, + ff: str = "ffmpeg") -> dict: + """Run stages 2–4 for one clip; return {'master','proxy','loop','mezzanine'} paths. + `duration` defaults to the full source length (minus the trim start).""" + src = Path(src) + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + if duration is None: + duration = probe_duration(src) - start + + mezz = out_dir / "mezzanine.mp4" + loop = out_dir / "loop.mp4" + master = out_dir / "master.mp4" + proxy = out_dir / "base.mp4" # the proxy is what the manifest's base_file points at + + _run(frame_args(src, mezz, start=start, duration=duration, + width=master_w, height=master_h, ff=ff)) + _run(crossfade_loop_args(mezz, loop, duration=duration, overlap=overlap, ff=ff)) + _run(transcode_args(loop, master, width=master_w, height=master_h, crf=18, ff=ff)) + _run(transcode_args(loop, proxy, width=1920, height=1080, crf=20, ff=ff)) + return {"mezzanine": mezz, "loop": loop, "master": master, "proxy": proxy} +``` + +- [ ] **Step 4: Run test to verify it passes (or skips cleanly)** + +Run: `pytest tests/test_pipeline_integration.py -q` +Expected: PASS if ffmpeg + the forest base are present; otherwise SKIP with the gate +reason. If skipped, also run `python -c "from tools.pipeline.run import process_clip"` +and expect no error. + +- [ ] **Step 5: Commit** + +```bash +git add tools/pipeline/run.py tests/test_pipeline_integration.py +git commit -m "feat(pipeline): runner + opt-in integration test on real forest base (content pipeline §3)" +``` + +--- + +### Task 7: Client annotation-track interpolation + a hand-authored track + +**Files:** +- Modify: `simulator/static/app.js` (`renderOverlay` at lines ~235–269; `paintLoop` at ~173–186) +- Modify: `simulator/sample_media/manifest.json` (add a `track` to the forest `detected.water` annotation) + +The track is *data* on an existing annotation; the Python `Clip` already passes +`annotations` through opaquely (`simulator/clips.py` `_clip_from_dict` does +`d.get("annotations", [])`), so **no Python change is needed**. Interpolation is +client-side (spec §4). Because the overlay currently re-renders only on knob change, +a tracked box must update as the video plays — drive a re-render from the existing +`paintLoop` rAF when the active clip has any track and the overlay is visible. + +- [ ] **Step 1: Add the data — a hand-authored track on forest's water detection** + +In `simulator/sample_media/manifest.json`, replace the forest `detected.water` +annotation (currently `{"key": "detected.water", "box": [0.30, 0.10, 0.18, 0.70], "min_level": 1}`) +with a keyframed track (a small horizontal drift, looping at t=0 and t=1 so it is +seamless): + +```json +{ + "key": "detected.water", + "min_level": 1, + "track": [ + {"t": 0.0, "box": [0.30, 0.10, 0.18, 0.70]}, + {"t": 0.5, "box": [0.33, 0.10, 0.18, 0.70]}, + {"t": 1.0, "box": [0.30, 0.10, 0.18, 0.70]} + ] +} +``` + +- [ ] **Step 2: Add the interpolation helper + use it in `renderOverlay`** + +In `simulator/static/app.js`, add a helper above `renderOverlay` and use it instead +of `a.box`: + +```javascript +// Interpolate an annotation's box at loop-normalized time t (0..1). Static labels +// (no track) return their fixed box. Linear between sorted keyframes; clamps to ends. +function boxAt(a, t) { + if (!a.track || !a.track.length) return a.box; + const ks = a.track; + if (t <= ks[0].t) return ks[0].box; + if (t >= ks[ks.length - 1].t) return ks[ks.length - 1].box; + for (let i = 1; i < ks.length; i++) { + if (t <= ks[i].t) { + const a0 = ks[i - 1], a1 = ks[i]; + const f = (t - a0.t) / (a1.t - a0.t); + return a0.box.map((v, j) => v + (a1.box[j] - v) * f); + } + } + return ks[ks.length - 1].box; +} + +// Loop-normalized playback time of the base video (0..1), for track interpolation. +function loopT() { + return vid && vid.duration ? (vid.currentTime % vid.duration) / vid.duration : 0; +} +``` + +Then in `renderOverlay`, change the box line: + +```javascript + const [x, y, w, h] = boxAt(a, loopT()).map((n) => n * 100); +``` + +- [ ] **Step 3: Re-render tracked overlays each frame from `paintLoop`** + +Store the last overlay args and, in `paintLoop`, re-run `renderOverlay` when the +active clip has any tracked annotation and the overlay is visible. Add near the top +of `app.js` (module scope): `let lastOverlay = null;`. At the end of `renderOverlay`, +record the call: set `lastOverlay = { level, intensity };` (add this line just before +the closing brace). In `paintLoop` (after the existing shader work, before +`requestAnimationFrame(paintLoop)`), add: + +```javascript + // Animate keyframed annotation tracks: re-place boxes against playback time. + const c = activeClip(); + if (lastOverlay && lastOverlay.level > 0 && c && + c.annotations.some((a) => a.track && a.track.length)) { + renderOverlay(lastOverlay.level, lastOverlay.intensity); + } +``` + +- [ ] **Step 4: Verify by eye (no JS unit harness in this repo)** + +Run the sim and watch the forest water reticle drift with playback, looping +seamlessly; static labels (rock_face, conifer, the measurement) stay put. + +```bash +python -m simulator.app # then open the printed localhost URL +``` + +Expected: with Left up on forest, the `flowing water` box drifts right then back +over the loop with no jump at the loop seam; affect/measure labels unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add simulator/static/app.js simulator/sample_media/manifest.json +git commit -m "feat(sim): client annotation-track interpolation + hand-authored forest track (content pipeline §4)" +``` + +--- + +### Task 8: Extend the ring to 5 scales (orbit + reef placeholders) + +**Files:** +- Modify: `simulator/setup_scales_media.py` (`SCALES` lines ~32–36; `EDGES` line ~39) +- Modify: `simulator/sample_media/manifest.json` (add `orbit` + `reef` clips; reorder `ring.scales`; rebuild `ring.transitions`) + +Real strict-PD bases are sourced separately (Task 9); to keep the 5-scale ring +demonstrable now, generate labelled placeholders for the two new scales with the +existing generator, exactly as cosmos/abyss are today. + +- [ ] **Step 1: Add orbit + reef to the placeholder generator** + +In `simulator/setup_scales_media.py`, extend `SCALES` and `EDGES` to the 5-scale +ring (order: cosmos → orbit → forest → reef → abyss → wrap): + +```python +SCALES = { + "cosmos": ("0x05060f", "COSMOS — NASA/Hubble (PD placeholder)"), + "orbit": ("0x081427", "ORBIT — NASA/ISS (PD placeholder)"), + "forest": ("0x12301a", "FOREST — NPS/USGS (POC/placeholder)"), + "reef": ("0x06303a", "REEF — NOAA Ocean Exploration (PD placeholder)"), + "abyss": ("0x021016", "ABYSS — NOAA Ocean Exploration (PD placeholder)"), +} + +EDGES = [("cosmos", "orbit"), ("orbit", "forest"), ("forest", "reef"), + ("reef", "abyss"), ("abyss", "cosmos")] +``` + +- [ ] **Step 2: Add the orbit + reef clip entries to the manifest** + +In `simulator/sample_media/manifest.json`, add two clip objects (after `cosmos` and +after `forest` respectively) mirroring the existing placeholder clips' shape, e.g. +orbit: + +```json +{ + "id": "orbit", + "title": "Earth from orbit (NASA/ISS, neutral base)", + "base_file": "orbit/base.mp4", + "license": "public-domain (US Gov, 17 U.S.C. §105)", + "source": "NASA/ISS — https://images.nasa.gov/ (true PD; placeholder bytes until ingest)", + "right_variants": {}, + "annotations": [ + {"key": "detected.cloud_band", "box": [0.30, 0.30, 0.30, 0.20], "min_level": 1}, + {"key": "measure.altitude", "box": [0.06, 0.06, 0.18, 0.08], "min_level": 2} + ], + "affect": [ + {"key": "feel.serenity", "at": [0.50, 0.46], "min_level": 1}, + {"key": "feel.fragility", "at": [0.22, 0.68], "min_level": 2}, + {"key": "feel.unity", "at": [0.66, 0.58], "min_level": 3}, + {"key": "feel.distance", "at": [0.42, 0.82], "min_level": 4} + ], + "strings": { + "en": { + "detected.cloud_band": "cloud band", + "measure.altitude": "~408 km", + "feel.serenity": "serenity", "feel.fragility": "fragility", + "feel.unity": "unity", "feel.distance": "distance" + } + } +} +``` + +And reef: + +```json +{ + "id": "reef", + "title": "Coral reef (NOAA Ocean Exploration, neutral base)", + "base_file": "reef/base.mp4", + "license": "public-domain (US Gov, 17 U.S.C. §105)", + "source": "NOAA Ocean Exploration — https://oceanexplorer.noaa.gov/ (true PD, worldwide; placeholder bytes until ingest)", + "right_variants": {}, + "annotations": [ + {"key": "detected.coral", "box": [0.20, 0.45, 0.30, 0.30], "min_level": 1}, + {"key": "detected.fish", "min_level": 2, + "track": [ + {"t": 0.0, "box": [0.20, 0.30, 0.10, 0.08]}, + {"t": 0.5, "box": [0.62, 0.34, 0.10, 0.08]}, + {"t": 1.0, "box": [0.20, 0.30, 0.10, 0.08]} + ]}, + {"key": "measure.depth", "box": [0.06, 0.06, 0.16, 0.08], "min_level": 3} + ], + "affect": [ + {"key": "feel.delight", "at": [0.50, 0.46], "min_level": 1}, + {"key": "feel.abundance", "at": [0.22, 0.68], "min_level": 2}, + {"key": "feel.curiosity", "at": [0.66, 0.58], "min_level": 3}, + {"key": "feel.immersion", "at": [0.42, 0.82], "min_level": 4} + ], + "strings": { + "en": { + "detected.coral": "coral colony", "detected.fish": "reef fish", + "measure.depth": "−18 m", + "feel.delight": "delight", "feel.abundance": "abundance", + "feel.curiosity": "curiosity", "feel.immersion": "immersion" + } + } +} +``` + +- [ ] **Step 3: Reorder the ring + rebuild edges in the manifest** + +Set `ring.scales` to the 5-scale order and `ring.transitions` to the 5 edges: + +```json +"ring": { + "scales": [ + {"id": "cosmos", "clip_id": "cosmos"}, + {"id": "orbit", "clip_id": "orbit"}, + {"id": "forest", "clip_id": "forest"}, + {"id": "reef", "clip_id": "reef"}, + {"id": "abyss", "clip_id": "abyss"} + ], + "transitions": [ + {"file": "transitions/cosmos-orbit.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/orbit-forest.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/forest-reef.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/reef-abyss.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/abyss-cosmos.mp4", "model": "placeholder-zoom"} + ] +} +``` + +- [ ] **Step 4: Regenerate placeholder media + verify the 5-scale ring** + +```bash +python simulator/setup_scales_media.py +python -m simulator.app # open the URL; spin the encoder full circle +``` + +Expected: the encoder walks cosmos → orbit → forest → reef → abyss → (wrap) cosmos +with a transition on every edge; `GET /api/ring` lists 5 scales + 5 transitions; the +reef `reef fish` box drifts with playback (Task 7 interpolation) when Left ≥ 2. + +- [ ] **Step 5: Commit** + +```bash +git add simulator/setup_scales_media.py simulator/sample_media/manifest.json +git commit -m "feat(sim): extend scale ring to 5 (orbit + reef) (content pipeline §5)" +``` + +--- + +### Task 9: Sourcing procedure doc + ROADMAP update + +**Files:** +- Create: `docs/content-sourcing.md` +- Modify: `docs/ROADMAP.md` (sub-project 3 "Offline v2v variant pipeline" bullet, lines ~174–187) + +Sourcing the real strict-PD bytes is a curatorial task, not a code step. Document the +procedure so anyone can run the pipeline per clip, and record the new content-pipeline +work on the roadmap. + +- [ ] **Step 1: Write the sourcing procedure** + +```markdown +# Content sourcing — strict-PD neutral bases + +For each scale, acquire a calm, neutral source clip from a **strict public-domain** +US-government source, verify the license, then run it through the pipeline. + +| Scale | Source | Where | +|--------|--------------------------------|------------------------------------| +| cosmos | NASA / Hubble / JWST | https://images.nasa.gov/ | +| orbit | NASA / ISS | https://images.nasa.gov/ | +| forest | NPS / USGS | https://www.nps.gov/ , https://www.usgs.gov/ | +| reef | NOAA Ocean Exploration | https://oceanexplorer.noaa.gov/ | +| abyss | NOAA Ocean Exploration | https://oceanexplorer.noaa.gov/ | + +**License check (mandatory):** confirm the asset is US-gov PD (17 U.S.C. §105) or +explicitly CC0. "Free stock" (Pexels/Pixabay/etc.) is royalty-free but NOT public +domain — do not use. Record a `Provenance` (tools/pipeline/provenance.py) per clip. + +**Run the pipeline** (replaces the placeholder bytes for a scale): + +```bash +python -c "from tools.pipeline.run import process_clip; \ + process_clip('downloads/.mp4', 'simulator/sample_media/', \ + start=, duration=, overlap=<0.5..1.0>)" +``` + +This writes `simulator/sample_media//base.mp4` (proxy) + `master.mp4`. Then +author the labels (Increment 2: the hybrid track pass + simulator author mode) or +hand-edit the annotation track for now (spec §4). The ring + manifest entry already +exist (Task 8); `process_clip` only replaces the media bytes. +``` + +- [ ] **Step 2: Update the roadmap** + +In `docs/ROADMAP.md`, under sub-project 3, replace the "Offline v2v variant pipeline" +framing with the content-pipeline reality: the Right dream is real-time (no ML bake), +so the content pipeline is **source → crossfade-loop → master+proxy transcode → +hand-authored/motion-tracked labels → manifest**. Note Increment 1 (this plan: tooling ++ track schema + 5-scale ring, no ML) done on merge; Increment 2 (hybrid track pass + +simulator author mode) and real i2v ring transitions still pending. Cite the spec +`docs/superpowers/specs/2026-06-24-content-pipeline-design.md`. + +- [ ] **Step 3: Run the full suite** + +Run: `pytest -q` +Expected: all prior tests still pass + the new pipeline tests (ffmpeg_ops, +provenance, manifest) pass; the integration test passes or skips cleanly. + +- [ ] **Step 4: Commit** + +```bash +git add docs/content-sourcing.md docs/ROADMAP.md +git commit -m "docs: content-sourcing procedure + roadmap update (content pipeline Increment 1)" +``` + +--- + +## Notes for the executor + +- **Run from the main clone, not a nested worktree** — `resolve-app.py` errors + "ambiguous" when a `.worktrees/` checkout shadows the main clone (session-0013 + gotcha). If you must use a worktree, expect the `WGL_APP_SCAN_DEPTH` / explicit + `publish-transcript.sh` flag workarounds. +- **PRs go through the Gitea API** (`wgl-gitea-admin` `gitea-api.sh`, host + `git.benstull.org`) — no `tea`/`gh` here: create via `POST /repos/.../pulls`, + merge via `.../merge`. +- The project `.venv` has the deps (pytest, cv2/torch for later ML work). Activate it + before running tests. +- **Increment 2 (separate plan):** `tools/pipeline/track.py` (classical OpenCV + tracker + optional lazy-imported ML detect+track) and the simulator **author mode** + (draw/seed/correct boxes, assign keys/min_level/strings, place affect, write the + manifest). Plus real i2v ring transitions once a generative-video model + adjacent + real bases exist. +``` diff --git a/docs/superpowers/specs/2026-06-24-content-pipeline-design.md b/docs/superpowers/specs/2026-06-24-content-pipeline-design.md new file mode 100644 index 0000000..a5b2107 --- /dev/null +++ b/docs/superpowers/specs/2026-06-24-content-pipeline-design.md @@ -0,0 +1,248 @@ +# HEF — Content Pipeline (source → process → label → manifest) + +**Date:** 2026-06-24 +**Status:** Proposed design (pre-implementation) — session 0014 +**Repo:** `human-experience-filter-art` +**Anchor:** `docs/ROADMAP.md` sub-project 3 (Player Runtime) content needs; the +spiritual successor to sub-project 2 (Ingest & Tagging tools), retargeted from the +retired coordinate-catalog model to the manifest content model. +**Refines:** [`2026-06-07-scales-library-and-right-axis-pipeline-design.md`](./2026-06-07-scales-library-and-right-axis-pipeline-design.md) +§2 (scales library), §2.1 (strict-PD sourcing), §6 (open sourcing questions). +**Builds on:** [`2026-06-22-right-axis-deterministic-dream-design.md`](./2026-06-22-right-axis-deterministic-dream-design.md) +(the real-time dream — **no per-clip ML bake**) and +[`2026-06-22-affect-channel-hud-design.md`](./2026-06-22-affect-channel-hud-design.md) +(scene-anchored affect words). + +--- + +## 1. Why this exists + +The installation has an alteration engine, a scale ring, a Left analytical HUD, an +affect channel, and a real-time Right "dream" — but it has **no real content**. The +three ring scales today are: `cosmos` and `abyss` carrying *placeholder bytes* (PD +provenance recorded, but the media is synthetic), and `forest` carrying a *POC +sample* (a trimmed Yosemite clip licensed "look-tuning only; not shipped content"). +There is no repeatable way to turn a public-domain source clip into a shippable, +seamless-looping, labelled neutral base. + +This design defines that pipeline: **source → frame → seamless-loop → transcode → +label/author → manifest entry.** One big anchor fact reshapes it versus the old +plan — the Right dream is now a **real-time deterministic shader** (session 0013), +so there is **no per-clip ML restyle bake** anymore. The pipeline's offline cost is +therefore **transcode + human authoring**, not generative-video baking. The one +deliberate exception is the motion-tracking label pass chosen this session (§3), +which re-introduces a bounded offline compute step — by choice, for realism. + +## 2. Decisions settled this session + +| # | Decision | Choice | Consequence | +|---|----------|--------|-------------| +| 1 | Scale set | **Lean 5** — cosmos · orbit · forest/land · reef · abyss (microscopic dropped) | Ring wraps **abyss → cosmos**; replaces the POC forest + both placeholder bases with real strict-PD footage | +| 2 | Seamless loop | **Crossfade loop** (tail→head overlap) | Any source clip becomes an indefinite seamless dwell loop; a transcode-time step | +| 3 | Label placement | **Motion-tracked, hybrid** | ML proposes tracks where it can; hand-seed + classical tracker elsewhere. Re-introduces an offline pass + an authoring step. Semantics always hand-authored | +| 4 | Format | **Master + 1080p proxy** | High-quality master retained per clip; a 1080p H.264 proxy drives the sim; Pi chooses later | + +Microscopic was dropped because PD microscopy *video* (vs stills) is the thinnest +pool and hardest to source well; it can be added later by re-running the pipeline. + +## 3. The pipeline — six stages + +Each stage is a deterministic unit with one job (handbook §4.2: deterministic tools, +not prose an agent re-derives). Stages 1–4 and 6 are mechanical; stage 5 is the one +human-in-the-loop step. + +``` + (1) source ──▶ (2) frame ──▶ (3) loop ──▶ (4) transcode ──▶ (6) manifest + provenance trim/crop crossfade master + proxy entry + & scale seam │ ▲ + └──▶ (5) label/author (hybrid track) +``` + +### 3.1 Stage 1 — source & provenance +Acquire the strict-PD source clip and record license + source URL + capture notes. +Reuse sub-project-2 fetchers where they map (a NASA fetcher already exists); add +NOAA Ocean Exploration / NPS / USGS fetchers (or accept a manual download + a +provenance record). The "no explicit license → assume PD, verify" trap from the +scales design §2.1 applies: **"free stock" (Pexels/Pixabay) is NOT public domain.** +Output: the raw source file + a provenance record (license, source, url, fetched_at). + +### 3.2 Stage 2 — frame (trim / crop / scale) +Pick the in/out points; crop to the target aspect; scale toward the format target +(§6). The panoramic projector aspect is still unmeasured (open question), so v1 +frames to a standard wide 16:9 for the sim; the master retains maximum available +width so a later pano crop/extension has pixels to work with. Output: a framed, +silent (audio stripped — bases are video-only; audio is the separate content-dial +channel) intermediate. + +### 3.3 Stage 3 — seamless crossfade loop +Make the clip loop indefinitely with no visible seam by overlapping the tail into +the head with a short crossfade (~0.5–1.0 s). The final looped clip is shorter than +the source by the overlap; during the overlap region motion is a blend (acceptable +for calm ambient nature footage). This is the chosen loop strategy (clean-cut and +palindrome were rejected — seam risk and reversed motion respectively). Output: a +loop-clean clip whose first and last frames join seamlessly. + +### 3.4 Stage 4 — transcode (master + proxy) +Emit two encodes per clip (§6 for exact params): a **master** (high quality, up to +4K, retained as the archival/Pi-candidate source) and a **1080p H.264 proxy** (what +the sim plays today, matching current `base.mp4` behaviour). Both video-only, +`yuv420p`, even dimensions, `+faststart`. + +### 3.5 Stage 5 — label & author (the hybrid motion-track pass) +The only human-in-the-loop stage, and the only one that adds offline ML compute. + +- **Semantics are always hand-authored.** A generic detector cannot produce the + installation's semantic keys — "spiral galaxy", "siphonophore", `z ≈ 0.04`, + `~2.1 m³/s` are out-of-distribution. The author owns the label **keys**, + **strings**, and `min_level`; tracking only propagates **box geometry**. +- **Hybrid geometry (decision 3).** Where a model can lock onto a subject (a fish + on the reef, an animal in the forest), an ML detect+track pass (YOLO+ByteTrack / + SAM2-class, lazy-imported like the parked baker imported torch) proposes a track + the author maps to a key. Where it can't (cosmos, abyss, microscopic-type + content), the author **hand-seeds** a box on a keyframe and a **classical tracker** + (OpenCV CSRT / optical-flow) propagates it; the author fixes drift. Both paths + emit the same artifact. +- **Channel rules.** `detected.*` object labels **track**; `measure.*` readouts and + `feel.*` affect words stay **static / scene-anchored** (a depth readout or "awe" + is not pinned to a moving pixel). This matches the affect-channel design (affect + is scene-level) and keeps measurement chips legible. +- Output: an **annotation track** per object label (§4) plus the static measurement + and affect placements, all keyed and stringed. + +### 3.6 Stage 6 — manifest assembly +Write the clip entry (base/proxy/master paths, license, source, annotations with +tracks, affect, strings) into `simulator/sample_media/manifest.json`, and register +the scale in the `ring` section with its transition edges (§5). Idempotent: stage 6 +owns the manifest entry so re-running earlier stages never clobbers authored labels +(mirrors how `setup_sample_media.py` was made to never clobber a real bake). + +## 4. Data model — annotation tracks + +Today an annotation is a single static box: `{"key", "box":[x,y,w,h], "min_level"}`. +Add an **optional keyframed track**; static labels are unchanged (backward +compatible): + +```json +{ + "key": "detected.fish", + "min_level": 1, + "track": [ + {"t": 0.0, "box": [0.30, 0.40, 0.10, 0.08]}, + {"t": 0.5, "box": [0.52, 0.38, 0.10, 0.08]}, + {"t": 1.0, "box": [0.30, 0.40, 0.10, 0.08]} + ] +} +``` + +- `t` is **normalized 0→1 over the looped clip** — resolution- and fps-independent, + so the same track is correct for both master and proxy and survives a re-transcode. + (A track that loops cleanly has matching boxes at `t=0` and `t=1`.) +- **Runtime:** if `track` is present, the client interpolates the box at the current + loop-normalized playback time (linear between keyframes); otherwise it uses the + static `box`. This lives **client-side**, consistent with the existing split + (alteration math in Python; label selection/placement in the client). `Clip` gains + the track only as data inside `annotations`; `player/alteration.py` is **unchanged** + (labels are not part of `RenderPlan`). +- `measure.*` keep a single static `box`; `feel.*` keep a single static `at:[x,y]`. + +## 5. The 5 scales, sources, and ring + +| Scale | Strict-PD source | Notes | +|-------|------------------|-------| +| `cosmos` | NASA / Hubble / JWST | 🟢 abundant true PD (17 U.S.C. §105); replaces placeholder | +| `orbit` | NASA / ISS (Earth from orbit) | 🟢 true PD; **new** scale | +| `forest` | NPS / USGS (US land/wildlife) | 🟢 true PD (US locations); replaces the non-shippable POC sample | +| `reef` | NOAA Ocean Exploration (shallow) | 🟢 true PD worldwide; **new** scale | +| `abyss` | NOAA Ocean Exploration (deep sea) | 🟢 true PD worldwide; replaces placeholder | + +**Ring order (large → small, zooms inward; wraps small → large):** +`cosmos → orbit → forest → reef → abyss → (wrap) cosmos`. Five scales → five ring +edges. The wrap seam is now **abyss → cosmos** (deep sea back out to the cosmos) — +the infinite-zoom payoff, minus the microscopic step. + +**Transitions** between the new adjacent edges remain the **deferred i2v generative +work** (scales design §3 / §6) — they need a generative-video model and two real +adjacent bases, heavier than anything in this pipeline. Until then new edges carry +**placeholder transitions** (the `setup_scales_media.py` ffmpeg `xfade=zoomin` +generator already produces these), exactly as cosmos/forest/abyss do today. This +pipeline's job is the **bases + labels**; transitions are tracked but out of scope +here (§8). + +## 6. Format target + +| | Master | Proxy (sim) | +|---|--------|-------------| +| Resolution | source, capped ~3840 wide | 1920×1080 (fit wide) | +| Codec | H.264 High | H.264 High | +| Quality | CRF ~18 | CRF ~20 | +| FPS | preserve source (cap 30) | 30 | +| Pixel fmt | `yuv420p`, even dims | `yuv420p`, even dims | +| Audio | none (stripped) | none (stripped) | +| Flags | `+faststart` | `+faststart` | + +The sim plays the **proxy** as today's `base_file`. The **master** is retained per +clip for the eventual Pi/pano encode once the panoramic resolution is measured. +Browser HEVC support is too spotty to make HEVC the sim path, which is why the proxy +stays H.264 (the master can be re-encoded to HEVC for the Pi later without +re-sourcing). + +## 7. Tooling & where it lives + +- **New `tools/pipeline/` package** — deterministic ffmpeg-backed verbs for stages + 1–4 and 6 (`fetch`/provenance, `frame`, `loop`, `transcode`, `manifest`), reusing + sub-project-2 fetchers where valid. This **supersedes** the coordinate-era + ingest *drafting* (the `left/right/dark/light` Record model retired with the + machine-altered-perception pivot); the pipeline targets the **manifest**, not + catalog `Record`s. +- **Stage 5 tracking** — `tools/pipeline/track.py`: classical OpenCV tracker + (always available) + an optional lazy-imported ML detect+track path. Emits + candidate keyframed tracks for the author to accept/correct. +- **Authoring surface** — extend the **simulator** with an **author mode** (it + already renders the exact HUD overlay and is the by-eye judge): scrub the clip, + draw/seed/adjust boxes, run the tracker, assign keys/`min_level`/strings, place + affect anchors, write the manifest entry. Reusing the preview as the editor keeps + "what you author is what plays." **This is the largest new piece** — see the + delivery order (§8), which lets v1 prove the schema before the full editor exists. + +## 8. Delivery / rollout order + +Build order within this Feature (Solution-Design rollout strategy — not a task +list): + +1. **Increment 1 — real footage, no ML.** Stages 1–4 + 6 + the §4 track schema + + client interpolation. Source the 5 strict-PD bases, crossfade-loop them, emit + master+proxy, wire them into the ring. Author a *couple* of tracks **by hand as + JSON** to prove the track schema and the runtime interpolation. Outcome: the + whole ring is **real, seamless-looping footage** judgeable by eye — the + highest-value, lowest-risk step, and it stands alone. +2. **Increment 2 — the hybrid track pass + author mode.** `track.py` (classical + + optional ML) and the simulator author mode; label all five scales properly. + +**Deferred (tracked, not built here):** the real **i2v ring transitions** for the +new edges (need a generative-video model + adjacent real bases); the Pi renderer + +serial/firmware (per the simulator-first directive); the panoramic-resolution +master encode (waits on a measured pano spec). + +> Scope note: if Increment 2's author mode grows large enough to be its own +> increment in practice, split it at execution time. The design is one coherent +> pipeline; the increments above are its sensible build order, not separate +> Features. + +## 9. Open questions (for the plan, not blockers) + +- **Panoramic resolution/aspect** — unmeasured (scales design §6). Drives the master + crop/extension strategy; v1 frames 16:9 for the sim and retains max width. +- **ML model choice** for the hybrid track path (YOLO+ByteTrack vs SAM2-class) and + whether it earns its place given how few scales it can usefully fire on. +- **Classical tracker drift** on long/fast clips — keyframe interval, re-seed cadence. +- **Source fetchers** — which of NOAA / NPS / USGS get real fetchers vs manual + download + provenance record for v1. +- **Crossfade overlap length** per scale (slow cosmos vs busier reef) — by eye. + +## 10. Out of scope (YAGNI) + +- Generative i2v **ring transitions** for the new edges (deferred; placeholders hold). +- **Audio** sourcing/alteration (separate content-dial channel; already deferred). +- The **Pi/pano** encode + renderer and serial/firmware (simulator-first). +- Reviving the retired coordinate-catalog `Record` drafting. +- Per-frame **dense** tracks (sparse keyframed tracks interpolated at runtime instead). diff --git a/simulator/sample_media/manifest.json b/simulator/sample_media/manifest.json index 3f79f74..20ea3e1 100644 --- a/simulator/sample_media/manifest.json +++ b/simulator/sample_media/manifest.json @@ -28,6 +28,32 @@ } } }, + { + "id": "orbit", + "title": "Earth from orbit (NASA/ISS, neutral base)", + "base_file": "orbit/base.mp4", + "license": "public-domain (US Gov, 17 U.S.C. §105)", + "source": "NASA/ISS — https://images.nasa.gov/ (true PD; placeholder bytes until ingest)", + "right_variants": {}, + "annotations": [ + {"key": "detected.cloud_band", "box": [0.30, 0.30, 0.30, 0.20], "min_level": 1}, + {"key": "measure.altitude", "box": [0.06, 0.06, 0.18, 0.08], "min_level": 2} + ], + "affect": [ + {"key": "feel.serenity", "at": [0.50, 0.46], "min_level": 1}, + {"key": "feel.fragility", "at": [0.22, 0.68], "min_level": 2}, + {"key": "feel.unity", "at": [0.66, 0.58], "min_level": 3}, + {"key": "feel.distance", "at": [0.42, 0.82], "min_level": 4} + ], + "strings": { + "en": { + "detected.cloud_band": "cloud band", + "measure.altitude": "~408 km", + "feel.serenity": "serenity", "feel.fragility": "fragility", + "feel.unity": "unity", "feel.distance": "distance" + } + } + }, { "id": "forest", "title": "Yosemite Falls (neutral base, POC sample)", @@ -41,7 +67,11 @@ "4": {"file": "forest/right4.mp4", "model": "sd-turbo+farneback-flow"} }, "annotations": [ - {"key": "detected.water", "box": [0.30, 0.10, 0.18, 0.70], "min_level": 1}, + {"key": "detected.water", "min_level": 1, "track": [ + {"t": 0.0, "box": [0.30, 0.10, 0.18, 0.70]}, + {"t": 0.5, "box": [0.33, 0.10, 0.18, 0.70]}, + {"t": 1.0, "box": [0.30, 0.10, 0.18, 0.70]} + ]}, {"key": "detected.rock_face", "box": [0.05, 0.30, 0.20, 0.55], "min_level": 2}, {"key": "detected.conifer", "box": [0.70, 0.20, 0.22, 0.45], "min_level": 3}, {"key": "measure.flow_rate", "box": [0.34, 0.55, 0.14, 0.08], "min_level": 4} @@ -65,6 +95,38 @@ } } }, + { + "id": "reef", + "title": "Coral reef (NOAA Ocean Exploration, neutral base)", + "base_file": "reef/base.mp4", + "license": "public-domain (US Gov, 17 U.S.C. §105)", + "source": "NOAA Ocean Exploration — https://oceanexplorer.noaa.gov/ (true PD, worldwide; placeholder bytes until ingest)", + "right_variants": {}, + "annotations": [ + {"key": "detected.coral", "box": [0.20, 0.45, 0.30, 0.30], "min_level": 1}, + {"key": "detected.fish", "min_level": 2, + "track": [ + {"t": 0.0, "box": [0.20, 0.30, 0.10, 0.08]}, + {"t": 0.5, "box": [0.62, 0.34, 0.10, 0.08]}, + {"t": 1.0, "box": [0.20, 0.30, 0.10, 0.08]} + ]}, + {"key": "measure.depth", "box": [0.06, 0.06, 0.16, 0.08], "min_level": 3} + ], + "affect": [ + {"key": "feel.delight", "at": [0.50, 0.46], "min_level": 1}, + {"key": "feel.abundance", "at": [0.22, 0.68], "min_level": 2}, + {"key": "feel.curiosity", "at": [0.66, 0.58], "min_level": 3}, + {"key": "feel.immersion", "at": [0.42, 0.82], "min_level": 4} + ], + "strings": { + "en": { + "detected.coral": "coral colony", "detected.fish": "reef fish", + "measure.depth": "−18 m", + "feel.delight": "delight", "feel.abundance": "abundance", + "feel.curiosity": "curiosity", "feel.immersion": "immersion" + } + } + }, { "id": "abyss", "title": "Deep sea (NOAA Ocean Exploration, neutral base)", @@ -97,12 +159,16 @@ "ring": { "scales": [ {"id": "cosmos", "clip_id": "cosmos"}, + {"id": "orbit", "clip_id": "orbit"}, {"id": "forest", "clip_id": "forest"}, + {"id": "reef", "clip_id": "reef"}, {"id": "abyss", "clip_id": "abyss"} ], "transitions": [ - {"file": "transitions/cosmos-forest.mp4", "model": "placeholder-zoom"}, - {"file": "transitions/forest-abyss.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/cosmos-orbit.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/orbit-forest.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/forest-reef.mp4", "model": "placeholder-zoom"}, + {"file": "transitions/reef-abyss.mp4", "model": "placeholder-zoom"}, {"file": "transitions/abyss-cosmos.mp4", "model": "placeholder-zoom"} ] } diff --git a/simulator/setup_scales_media.py b/simulator/setup_scales_media.py index 54cd32f..2c3a767 100644 --- a/simulator/setup_scales_media.py +++ b/simulator/setup_scales_media.py @@ -1,16 +1,18 @@ -"""Generate cheap placeholder media for the scale RING (scales design §3). +"""Generate cheap placeholder media for the full 5-scale RING (scales design §3). -The ring needs >=2 neutral scales to be demonstrable. This script produces -cheap, offline, synthetic placeholders for the two true-PD scales the ring adds — -**cosmos** (NASA/Hubble) and **abyss** (NOAA Ocean Exploration) — plus the short -zoom/warp **transition** clips between adjacent scales. Forest reuses the real -POC base from `setup_sample_media.py` (or a synthetic placeholder if absent). +Produces offline synthetic placeholders for all five ring scales — **cosmos** +(NASA/Hubble), **orbit** (NASA/ISS), **forest** (NPS/USGS), **reef** (NOAA Ocean +Exploration), and **abyss** (NOAA Ocean Exploration) — plus the short zoom/warp +**transition** clips between adjacent scales. Forest reuses the real POC base from +`setup_sample_media.py` (or a synthetic placeholder if absent); all others are +generated as labelled solid-tint clips. This is deliberately the CHEAP path: the manifest records the real true-PD -provenance (NASA 17 U.S.C. §105 / NOAA PD), but the bytes here are labelled -synthetic placeholders for look-tuning the ring navigation. The expensive real -multi-strength flow-stabilized Right re-bake is DEFERRED until the ring is liked, -so the new scales carry a raw base only (no Right variants). +provenance (NASA / NPS / USGS / NOAA, 17 U.S.C. §105), but the bytes here are +labelled synthetic placeholders for look-tuning the ring navigation. The Right dream +is real-time (Kuwahara WebGL shader), so no per-clip ML bake is needed; each scale +carries a raw base only. Real strict-PD bases replace these placeholders via the +content pipeline (see `docs/content-sourcing.md`). Usage: python simulator/setup_scales_media.py Requires: ffmpeg on PATH (or `pip install imageio-ffmpeg`). @@ -31,12 +33,15 @@ FPS = 25 # Each scale: (dir, base color, on-screen placeholder label). SCALES = { "cosmos": ("0x05060f", "COSMOS — NASA/Hubble (PD placeholder)"), - "forest": ("0x12301a", "FOREST — Yosemite (POC/placeholder)"), + "orbit": ("0x081427", "ORBIT — NASA/ISS (PD placeholder)"), + "forest": ("0x12301a", "FOREST — NPS/USGS (POC/placeholder)"), + "reef": ("0x06303a", "REEF — NOAA Ocean Exploration (PD placeholder)"), "abyss": ("0x021016", "ABYSS — NOAA Ocean Exploration (PD placeholder)"), } # Ring edges (adjacent pairs + the abyss->cosmos wrap), matching the manifest. -EDGES = [("cosmos", "forest"), ("forest", "abyss"), ("abyss", "cosmos")] +EDGES = [("cosmos", "orbit"), ("orbit", "forest"), ("forest", "reef"), + ("reef", "abyss"), ("abyss", "cosmos")] def _ffmpeg() -> str: diff --git a/simulator/static/app.js b/simulator/static/app.js index ac01e1a..bdf6bad 100644 --- a/simulator/static/app.js +++ b/simulator/static/app.js @@ -35,6 +35,7 @@ let ring = null; // {scales:[{id,clip_id,title}], transitions:[...]} o let ringIndex = 0; // current scale position on the ring let currentClipId = null; // base media currently loaded (reset to force a reload) let busy = false; // true while a ring transition is playing +let lastOverlay = null; // {level, intensity} from the most-recent renderOverlay call; drives per-frame track animation async function loadData() { const clips = (await (await fetch("/api/clips")).json()).clips || []; @@ -182,6 +183,12 @@ function paintLoop() { gl.drawArrays(gl.TRIANGLES, 0, 3); paint.style.filter = busy ? "none" : gradeFilter; } + // Animate keyframed annotation tracks: re-place boxes against playback time. + const c = activeClip(); + if (lastOverlay && lastOverlay.level > 0 && c && + c.annotations.some((a) => a.track && a.track.length)) { + renderOverlay(lastOverlay.level, lastOverlay.intensity); + } requestAnimationFrame(paintLoop); } @@ -232,6 +239,28 @@ function chip(x, y, label, conf, kind, parent) { } } +// Interpolate an annotation's box at loop-normalized time t (0..1). Static labels +// (no track) return their fixed box. Linear between sorted keyframes; clamps to ends. +function boxAt(a, t) { + if (!a.track || !a.track.length) return a.box; + const ks = a.track; + if (t <= ks[0].t) return ks[0].box; + if (t >= ks[ks.length - 1].t) return ks[ks.length - 1].box; + for (let i = 1; i < ks.length; i++) { + if (t <= ks[i].t) { + const a0 = ks[i - 1], a1 = ks[i]; + const f = (t - a0.t) / (a1.t - a0.t); + return a0.box.map((v, j) => v + (a1.box[j] - v) * f); + } + } + return ks[ks.length - 1].box; +} + +// Loop-normalized playback time of the base video (0..1), for track interpolation. +function loopT() { + return vid && vid.duration ? (vid.currentTime % vid.duration) / vid.duration : 0; +} + function renderOverlay(level, intensity) { const clip = activeClip(); overlay.innerHTML = ""; @@ -242,7 +271,7 @@ function renderOverlay(level, intensity) { for (const a of clip.annotations) { if (a.min_level > level) continue; shown++; - const [x, y, w, h] = a.box.map((n) => n * 100); + const [x, y, w, h] = boxAt(a, loopT()).map((n) => n * 100); const measure = a.key.startsWith("measure."); const kind = measure ? "measure" : "detect"; reticle(x, y, w, h, "hud-reticle " + kind, overlay); @@ -266,6 +295,7 @@ function renderOverlay(level, intensity) { height: b.height + pad * 1.2, rx: 0.4, class: "hud-chip-bg detect" }); overlay.insertBefore(bg, st); } + lastOverlay = { level, intensity }; } // Affect channel: soft, glowing emotion-words surfaced only when BOTH knobs are diff --git a/tests/test_pipeline_ffmpeg_ops.py b/tests/test_pipeline_ffmpeg_ops.py new file mode 100644 index 0000000..da28c4e --- /dev/null +++ b/tests/test_pipeline_ffmpeg_ops.py @@ -0,0 +1,54 @@ +from tools.pipeline.ffmpeg_ops import frame_args, crossfade_loop_args, transcode_args + + +def test_frame_args_trims_scales_pads_and_strips_audio(): + args = frame_args( + "src.mp4", "out.mp4", + start=2.0, duration=8.0, width=1920, height=1080, ff="ffmpeg", + ) + assert args[0] == "ffmpeg" + assert "-ss" in args and args[args.index("-ss") + 1] == "2.0" + assert "-t" in args and args[args.index("-t") + 1] == "8.0" + assert "-an" in args # audio stripped (bases are video-only) + vf = args[args.index("-vf") + 1] + assert "scale=1920:1080:force_original_aspect_ratio=decrease" in vf + assert "pad=1920:1080" in vf + assert "format=yuv420p" in vf + assert args[-1] == "out.mp4" + + +def test_crossfade_loop_builds_head_mid_tail_xfade_concat(): + args = crossfade_loop_args("in.mp4", "loop.mp4", duration=8.0, overlap=1.0) + fc = args[args.index("-filter_complex") + 1] + assert "trim=0:1.0" in fc # head = first O secs + assert "trim=1.0:7.0" in fc # mid = [O, D-O] + assert "trim=7.0:8.0" in fc # tail = last O secs + assert "xfade=transition=fade:duration=1.0:offset=0" in fc + assert "concat=n=2:v=1" in fc + assert "-an" in args + assert args[-1] == "loop.mp4" + + +def test_crossfade_loop_rejects_overlap_too_large(): + import pytest + with pytest.raises(ValueError): + crossfade_loop_args("in.mp4", "loop.mp4", duration=4.0, overlap=2.0) + + +def test_crossfade_loop_rejects_nonpositive_overlap(): + import pytest + with pytest.raises(ValueError): + crossfade_loop_args("in.mp4", "loop.mp4", duration=8.0, overlap=0.0) + + +def test_transcode_args_high_profile_even_dims_faststart(): + args = transcode_args("loop.mp4", "proxy.mp4", width=1920, height=1080, + crf=20, fps=30) + assert "-profile:v" in args and args[args.index("-profile:v") + 1] == "high" + assert "-crf" in args and args[args.index("-crf") + 1] == "20" + vf = args[args.index("-vf") + 1] + assert "scale=1920:1080:force_original_aspect_ratio=decrease" in vf + assert "fps=30" in vf + assert "+faststart" in args + assert "-an" in args + assert args[-1] == "proxy.mp4" diff --git a/tests/test_pipeline_integration.py b/tests/test_pipeline_integration.py new file mode 100644 index 0000000..6cd2b35 --- /dev/null +++ b/tests/test_pipeline_integration.py @@ -0,0 +1,42 @@ +from pathlib import Path + +import pytest + +from tools.pipeline.run import probe_duration, process_clip, resolve_ffmpeg + +_FOREST = Path("simulator/sample_media/forest/base.mp4") + + +def _ffmpeg_available() -> bool: + try: + resolve_ffmpeg() + return True + except Exception: + return False + + +_HAS_FF = _ffmpeg_available() + + +@pytest.mark.skipif(not _FOREST.exists(), + reason="forest POC base absent (run simulator/setup_sample_media.py)") +def test_probe_duration_reads_forest_base(): + if not _FOREST.exists(): + pytest.skip("forest POC base absent") + assert probe_duration(_FOREST) > 0 + + +@pytest.mark.skipif(not _HAS_FF, + reason="neither system ffmpeg nor imageio-ffmpeg available") +@pytest.mark.skipif(not _FOREST.exists(), + reason="forest POC base absent (run simulator/setup_sample_media.py)") +def test_process_clip_emits_proxy_and_master(tmp_path): + out = process_clip(_FOREST, tmp_path, overlap=0.5, start=0.0, duration=4.0) + assert out["proxy"].exists() and out["master"].exists() + # Proxy is exactly 1920×1080 — verified with cv2, no ffprobe dependency. + import cv2 + cap = cv2.VideoCapture(str(out["proxy"])) + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + cap.release() + assert (w, h) == (1920, 1080) diff --git a/tests/test_pipeline_manifest.py b/tests/test_pipeline_manifest.py new file mode 100644 index 0000000..bb19e07 --- /dev/null +++ b/tests/test_pipeline_manifest.py @@ -0,0 +1,47 @@ +from tools.pipeline.manifest import upsert_clip, add_ring_scale, rebuild_ring_edges + + +def _base(): + return {"clips": [{"id": "cosmos", "title": "Cosmos"}], + "ring": {"scales": [{"id": "cosmos", "clip_id": "cosmos"}], + "transitions": []}} + + +def test_upsert_clip_replaces_by_id_else_appends(): + m = _base() + m = upsert_clip(m, {"id": "reef", "title": "Reef"}) + assert [c["id"] for c in m["clips"]] == ["cosmos", "reef"] + m = upsert_clip(m, {"id": "reef", "title": "Coral Reef"}) # replace, not dup + assert [c["id"] for c in m["clips"]] == ["cosmos", "reef"] + assert m["clips"][1]["title"] == "Coral Reef" + + +def test_add_ring_scale_inserts_after_named_scale(): + m = _base() + m = add_ring_scale(m, "orbit", "orbit", after="cosmos") + assert [s["id"] for s in m["ring"]["scales"]] == ["cosmos", "orbit"] + + +def test_rebuild_ring_edges_one_transition_per_edge_with_wrap(): + m = _base() + m = add_ring_scale(m, "orbit", "orbit", after="cosmos") + m = add_ring_scale(m, "abyss", "abyss", after="orbit") + m = rebuild_ring_edges(m) + files = [t["file"] for t in m["ring"]["transitions"]] + assert files == [ + "transitions/cosmos-orbit.mp4", + "transitions/orbit-abyss.mp4", + "transitions/abyss-cosmos.mp4", # wrap edge + ] + + +def test_add_ring_scale_appends_when_after_missing(): + m = _base() + m = add_ring_scale(m, "abyss", "abyss", after="nope") + assert [s["id"] for s in m["ring"]["scales"]] == ["cosmos", "abyss"] + + +def test_rebuild_ring_edges_single_scale_self_wrap(): + m = _base() # one scale: cosmos + m = rebuild_ring_edges(m) + assert [t["file"] for t in m["ring"]["transitions"]] == ["transitions/cosmos-cosmos.mp4"] diff --git a/tests/test_pipeline_provenance.py b/tests/test_pipeline_provenance.py new file mode 100644 index 0000000..13624ce --- /dev/null +++ b/tests/test_pipeline_provenance.py @@ -0,0 +1,20 @@ +from tools.pipeline.provenance import Provenance + + +def test_provenance_to_dict_roundtrip(): + p = Provenance( + scale="abyss", + license="public-domain (US Gov, 17 U.S.C. §105)", + source="NOAA Ocean Exploration", + url="https://oceanexplorer.noaa.gov/", + fetched_at="2026-06-24", + ) + d = p.to_dict() + assert d == { + "scale": "abyss", + "license": "public-domain (US Gov, 17 U.S.C. §105)", + "source": "NOAA Ocean Exploration", + "url": "https://oceanexplorer.noaa.gov/", + "fetched_at": "2026-06-24", + } + assert Provenance.from_dict(d) == p diff --git a/tools/pipeline/__init__.py b/tools/pipeline/__init__.py new file mode 100644 index 0000000..7f38557 --- /dev/null +++ b/tools/pipeline/__init__.py @@ -0,0 +1,5 @@ +"""Content pipeline: source -> frame -> loop -> transcode -> manifest. + +Pure ffmpeg-argument builders (ffmpeg_ops), a thin runner (run), provenance +records, and manifest assembly. See docs/superpowers/specs/2026-06-24-content-pipeline-design.md. +""" diff --git a/tools/pipeline/ffmpeg_ops.py b/tools/pipeline/ffmpeg_ops.py new file mode 100644 index 0000000..6a7f232 --- /dev/null +++ b/tools/pipeline/ffmpeg_ops.py @@ -0,0 +1,74 @@ +"""Pure ffmpeg argument builders (no I/O) for the content pipeline. + +Each function returns a list[str] ready for ffmpeg, so command construction is +unit-testable without running ffmpeg. tools/pipeline/run.py executes them. The +ffmpeg binary name is injected (default "ffmpeg"). +""" + +from __future__ import annotations + + +def _fit(width: int, height: int) -> str: + """A scale+pad filter chain fitting any input into width×height (letterbox), + fixing SAR and pixel format so downstream concat/xfade get identical geometry.""" + return ( + f"scale={width}:{height}:force_original_aspect_ratio=decrease," + f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2," + "setsar=1,format=yuv420p" + ) + + +def frame_args(src, dst, *, start: float, duration: float, + width: int, height: int, ff: str = "ffmpeg") -> list[str]: + """Stage 2 — trim [start, start+duration], fit to width×height, strip audio. + Produces a high-quality mezzanine for the loop stage.""" + return [ + ff, "-y", "-ss", str(start), "-t", str(duration), + "-i", str(src), "-vf", _fit(width, height), "-an", + "-c:v", "libx264", "-preset", "medium", "-crf", "16", + "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(dst), + ] + + +def crossfade_loop_args(src, dst, *, duration: float, overlap: float, + fps: int = 30, ff: str = "ffmpeg") -> list[str]: + """Stage 3 — make `src` loop seamlessly by crossfading its tail over its head. + Output length = duration − overlap. Requires overlap < duration/2 so a non-empty + middle remains. + + fps is applied after each trim so xfade receives a constant-frame-rate stream + (xfade requires CFR; a raw trim output reports rate 1/0 which xfade rejects).""" + if overlap <= 0 or overlap >= duration / 2: + raise ValueError(f"overlap {overlap} must be in (0, duration/2={duration/2})") + d, o = duration, overlap + fc = ( + f"[0:v]trim=0:{o},setpts=PTS-STARTPTS,fps={fps}[head];" + f"[0:v]trim={o}:{d - o},setpts=PTS-STARTPTS,fps={fps}[mid];" + f"[0:v]trim={d - o}:{d},setpts=PTS-STARTPTS,fps={fps}[tail];" + f"[tail][head]xfade=transition=fade:duration={o}:offset=0[xf];" + f"[xf][mid]concat=n=2:v=1,format=yuv420p[out]" + ) + return [ + ff, "-y", "-i", str(src), "-filter_complex", fc, + "-map", "[out]", "-an", + "-c:v", "libx264", "-preset", "medium", "-crf", "16", + "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(dst), + ] + + +def transcode_args(src, dst, *, width: int, height: int, crf: int, + fps: int = 30, ff: str = "ffmpeg") -> list[str]: + """Stage 4 — encode a final deliverable (master or proxy). Master: source res + capped ~3840 wide, crf 18. Proxy: 1920×1080, crf 20. Both H.264 High, yuv420p, + even dims, +faststart, video-only (spec §6).""" + vf = ( + f"scale={width}:{height}:force_original_aspect_ratio=decrease," + f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2," + f"setsar=1,fps={fps},format=yuv420p" + ) + return [ + ff, "-y", "-i", str(src), "-vf", vf, "-an", + "-c:v", "libx264", "-profile:v", "high", "-preset", "slow", + "-crf", str(crf), "-pix_fmt", "yuv420p", + "-movflags", "+faststart", str(dst), + ] diff --git a/tools/pipeline/manifest.py b/tools/pipeline/manifest.py new file mode 100644 index 0000000..c9a36bd --- /dev/null +++ b/tools/pipeline/manifest.py @@ -0,0 +1,55 @@ +"""Stage 6 — assemble manifest entries (spec §3.6). Pure dict transforms over a +loaded manifest dict; the caller does the json read/write. Idempotent by id so +re-running earlier stages never clobbers authored labels.""" + +from __future__ import annotations + +import copy + + +def upsert_clip(manifest: dict, clip: dict) -> dict: + """Replace the clip with the same id, or append it. Returns a new manifest.""" + m = copy.deepcopy(manifest) + clips = m.setdefault("clips", []) + for i, c in enumerate(clips): + if c["id"] == clip["id"]: + clips[i] = clip + return m + clips.append(clip) + return m + + +def add_ring_scale(manifest: dict, scale_id: str, clip_id: str, + after: str | None = None) -> dict: + """Insert a scale into ring.scales after the named scale (or append if `after` + is None / not found). No-op if scale_id already present. Returns a new manifest.""" + m = copy.deepcopy(manifest) + ring = m.setdefault("ring", {"scales": [], "transitions": []}) + scales = ring.setdefault("scales", []) + if any(s["id"] == scale_id for s in scales): + return m + entry = {"id": scale_id, "clip_id": clip_id} + idx = next((i for i, s in enumerate(scales) if s["id"] == after), None) + if idx is None: + scales.append(entry) + else: + scales.insert(idx + 1, entry) + return m + + +def rebuild_ring_edges(manifest: dict) -> dict: + """Rebuild ring.transitions to one placeholder entry per ring edge (N scales -> + N edges incl. the wrap), preserving any existing model string for an edge. + Returns a new manifest.""" + m = copy.deepcopy(manifest) + ring = m.setdefault("ring", {"scales": [], "transitions": []}) + scales = ring.get("scales", []) + existing = {t["file"]: t.get("model", "") for t in ring.get("transitions", [])} + edges = [] + n = len(scales) + for i in range(n): + a, b = scales[i]["id"], scales[(i + 1) % n]["id"] + file = f"transitions/{a}-{b}.mp4" + edges.append({"file": file, "model": existing.get(file, "placeholder-zoom")}) + ring["transitions"] = edges + return m diff --git a/tools/pipeline/provenance.py b/tools/pipeline/provenance.py new file mode 100644 index 0000000..95c00ef --- /dev/null +++ b/tools/pipeline/provenance.py @@ -0,0 +1,29 @@ +"""Stage 1 — the per-clip strict-PD provenance record (spec §3.1). + +Captured at source time so license + source + URL travel with the media. The +'no explicit license -> assume PD, verify' trap (scales design §2.1) applies: +"free stock" (Pexels/Pixabay) is NOT public domain. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class Provenance: + scale: str + license: str + source: str + url: str + fetched_at: str # ISO date; passed in (no clock here -> deterministic) + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> "Provenance": + return cls( + scale=d["scale"], license=d["license"], source=d["source"], + url=d["url"], fetched_at=d["fetched_at"], + ) diff --git a/tools/pipeline/run.py b/tools/pipeline/run.py new file mode 100644 index 0000000..d9d4464 --- /dev/null +++ b/tools/pipeline/run.py @@ -0,0 +1,67 @@ +"""Thin runner: probe the source duration, then execute the frame -> loop -> +transcode stages with tools.pipeline.ffmpeg_ops. The pure arg builders are +unit-tested; this glue is covered by the opt-in integration test.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +from .ffmpeg_ops import crossfade_loop_args, frame_args, transcode_args + +MASTER_MAX_W = 3840 + + +def resolve_ffmpeg() -> str: + """ffmpeg from PATH, else the bundled imageio-ffmpeg binary (matches + simulator/setup_scales_media.py). The Pi/dev box may lack a system ffmpeg.""" + if shutil.which("ffmpeg"): + return "ffmpeg" + import imageio_ffmpeg + return imageio_ffmpeg.get_ffmpeg_exe() + + +def probe_duration(src) -> float: + """Clip duration in seconds via cv2 (frame_count / fps) — avoids a system + ffprobe dependency (imageio-ffmpeg ships ffmpeg only; the Pi has no ffprobe).""" + import cv2 + cap = cv2.VideoCapture(str(src)) + fps = cap.get(cv2.CAP_PROP_FPS) or 0.0 + frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0.0 + cap.release() + if fps <= 0: + raise ValueError(f"could not read fps from {src}") + return frames / fps + + +def _run(args: list[str]) -> None: + subprocess.run(args, check=True, capture_output=True) + + +def process_clip(src, out_dir, *, start: float = 0.0, duration: float | None = None, + overlap: float = 1.0, master_w: int = MASTER_MAX_W, master_h: int = 2160, + ff: str | None = None) -> dict: + """Run stages 2–4 for one clip; return {'master','proxy','loop','mezzanine'} paths. + `duration` defaults to the full source length (minus the trim start). + Note: `crossfade_loop_args` uses a fixed fps only to satisfy xfade's + constant-frame-rate requirement; the final output fps is set by `transcode_args` + (both default 30 — keep in sync if overridden).""" + ff = ff or resolve_ffmpeg() + src = Path(src) + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + if duration is None: + duration = probe_duration(src) - start + + mezz = out_dir / "mezzanine.mp4" + loop = out_dir / "loop.mp4" + master = out_dir / "master.mp4" + proxy = out_dir / "base.mp4" # the proxy is what the manifest's base_file points at + + _run(frame_args(src, mezz, start=start, duration=duration, + width=master_w, height=master_h, ff=ff)) + _run(crossfade_loop_args(mezz, loop, duration=duration, overlap=overlap, ff=ff)) + _run(transcode_args(loop, master, width=master_w, height=master_h, crf=18, ff=ff)) + _run(transcode_args(loop, proxy, width=1920, height=1080, crf=20, ff=ff)) + return {"mezzanine": mezz, "loop": loop, "master": master, "proxy": proxy}