benstull/human-experience-filter-art
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(sim): double-buffer video so morph->loop handoff doesn't stall

Browse Source 
Operator eyeball: a jolt/pause when the morph ends and the next altitude's loop
loads. Root cause: swapping vid.src to the base + seeking to the tail reloaded the
ON-SCREEN element (decode + seek stall). Now a dedicated #vid-loop element carries
the steady-state base loop; during a scrub it is PRELOADED to the clip we're heading
toward, seeked to the tail and paused on that exact frame. The paint shader reads the
active source (busy ? morph : loop), so landing is an instant source swap — no reload,
no seek. E2E asserts the loop element is armed + playing from ~3s after landing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
BenStullsBets
2026-06-27 16:58:08 -07:00
parent 0752746f7b
commit ea530b3df6
4 changed files with 78 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
simulator/static/app.js
+65 -31
View File
@@ -7,8 +7,22 @@
const $ = (id) => document.getElementById(id);
const vid = $("vid"), paint = $("paint"), tint = $("tint"), overlay = $("overlay"),
black = $("black"), readout = $("readout");
const loopVid = $("vid-loop"); // steady-state base loop element (double-buffer; #vid carries the scrub morphs)
const affectLayer = $("affect");
// The currently-displayed source element: the morph (#vid) while scrubbing (busy),
// else the preloaded base loop (#vid-loop). The paint shader + fallback read this so
// a landing is a source swap, never a reload of the on-screen element.
function displayVid() { return busy ? vid : loopVid; }
// Show the active source, hide the other (matters only in the WebGL-less fallback;
// with the paint canvas it composites the active source anyway).
function showActiveSource() {
const a = displayVid();
a.style.opacity = "1";
(a === vid ? loopVid : vid).style.opacity = "0";
}
// Right-dream state, read by the painterly render loop each frame. update() (debounced
// on knob moves) writes these; the WebGL loop renders the live video continuously.
let dreamIntensity = 0; // 0..1 — painterly strength (and dream pastel/luminous)
@@ -222,31 +236,39 @@ function preloadChip() {
// to the loop's first frame, and the morph-covered intro never re-shows at rest.
const LOOP_TAIL_S = 3.0;
function seekToLoopStart() {
try { vid.currentTime = LOOP_TAIL_S; } catch (e) { /* not seekable yet */ }
// Load a clip's base into the LOOP element (#vid-loop), seeked to the tail and PAUSED
// on that exact frame. Idempotent per clip. Called during a scrub to PRELOAD the clip
// we're about to land on, so settle just plays an already-decoded element at the morph's
// last frame — no reload, no seek, no stall. `playLoop()` starts it at landing.
function loadLoop(clip) {
if (!clip || loopVid.dataset.clip === clip.id) return;
loopVid.dataset.clip = clip.id;
loopVid.dataset.loopTail = "1"; // arm the [LOOP_TAIL_S, duration] wrap handler
loopVid.src = mediaUrl(clip.base_file);
loopVid.loop = false; // native loop restarts at 0; we loop from the tail instead
loopVid.muted = true;
const seek = () => { try { loopVid.currentTime = LOOP_TAIL_S; } catch (e) { /* not seekable yet */ } };
if (loopVid.readyState >= 1) seek();
else loopVid.addEventListener("loadedmetadata", seek, { once: true });
}
function playLoop() {
if (loopVid.paused) loopVid.play().catch(() => {});
}
function ensureClipMedia() {
const clip = activeClip();
if (!clip || clip.id === currentClipId) return;
if (!clip) return;
currentClipId = clip.id;
delete vid.dataset.morph; // this is a base loop, not a morph
vid.dataset.loopTail = "1"; // arm the [LOOP_TAIL_S, duration] loop handler
vid.src = mediaUrl(clip.base_file);
vid.loop = false; // native loop restarts at 0; we loop from the tail instead
vid.muted = true;
if (vid.readyState >= 1) seekToLoopStart();
else vid.addEventListener("loadedmetadata", seekToLoopStart, { once: true });
vid.play().catch(() => {});
vid.style.opacity = "1";
loadLoop(clip); // idempotent: a no-op if the scrub already preloaded this clip
playLoop(); // steady state runs the loop
}
// The custom loop: when a base loop reaches the end, jump back to the tail offset
// (not 0). Inert during a morph (loopTail cleared in rebuildSegment), where the
// scrub drives currentTime directly.
vid.addEventListener("timeupdate", () => {
if (vid.dataset.loopTail === "1" && vid.duration && vid.currentTime >= vid.duration - 0.08) {
seekToLoopStart();
// The custom loop on the LOOP element: when a base loop reaches the end, jump back to
// the tail offset (not 0). The morph element (#vid) is driven directly by the scrub.
loopVid.addEventListener("timeupdate", () => {
if (loopVid.dataset.loopTail === "1" && loopVid.duration && loopVid.currentTime >= loopVid.duration - 0.08) {
try { loopVid.currentTime = LOOP_TAIL_S; } catch (e) { /* not seekable */ }
}
});
@@ -263,8 +285,8 @@ function applyVideoLook(tone, dream) {
gradeFilter = `brightness(${bright.toFixed(3)}) saturate(${sat.toFixed(3)}) sepia(${sepia.toFixed(3)})`;
dreamIntensity = dream;
tint.style.opacity = (cool * 0.6).toFixed(3);
// WebGL-less fallback: the canvas is hidden, so grade the visible #vid directly.
if (!paintOK) vid.style.filter = gradeFilter;
// WebGL-less fallback: the canvas is hidden, so grade the visible source directly.
if (!paintOK) displayVid().style.filter = gradeFilter;
}
// --- Right dream: real-time painterly restyle (WebGL2 Kuwahara) ---
@@ -352,11 +374,12 @@ function initPaint() {
return true;
}
function paintLoop() {
if (paintOK && vid.readyState >= 2 && vid.videoWidth) {
const w = vid.videoWidth, h = vid.videoHeight;
const src = displayVid(); // morph while scrubbing, else the preloaded base loop
if (paintOK && src.readyState >= 2 && src.videoWidth) {
const w = src.videoWidth, h = src.videoHeight;
if (paint.width !== w || paint.height !== h) { paint.width = w; paint.height = h; }
gl.viewport(0, 0, w, h);
try { gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, vid); } catch (_) {}
try { gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, src); } catch (_) {}
// No painterly restyle during a ring transition (busy) — show it raw.
gl.uniform1f(uAmt, busy ? 0 : dreamIntensity);
gl.uniform2f(uTexel, 1 / w, 1 / h);
@@ -440,7 +463,8 @@ function boxAt(a, t) {
// Loop-normalized playback time of the base video (0..1), for track interpolation.
function loopT() {
return vid && vid.duration ? (vid.currentTime % vid.duration) / vid.duration : 0;
const v = displayVid();
return v && v.duration ? (v.currentTime % v.duration) / v.duration : 0;
}
// --- Progressive labels & emotions (content-pipeline §11.3 / §11.4) ---
@@ -591,11 +615,12 @@ async function update() {
// the <video>/<canvas> are GPU-composited and can otherwise show through the
// overlay on a real GPU (Safari/Chrome).
black.style.opacity = "1"; black.classList.remove("hidden");
vid.style.opacity = "0"; paint.style.opacity = "0";
vid.style.opacity = "0"; loopVid.style.opacity = "0"; paint.style.opacity = "0";
return;
}
black.classList.add("hidden");
vid.style.opacity = ""; paint.style.opacity = ""; // restore the video layers
paint.style.opacity = ""; // restore the painterly canvas
showActiveSource(); // show the active source (WebGL-less fallback)
try {
ensureClipMedia();
@@ -660,6 +685,7 @@ let dialDrag = null; // {lastAng, accum, moved, restPos, target} while
let pos = 0; // continuous knob position; rest value == ringIndex
let activeSeg = null; // { lo, clipLo, clipHi, file } for the segment being scrubbed
let seekPending = false; // throttle: at most one currentTime seek per animation frame
let scrubDir = 1; // last travel direction (+1 descend / -1 ascend) — which end we'll land on
// Diagnostic seam (sibling of window.__hefMorphs): the committed altitude state, so
// E2E can assert the locked clip mid-scrub when #scale-name (settle-only) is stale.
@@ -755,8 +781,6 @@ function rebuildSegment(lo, enteredFrom) {
tint.style.opacity = "0";
vid.style.filter = "none";
vid.loop = false;
vid.dataset.loopTail = "0"; // morph scrub drives currentTime; disarm the base loop handler
vid.style.opacity = "1";
if (vid.dataset.morph !== file) {
vid.dataset.morph = file;
vid.src = mediaUrl(file);
@@ -774,6 +798,8 @@ function rebuildSegment(lo, enteredFrom) {
function setPos(next) {
if (!ring || ring.scales.length < 2) return;
const n = ring.scales.length;
const dir = next > pos ? 1 : (next < pos ? -1 : scrubDir); // travel direction (for which end we'll land on)
scrubDir = dir;
for (const c of HEFScrub.integerCrossings(pos, next)) {
ringIndex = HEFScrub.wrapIndex(c.index, n);
if (activeSeg) {
@@ -787,15 +813,22 @@ function setPos(next) {
busy = false;
delete vid.dataset.morph; // clear so re-entering the same segment reloads the morph (not the base)
currentClipId = null; // force ensureClipMedia to reload + loop the locked clip
playLoop(); // the loop element was preloaded to this clip @ tail during the scrub → instant, no reload
showActiveSource();
setNeedle(ringIndex * dialStep());
renderScaleReadout();
update(); // ensureClipMedia loads + loops the locked clip
update(); // ensureClipMedia: idempotent loadLoop + keep the loop running
restAudio(ringIndex);
refreshReachablePreload(); // warm the morphs now reachable from the locked clip
return;
}
busy = true; // mid-morph: block update() from reloading the base clip under us
if (!activeSeg || activeSeg.lo !== lo) rebuildSegment(lo, ringIndex);
// Preload the clip we're heading toward into the LOOP element (paused @ tail) so the
// landing is a swap, not a reload+seek. dir>0 lands on clipHi, dir<0 on clipLo.
const headingId = dir > 0 ? activeSeg.clipHi : activeSeg.clipLo;
loadLoop(clipsById[headingId]);
showActiveSource(); // morph element is the live source while scrubbing
setNeedle(pos * dialStep());
blendAudio(lo, frac);
if (activeSeg.file && !seekPending) { // throttle seeks to one per frame (avoid seek thrash)
@@ -942,8 +975,9 @@ function renderDevStats() {
const total = preloadList().length;
const cached = Object.keys(mediaBlobs).length;
const fromMem = !!(c && mediaBlobs[c.base_file]);
const res = vid.videoWidth ? `${vid.videoWidth}×${vid.videoHeight}` : "—";
const dur = vid.duration ? `${vid.duration.toFixed(1)}s` : "—";
const dv = displayVid();
const res = dv.videoWidth ? `${dv.videoWidth}×${dv.videoHeight}` : "—";
const dur = dv.duration ? `${dv.duration.toFixed(1)}s` : "—";
const loop = vid.duration ? `${Math.round(loopT() * 100)}%` : "—";
dl.innerHTML =
`<dt>cache</dt><dd>${cached}/${total} cached</dd>` +
simulator/static/index.html
+1
View File
@@ -18,6 +18,7 @@
<section class="stage" id="stage">
<div class="screen">
<video id="vid" loop muted playsinline></video>
<video id="vid-loop" loop muted playsinline></video>
<audio id="aud" loop preload="auto"></audio>
<audio id="aud-b" loop preload="auto"></audio>
<canvas id="paint"></canvas>
simulator/static/style.css
+5 -1
View File
@@ -7,7 +7,11 @@ main { display: flex; gap: 1rem; padding: 1rem; flex-wrap: wrap; align-items: fl
.stage { flex: 1 1 640px; position: sticky; top: 1rem; }
.screen { position: relative; width: 100%; aspect-ratio: 16 / 9; background: #000;
border-radius: 6px; overflow: hidden; }
#vid { width: 100%; height: 100%; object-fit: cover; transition: opacity 0.15s ease; }
/* Two stacked source videos: #vid plays the scrub morphs, #vid-loop the steady-state
base loop, preloaded during a scrub so landing is a swap (no reload stall). The
paint canvas (WebGL) composites whichever is active; in the WebGL-less fallback the
active one is shown via opacity. */
#vid, #vid-loop { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: cover; transition: opacity 0.15s ease; }
/* Right-dream painterly canvas: a WebGL Kuwahara restyle of the LIVE video frames,
drawn over the base. Edge-preserving, so motion stays as crisp as the source —
the dream is the same footage, just stylized. The mood grade rides as a CSS