fix(audio): off/on Audio + Video toggles; fix Safari autoplay + GPU video-off
Operator-driven live-UI revision:
- Video + Audio are now Dev-Mode-style on/off toggles (Audio on = soundtrack);
white-noise deferred (removed from the live control; pure machinery kept).
- FIX video-off not blanking: the <video>/<canvas> are GPU-composited and could
paint above the auto-z #black on a real GPU — give #black z-index + its own
layer AND hide the video layers (opacity 0) on video-off.
- FIX no soundtrack on Safari/iOS: Safari unlocks <audio> only when play() runs
INSIDE the user gesture; the start gesture now primes both A/B elements
synchronously, so the async crossfades are allowed to play.
- Toggles wired on 'change' (matches the Dev Mode switch).
- player/audio.py AUDIO_SOURCES -> {off, soundtrack}; tests + E2E updated.
- Verified in headless Chromium AND WebKit: audio plays, video-off blanks both
layers, zero JS errors. 288 tests + 3 Playwright E2E pass.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
BenStullsBets
+1
-4
@@ -39,9 +39,6 @@ STATIC_DIR = Path(__file__).parent / "static"
|
||||
MEDIA_DIR = Path(__file__).parent / "sample_media"
|
||||
DEFAULT_MANIFEST = MEDIA_DIR / "manifest.json"
|
||||
|
||||
# The global white-noise bed (audio spec §5.2): synthesized, altitude-independent.
|
||||
NOISE_URL = "/media/audio/noise/pink.mp3"
|
||||
|
||||
# Per-process boot token: changes on every server restart so the dev live-reload
|
||||
# (below) also fires when Python code changes (which needs a restart), not just
|
||||
# when a static asset's mtime changes.
|
||||
@@ -190,7 +187,7 @@ def create_app(manifest_path: Optional[Path] = None) -> FastAPI:
|
||||
scale_audio = ""
|
||||
if app.state.ring is not None:
|
||||
scale_audio = scale_at(app.state.ring, req.altitude_index).audio
|
||||
audio = resolve_audio(c.audio, scale_audio=scale_audio, noise_url=NOISE_URL)
|
||||
audio = resolve_audio(c.audio, scale_audio=scale_audio)
|
||||
return {
|
||||
"plan": render_plan_to_dict(plan),
|
||||
"render": {
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
"""Production pass for the 5 per-altitude soundtracks + the white-noise bed
|
||||
(audio spec §5.3). The audio analogue of build_pool_manifest.py --media: read the
|
||||
sourced ambience clips (re-downloadable per docs/audio-candidate-pool.md), make
|
||||
each a seamless loudness-normalized loop, and synthesize the pink-noise bed.
|
||||
"""Production pass for the 5 per-altitude soundtracks (audio spec §5.3). The audio
|
||||
analogue of build_pool_manifest.py --media: read the sourced ambience clips
|
||||
(re-downloadable per docs/audio-candidate-pool.md), and make each a seamless
|
||||
loudness-normalized loop.
|
||||
|
||||
Media is gitignored; this script is the reproducible record. Outputs land under
|
||||
simulator/sample_media/audio/<scale>/<name>.loop.mp3 and audio/noise/pink.mp3,
|
||||
served at /media/audio/... Run: python simulator/build_audio_media.py
|
||||
simulator/sample_media/audio/<scale>/<name>.loop.mp3, served at /media/audio/...
|
||||
Run: python simulator/build_audio_media.py
|
||||
|
||||
(White-noise is deferred — the live Audio control is a simple on/off soundtrack
|
||||
toggle. `tools.pipeline.audio_run.generate_white_noise` remains for when it's
|
||||
wanted, but no noise bed is built here.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from tools.pipeline.audio_run import generate_white_noise, process_soundtrack
|
||||
from tools.pipeline.audio_run import process_soundtrack
|
||||
|
||||
AUDIO = Path(__file__).parent / "sample_media" / "audio"
|
||||
|
||||
@@ -35,8 +39,6 @@ def main() -> None:
|
||||
continue
|
||||
dst = process_soundtrack(src, AUDIO / scale / out)
|
||||
print(f" produced {dst.relative_to(AUDIO.parent)}")
|
||||
noise = generate_white_noise(AUDIO / "noise" / "pink.mp3")
|
||||
print(f" produced {noise.relative_to(AUDIO.parent)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+39
-12
@@ -495,14 +495,14 @@ function renderScaleReadout() {
|
||||
}
|
||||
|
||||
function controls() {
|
||||
// Visual (on/off) and Audio (off/soundtrack/white_noise) are orthogonal dials
|
||||
// (audio spec §2). One bipolar Mood dial (-4 dark .. +4 light) maps onto the
|
||||
// engine's two poles. Calibration gains are pinned to 1.0, so the simulator
|
||||
// sends no calibration — the server uses DEFAULT_CALIBRATION.
|
||||
// Video (on/off) and Audio (on/off) are orthogonal toggles (audio spec §2).
|
||||
// Audio on = the per-altitude soundtrack (white-noise is deferred). One bipolar
|
||||
// Mood dial (-4 dark .. +4 light) maps onto the engine's two poles. Calibration
|
||||
// gains are pinned to 1.0, so the simulator sends no calibration.
|
||||
const mood = +$("mood").value;
|
||||
return {
|
||||
visual: $("visual").checked ? "on" : "off",
|
||||
audio: $("audio").value,
|
||||
audio: $("audio").checked ? "soundtrack" : "off",
|
||||
left: +$("left").value, right: +$("right").value,
|
||||
dark: mood < 0 ? -mood : 0, light: mood > 0 ? mood : 0,
|
||||
volume: 2, brightness: 2,
|
||||
@@ -520,8 +520,17 @@ async function update() {
|
||||
const data = await resp.json();
|
||||
readout.textContent = JSON.stringify(data, null, 2);
|
||||
applyAudio(data.render.audio); // reconcile the audio layer
|
||||
if (!data.render.video.shown) { black.style.opacity = "1"; black.classList.remove("hidden"); return; }
|
||||
if (!data.render.video.shown) {
|
||||
// Blank the screen. Cover with #black AND hide the video layers themselves —
|
||||
// the <video>/<canvas> are GPU-composited and can otherwise show through the
|
||||
// overlay on a real GPU (Safari/Chrome).
|
||||
black.style.opacity = "1"; black.classList.remove("hidden");
|
||||
vid.style.opacity = "0"; paint.style.opacity = "0";
|
||||
return;
|
||||
}
|
||||
black.classList.add("hidden");
|
||||
vid.style.opacity = ""; paint.style.opacity = ""; // restore the video layers
|
||||
|
||||
try {
|
||||
ensureClipMedia();
|
||||
applyVideoLook(data.plan.grade.tone, data.plan.dream.intensity);
|
||||
@@ -944,6 +953,13 @@ async function applyAudio(audio) {
|
||||
[audActive, audIdle] = [audIdle, audActive]; // swap roles
|
||||
}
|
||||
|
||||
// The current altitude's soundtrack url (the ring carries each scale's `audio`),
|
||||
// resolved client-side so the start gesture can unlock the elements synchronously.
|
||||
function soundtrackUrl() {
|
||||
const s = ring && ring.scales[ringIndex];
|
||||
return s && s.audio ? "/media/audio/" + s.audio : null;
|
||||
}
|
||||
|
||||
// Browsers block autoplay until a user gesture. The renderer is launched
|
||||
// full-screen by the operator; a one-time tap unlocks audio, then it follows
|
||||
// state (audio spec §8 — the documented launch step).
|
||||
@@ -953,11 +969,21 @@ function showStartGesture() {
|
||||
ov.id = "start-gesture";
|
||||
ov.textContent = "▶ tap to start sound";
|
||||
ov.addEventListener("click", () => {
|
||||
// A click anywhere on the page grants Chrome's autoplay permission
|
||||
// document-wide for the session, so later programmatic play() is allowed —
|
||||
// no per-element priming needed (priming srcless elements only pollutes state).
|
||||
audioReady = true;
|
||||
ov.remove();
|
||||
// Safari unlocks an <audio> element ONLY when play() is called inside the
|
||||
// user gesture itself — not in a later async continuation (Chrome is lenient;
|
||||
// Safari/iOS is not). Prime BOTH A/B elements now, synchronously, at volume 0,
|
||||
// so the async applyAudio() crossfades that follow are allowed to play.
|
||||
const url = soundtrackUrl();
|
||||
if (url) for (const el of [audA, audB]) {
|
||||
try {
|
||||
el.src = mediaUrl(url);
|
||||
el.volume = 0;
|
||||
const pr = el.play();
|
||||
if (pr) pr.then(() => el.pause()).catch(() => {});
|
||||
} catch (_) { /* priming is best-effort */ }
|
||||
}
|
||||
audUrl = null; // force the next applyAudio to (re)load
|
||||
update(); // re-apply current state, now with audio
|
||||
}, { once: true });
|
||||
@@ -973,9 +999,10 @@ async function main() {
|
||||
buildDial(); // draw the altitude knob from the ring's scales
|
||||
initDev(); // wire the Dev Mode toggle + pool picker (reads persisted state)
|
||||
renderScaleReadout();
|
||||
for (const id of ["visual", "audio", "left", "right", "mood"]) {
|
||||
$(id).addEventListener("input", debounced);
|
||||
}
|
||||
// Sliders stream on "input"; the Video/Audio toggles fire on "change" (matches
|
||||
// the Dev Mode switch and is the reliable checkbox event across browsers).
|
||||
for (const id of ["left", "right", "mood"]) $(id).addEventListener("input", debounced);
|
||||
for (const id of ["visual", "audio"]) $(id).addEventListener("change", debounced);
|
||||
showStartGesture(); // one-time tap unlocks audio (browser autoplay policy, §8)
|
||||
// Altitude knob: drag to turn (commit detents on release), scroll to step, tap a label to jump.
|
||||
dial.addEventListener("pointerdown", onDialDown);
|
||||
|
||||
+11
-11
@@ -24,17 +24,17 @@
|
||||
|
||||
<section class="panel">
|
||||
<fieldset>
|
||||
<legend>Visual</legend>
|
||||
<label class="toggle"><input type="checkbox" id="visual" checked /> show video</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Audio</legend>
|
||||
<select id="audio">
|
||||
<option value="off">off (silence)</option>
|
||||
<option value="soundtrack">soundtrack (follows altitude)</option>
|
||||
<option value="white_noise">white noise</option>
|
||||
</select>
|
||||
<legend>Output</legend>
|
||||
<label class="dev-switch" for="visual">
|
||||
<input type="checkbox" id="visual" checked />
|
||||
<span class="dev-switch-track"><span class="dev-switch-thumb"></span></span>
|
||||
<span class="dev-switch-label">Video</span>
|
||||
</label>
|
||||
<label class="dev-switch" for="audio">
|
||||
<input type="checkbox" id="audio" />
|
||||
<span class="dev-switch-track"><span class="dev-switch-thumb"></span></span>
|
||||
<span class="dev-switch-label">Audio</span>
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
|
||||
@@ -40,8 +40,15 @@ main { display: flex; gap: 1rem; padding: 1rem; flex-wrap: wrap; align-items: fl
|
||||
.hud-chip-text.measure { fill: #ffd79a; }
|
||||
.hud-conf { fill: #6cf; opacity: 0.8; }
|
||||
.hud-status { fill: #8fdcff; font: 2.4px monospace; opacity: 0.9; letter-spacing: 0.15px; }
|
||||
.black { position: absolute; inset: 0; background: #000; opacity: 1; transition: opacity 200ms ease; }
|
||||
/* z-index + own compositing layer: the <video> and WebGL <canvas> are GPU-
|
||||
composited layers that can paint ABOVE a plain auto-z sibling (Safari/Chrome
|
||||
with a real GPU) despite DOM order — so the black cover needs an explicit
|
||||
stacking order above them to actually blank the screen. */
|
||||
.black { position: absolute; inset: 0; background: #000; opacity: 1;
|
||||
z-index: 50; transform: translateZ(0); transition: opacity 200ms ease; }
|
||||
.hidden { display: none; }
|
||||
/* Stack the two Output toggles (Video / Audio) with a little breathing room. */
|
||||
.dev-switch + .dev-switch { margin-top: 0.45rem; }
|
||||
.panel { flex: 0 0 280px; display: flex; flex-direction: column; gap: 0.8rem;
|
||||
max-height: calc(100vh - 2rem); overflow-y: auto;
|
||||
position: sticky; top: 1rem; }
|
||||
|
||||
Reference in New Issue
Block a user