From c9fd266bf92a1fd07f0566a5344a812595308eda Mon Sep 17 00:00:00 2001 From: snakajima Date: Tue, 27 Oct 2026 17:06:47 -1000 Subject: [PATCH 1/2] The pending card: status at the top, NEUTRINO's progress, a card that scrolls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #49: the rendering note (or the failure) goes right under the title, in both views. #50: NEUTRINO runs with -t and writes its progress to a file in the work folder. The worker is blocked while it runs, so it sends the file's path and the render pool reads it every 500 ms. render() takes onProgress (singing 0…1, then mixing); it is only reported, never touches the audio. manage's deferred render keeps the latest (deferredRenderProgress), and the plugin's view asks for it once a second with a renderProgress request beside collectRenderedAudio. #51: MulmoChat gives the view an overflow: hidden box, and the card did not scroll itself, so long sheet music could not be scrolled before playback. The card is now height 100%, overflow-y auto. Co-Authored-By: Claude Opus 5.5 --- PLAN.md | 18 +++++++++ gui-plugin/README.md | 5 ++- gui-plugin/demo/main.ts | 29 ++++++++++----- gui-plugin/src/audio-request.ts | 33 +++++++++++++++- gui-plugin/src/index.ts | 16 +++++--- gui-plugin/src/vue/View.vue | 56 ++++++++++++++++++++++++---- gui-plugin/src/vue/playback.ts | 12 +++++- gui-plugin/tests/playback.test.ts | 9 +++++ gui-plugin/tests/plugin.test.ts | 22 ++++++++++- src/core.ts | 2 +- src/deferred-renders.ts | 14 ++++++- src/index.ts | 2 +- src/manage.ts | 19 +++++++--- src/render-pool.ts | 43 ++++++++++++++++++--- src/render-worker.ts | 22 ++++++++--- src/render.ts | 17 ++++++++- src/singer/neutrino.ts | 37 +++++++++++++++--- tests/browser.test.ts | 1 + tests/deferred-renders.test.ts | 21 ++++++++++- tests/render-pool.test.ts | 62 ++++++++++++++++++++++++++++++- tests/singer-neutrino.test.ts | 29 ++++++++++++++- 21 files changed, 410 insertions(+), 59 deletions(-) diff --git a/PLAN.md b/PLAN.md index f46749c..b4bc0ce 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1544,3 +1544,21 @@ Decided: getSchema() and the published schema files are unchanged. Not measured with the LLM eval. checkScore of a score that sings now also checks NEUTRINO and its voice (an error on the sung track), so a model can find out whether it can sing before rendering instead of guessing. +- 2026-10-27 — The pending card (#49, #50, #51). User: the rendering note should be at the top in + both views; NEUTRINO takes the longest, so show how far it is; the sheet music cannot be + scrolled by hand before playback. NEUTRINO prints `progress = N %` with `-t`; checked on the + real one that it arrives during the run through a pipe (8 s song: 8 % at 1.1 s … 100 % at + 13.7 s). It now runs with `-t` and its output in a file in the work folder; the worker thread + is blocked while it runs, so it sends the file's path and the pool reads it every 500 ms. + `render()` takes `onProgress` (singing 0…1, then mixing; in place only the two stages); it is + only reported and never touches the audio. `-t` itself: NEUTRINO's output differs run to run + by about as much as the signal (that is why it is cached), and with `-t` it differs by the same, + so no effect could be seen. manage's deferred render keeps the latest progress + (`deferredRenderProgress`); the plugin's view asks for it once a second (a `renderProgress` + request beside `collectRenderedAudio`) and shows the status, or the failure, under the title. + The sheet music would not scroll: MulmoChat's box is `overflow: hidden` and the card did not + scroll itself (during playback OSMD's `scrollIntoView` moved the clipped box, which looked like + scrolling). The card is now `height: 100%; overflow-y: auto`. Browser check with the demo's + stand-in runtime: status right under the title, 0 % → 84 % → audio; sheet music in a 300 px + box scrolled 400 px with the wheel; no console errors. Both packages 1.1.0 (new `onProgress` + and `deferredRenderProgress`; the plugin's peer ^1.1.0). diff --git a/gui-plugin/README.md b/gui-plugin/README.md index 9fdaaa8..e0db297 100644 --- a/gui-plugin/README.md +++ b/gui-plugin/README.md @@ -36,8 +36,9 @@ the data also carries the score that was rendered. Every other action answers th to minutes. In a host that gives the view a runtime (`dispatch`) and tells `execute` so with `deferredAudio: true` in its context, `renderScore` waits at most a second for the audio: a short jingle comes back finished as before; a longer one comes back at once as a `pending` card that -shows the score, its lanes, cues, lyrics and sheet music, says "Rendering the audio…", and keeps -play disabled. The view collects the finished audio through the host's dispatch (a request that +shows the score, its lanes, cues, lyrics and sheet music, says at the top that the audio is +rendering, and keeps play disabled. While NEUTRINO sings, it says how far it is ("Singing the +vocal… 52 %", then "Mixing…"), asked once a second through the same dispatch. The view collects the finished audio through the host's dispatch (a request that reaches the same `execute`), saves it into the result with `onUpdateResult` (so the chat keeps it), and only then plays. `autoplay` waits for the audio in the same way, and counts as fresh from when it was ready; the voice rules above still apply. If the audio cannot be made after all, the card diff --git a/gui-plugin/demo/main.ts b/gui-plugin/demo/main.ts index ac81b89..c4ec2cb 100644 --- a/gui-plugin/demo/main.ts +++ b/gui-plugin/demo/main.ts @@ -9,6 +9,7 @@ import { z } from "zod"; // The schema modules alone: the library's index needs Node (ffmpeg), which a page cannot load. import { ScoreBaseSchema } from "../../src/score-schema.ts"; import { TimingSchema } from "../../src/timing.ts"; +import { PROGRESS_REQUEST } from "../src/audio-request.ts"; import Preview from "../src/vue/Preview.vue"; import View from "../src/vue/View.vue"; @@ -62,21 +63,31 @@ const collectedById = new Map( parsed.data.flatMap((sample) => (sample.data.pending && sample.collected ? [[sample.data.pending, sample.collected] as const] : [])), ); -/** A stand-in for the host's runtime: its dispatch answers a pending card's request for its audio. */ +const loadedAt = Date.now(); + +/** How far the stand-in render is: NEUTRINO singing for most of COLLECT_MS, then the mix. */ +function standInProgress(): object { + const done = (Date.now() - loadedAt) / (0.8 * COLLECT_MS); + return done < 1 ? { progress: { stage: "singing", done }, message: "" } : { progress: { stage: "mixing" }, message: "" }; +} + +/** A stand-in for the host's runtime: its dispatch answers a pending card's requests for its audio and its progress. */ const runtime: BrowserPluginRuntime = { pubsub: { subscribe: () => () => undefined }, locale: ref("en"), log: { debug: () => undefined, info: () => undefined, warn: () => undefined, error: () => undefined }, openUrl: (url: string) => window.open(url, "_blank", "noopener,noreferrer"), dispatch: (args: object) => - new Promise((resolve) => { - const id: unknown = Reflect.get(args, "id"); - const data = typeof id === "string" ? collectedById.get(id) : undefined; - setTimeout( - () => resolve(data ? { outcome: "ready", data, message: "The audio is ready." } : { outcome: "gone", message: "Not in this demo." }), - COLLECT_MS, - ); - }), + Reflect.get(args, "kind") === PROGRESS_REQUEST + ? Promise.resolve(standInProgress()) + : new Promise((resolve) => { + const id: unknown = Reflect.get(args, "id"); + const data = typeof id === "string" ? collectedById.get(id) : undefined; + setTimeout( + () => resolve(data ? { outcome: "ready", data, message: "The audio is ready." } : { outcome: "gone", message: "Not in this demo." }), + COLLECT_MS, + ); + }), }; /** What the views would send the LLM, listed on the page. */ diff --git a/gui-plugin/src/audio-request.ts b/gui-plugin/src/audio-request.ts index 793664b..751f631 100644 --- a/gui-plugin/src/audio-request.ts +++ b/gui-plugin/src/audio-request.ts @@ -1,8 +1,8 @@ // The view's request for audio that was still rendering when its card appeared (a `pending` -// PlayerData), and the answer. The view sends it through the host's runtime dispatch, which reaches +// PlayerData), and the answer; and its request for how far that render is, asked while it waits. The view sends it through the host's runtime dispatch, which reaches // the same execute as the LLM's tool calls; execute answers it before reading the call as a tool // call. Shared by both entries, so the two sides cannot drift apart. -import type { PlayerData } from "jinglescript"; +import type { PlayerData, RenderProgress } from "jinglescript"; /** `kind` of the view's request; not a manageJingleScript action, so the LLM never sends it. */ export const AUDIO_REQUEST = "collectRenderedAudio"; @@ -16,6 +16,21 @@ export interface AudioRequest { /** What execute answers: the finished PlayerData, why the render failed, or that it is not on the server (any more). */ export type AudioReply = { outcome: "ready"; data: PlayerData; message: string } | { outcome: "failed" | "gone"; message: string }; +/** `kind` of the view's request for how far a pending render is; answered at once. */ +export const PROGRESS_REQUEST = "renderProgress"; + +export interface ProgressRequest { + kind: typeof PROGRESS_REQUEST; + /** The pending PlayerData's `pending` id. */ + id: string; +} + +/** What execute answers: how far the render is, when it has said (nothing for a render without a sung track, or one that is done or gone). */ +export interface ProgressReply { + progress?: RenderProgress; + message: string; +} + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; export function isAudioRequest(args: unknown): args is AudioRequest { @@ -42,3 +57,17 @@ export function parseAudioReply(raw: unknown): AudioReply | undefined { if (raw["outcome"] === "failed" || raw["outcome"] === "gone") return { outcome: raw["outcome"], message }; return undefined; } + +export function isProgressRequest(args: unknown): args is ProgressRequest { + return isRecord(args) && args["kind"] === PROGRESS_REQUEST && typeof args["id"] === "string"; +} + +/** The progress in a reply as the view receives it, checked; undefined when there is none. */ +export function parseProgressReply(raw: unknown): RenderProgress | undefined { + const progress = isRecord(raw) ? raw["progress"] : undefined; + if (!isRecord(progress)) return undefined; + if (progress["stage"] === "mixing") return { stage: "mixing" }; + const done = progress["done"]; + if (progress["stage"] === "singing" && typeof done === "number" && done >= 0 && done <= 1) return { stage: "singing", done }; + return undefined; +} diff --git a/gui-plugin/src/index.ts b/gui-plugin/src/index.ts index 4395a31..e505c40 100644 --- a/gui-plugin/src/index.ts +++ b/gui-plugin/src/index.ts @@ -4,10 +4,10 @@ // A score `path` is read through the host's `files.byPath` (as presentDocument's is): the host // decides what a path may reach and what a relative one means. Without it, `path` is refused. // A host whose view can call back (`deferredAudio`) gets the card before a long render's audio: -// the view collects it with an AudioRequest, answered here. +// the view collects it with an AudioRequest, answered here, and asks how far it is meanwhile. import type { FileOps, ToolContext, ToolPluginCore, ToolResult } from "gui-chat-protocol"; -import { awaitDeferredPlayer, MANAGE_TOOL, ManageInputSchema, manage, type PlayerData } from "jinglescript"; -import { isAudioRequest, type AudioReply, type AudioRequest } from "./audio-request.ts"; +import { awaitDeferredPlayer, deferredRenderProgress, MANAGE_TOOL, ManageInputSchema, manage, type PlayerData } from "jinglescript"; +import { isAudioRequest, isProgressRequest, type AudioReply, type AudioRequest, type ProgressReply, type ProgressRequest } from "./audio-request.ts"; import { TOOL_DEFINITION } from "./definition.ts"; export { TOOL_DEFINITION }; @@ -38,6 +38,12 @@ export async function collectAudio(request: AudioRequest): Promise { } } +/** The answer to a view's ProgressRequest: how far the render it names is, at once. */ +export function renderProgress(request: ProgressRequest): Promise { + const progress = deferredRenderProgress(request.id); + return Promise.resolve(progress === undefined ? { message: "No progress to tell." } : { progress, message: `Rendering: ${progress.stage}.` }); +} + export async function executeManage(context: ManageContext, args: object): Promise> { const input = ManageInputSchema.safeParse(args); if (!input.success) { @@ -77,8 +83,8 @@ export function instructionsFor(player: PlayerData): string { export const pluginCore: ToolPluginCore = { toolDefinition: TOOL_DEFINITION, - // The view's AudioRequest comes through the same execute (the host's dispatch route) as the tool calls. - execute: (context, args) => (isAudioRequest(args) ? collectAudio(args) : executeManage(context, args)), + // The view's requests come through the same execute (the host's dispatch route) as the tool calls. + execute: (context, args) => (isAudioRequest(args) ? collectAudio(args) : isProgressRequest(args) ? renderProgress(args) : executeManage(context, args)), generatingMessage: "Rendering the jingle…", isEnabled: () => true, }; diff --git a/gui-plugin/src/vue/View.vue b/gui-plugin/src/vue/View.vue index d9999ff..8905250 100644 --- a/gui-plugin/src/vue/View.vue +++ b/gui-plugin/src/vue/View.vue @@ -7,14 +7,26 @@ // host (`isAudioPlaying`), what plays by itself waits until the voice has been quiet a moment, and // pauses while it speaks again. A card shown while its audio still renders (`pending`) shows the // score at once, collects the audio through the host's runtime dispatch, keeps it in the result, and -// only then can play (by itself too, when that was asked for). +// only then can play (by itself too, when that was asked for); meanwhile it asks how far the render +// is and says so at the top. The card scrolls itself: a host may give it a box that does not. import type { ToolResult, ToolResultComplete } from "gui-chat-protocol"; import { PLUGIN_RUNTIME_KEY } from "gui-chat-protocol/vue"; -import type { PlayerData, PlayerView } from "jinglescript"; +import type { PlayerData, PlayerView, RenderProgress } from "jinglescript"; import { computed, defineAsyncComponent, inject, nextTick, onBeforeUnmount, onMounted, ref, watch } from "vue"; -import { AUDIO_REQUEST, parseAudioReply } from "../audio-request.ts"; +import { AUDIO_REQUEST, parseAudioReply, parseProgressReply, PROGRESS_REQUEST } from "../audio-request.ts"; import { formatTime, karaoke, lanes, waveformPath, WIDTH, xOf } from "./layout.ts"; -import { failedMessage, finishedMessage, memoryOf, quietWait, shouldAutoplay, VIEW_LABELS, type PlayerMemory, type VoiceState } from "./playback.ts"; +import { + failedMessage, + finishedMessage, + memoryOf, + PROGRESS_POLL_MS, + quietWait, + renderingLabel, + shouldAutoplay, + VIEW_LABELS, + type PlayerMemory, + type VoiceState, +} from "./playback.ts"; // The default keeps a host that never passes isAudioPlaying apart from one whose voice is quiet: // Vue would otherwise turn a missing boolean prop into false. @@ -39,6 +51,9 @@ const collected = ref(undefined); const failure = ref(undefined); const data = computed(() => collected.value ?? props.selectedResult.data); const pending = computed(() => data.value?.pending !== undefined); +/** How far a pending card's render is, as the server last said. */ +const renderProgress = ref(undefined); +let progressTimer: ReturnType | undefined; const memory = ref(memoryOf(props.selectedResult.viewState)); /** The view a result opens in: as last shown, else as the LLM asked; the waveform when there is no sheet music. */ const firstView = (): PlayerView => (data.value?.musicxml === undefined ? "waveform" : (memory.value.view ?? data.value.presentation?.view ?? "waveform")); @@ -178,6 +193,20 @@ async function collect(): Promise { if (!disposed) autoplay(); } +/** Asks how far the pending render is, again and again until the card has its audio or failed. */ +async function followProgress(): Promise { + clearTimeout(progressTimer); + const id = data.value?.pending; + if (id === undefined || runtime === null || disposed || failure.value !== undefined) return; + try { + const progress = parseProgressReply(await runtime.dispatch({ kind: PROGRESS_REQUEST, id })); + if (data.value?.pending === id) renderProgress.value = progress; + } catch { + // only the label is missing: the audio still comes through collect() + } + if (!disposed && data.value?.pending === id && failure.value === undefined) progressTimer = setTimeout(() => void followProgress(), PROGRESS_POLL_MS); +} + /** No audio after all: shown in the card, and told to the LLM once when it expected it to play. */ function fail(reason: string): void { failure.value = reason; @@ -191,8 +220,11 @@ function fail(reason: string): void { /** What a card does when it appears: collect its audio, or play by itself when asked. */ function start(): void { appeared(); - if (pending.value) void collect(); - else autoplay(); + renderProgress.value = undefined; + if (pending.value) { + void collect(); + void followProgress(); + } else autoplay(); } /** Plays what plays by itself once the voice has been quiet long enough; pauses it while the voice speaks. */ @@ -266,6 +298,7 @@ onBeforeUnmount(() => { disposed = true; cancelAnimationFrame(frame); clearTimeout(quietTimer); + clearTimeout(progressTimer); }); @@ -289,6 +322,9 @@ onBeforeUnmount(() => { + +

{{ renderingLabel(renderProgress) }}

+