feat(canvas/chat): inline video + audio HTML5 native players (RFC #2991 PR-2)
Second specialized renderer pair landing under RFC #2991. Stacks on PR-1 (#2997) — extends the AttachmentPreview dispatcher with video/ audio cases. Why HTML5-native (not custom JS player) --------------------------------------- - Browser vendors ship hardware-accelerated decoders, captions, pinch + scrub UX, and fullscreen UI. We get all of it for free. - Native fullscreen via the <video> control bar — no AttachmentLightbox needed for video (the browser's built-in fullscreen handles it). - Mobile-friendly without us writing the touch handlers. Auth model ---------- Identical to AttachmentImage (PR-1): platform-auth URIs need our cookie/token, so we fetch the bytes, wrap in a Blob, hand the browser an ObjectURL via <video src=> / <audio src=>. External http(s) URIs skip the fetch. Memory caveat: a Blob holds the entire media in JS memory until the bubble unmounts. The server's 25MB single-file cap (chat_files.go) bounds this; v2 can switch to MediaSource + streaming if larger files become a real shape. Failure modes ------------- - Fetch failure (404, 403, network) → AttachmentChip fallback. - Bytes that aren't valid media (corrupt, wrong Content-Type) → <video onError> / <audio onError> swap to chip. Tests ----- 5 new component tests in AttachmentPreview.test.tsx (now 14 total): - kind=video → <video controls> with blob URL src - kind=video fetch fails → falls back to chip - kind=video extension fallback (no mime) → routes to video path - kind=audio → <audio controls> + filename label visible - kind=audio fetch fails → falls back to chip The preview-kind unit tests from PR-1 (49 cases) already cover the MIME → video / audio dispatch logic; this PR's component tests pin the rendered DOM shape (controls attribute, blob URL src, fallback behavior). Hostile self-review ------------------- 1. Memory bound: 25MB cap protects us today; documented future migration path (MediaSource). 2. iOS Safari autoplay: playsInline pinned on <video> so mobile doesn't auto-fullscreen on play. 3. Captions accessibility: <track kind="captions" /> placeholder so the element is tagged correctly even though we don't have caption files yet (forward-compatible). Verified - tsc --noEmit clean - 173 chat tests green (49 unit + 14 component + 110 pre-existing) Stacks on PR-1 (#2997). PR-3 (PDF + text/code) is the final piece. Refs RFC #2991, PR #2997 (PR-1). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Hongming Wang
parent
04f7a07add
commit
95fdf86187
124
canvas/src/components/tabs/chat/AttachmentAudio.tsx
Normal file
124
canvas/src/components/tabs/chat/AttachmentAudio.tsx
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
// AttachmentAudio — inline native HTML5 <audio controls> player for
|
||||||
|
// chat attachments (RFC #2991, PR-2).
|
||||||
|
//
|
||||||
|
// Same auth + Blob-URL pattern as AttachmentImage / AttachmentVideo.
|
||||||
|
// Native audio control bar handles play/pause/scrub/volume/download,
|
||||||
|
// and there's no fullscreen UI to worry about (audio doesn't need
|
||||||
|
// AttachmentLightbox).
|
||||||
|
|
||||||
|
import { useState, useEffect, useRef } from "react";
|
||||||
|
import type { ChatAttachment } from "./types";
|
||||||
|
import { isPlatformAttachment, resolveAttachmentHref } from "./uploads";
|
||||||
|
import { AttachmentChip } from "./AttachmentViews";
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
workspaceId: string;
|
||||||
|
attachment: ChatAttachment;
|
||||||
|
onDownload: (a: ChatAttachment) => void;
|
||||||
|
tone: "user" | "agent";
|
||||||
|
}
|
||||||
|
|
||||||
|
type FetchState =
|
||||||
|
| { kind: "idle" }
|
||||||
|
| { kind: "loading" }
|
||||||
|
| { kind: "ready"; src: string }
|
||||||
|
| { kind: "error" };
|
||||||
|
|
||||||
|
export function AttachmentAudio({ workspaceId, attachment, onDownload, tone }: Props) {
|
||||||
|
const [state, setState] = useState<FetchState>({ kind: "idle" });
|
||||||
|
const blobUrlRef = useRef<string | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
setState({ kind: "loading" });
|
||||||
|
|
||||||
|
if (!isPlatformAttachment(attachment.uri)) {
|
||||||
|
const href = resolveAttachmentHref(workspaceId, attachment.uri);
|
||||||
|
if (!cancelled) setState({ kind: "ready", src: href });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
const href = resolveAttachmentHref(workspaceId, attachment.uri);
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
const adminToken = process.env.NEXT_PUBLIC_ADMIN_TOKEN;
|
||||||
|
if (adminToken) headers["Authorization"] = `Bearer ${adminToken}`;
|
||||||
|
const slug = getTenantSlug();
|
||||||
|
if (slug) headers["X-Molecule-Org-Slug"] = slug;
|
||||||
|
const res = await fetch(href, {
|
||||||
|
headers,
|
||||||
|
credentials: "include",
|
||||||
|
signal: AbortSignal.timeout(60_000),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
if (!cancelled) setState({ kind: "error" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const blob = await res.blob();
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
blobUrlRef.current = url;
|
||||||
|
if (cancelled) {
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setState({ kind: "ready", src: url });
|
||||||
|
} catch {
|
||||||
|
if (!cancelled) setState({ kind: "error" });
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
if (blobUrlRef.current) {
|
||||||
|
URL.revokeObjectURL(blobUrlRef.current);
|
||||||
|
blobUrlRef.current = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, [workspaceId, attachment.uri]);
|
||||||
|
|
||||||
|
if (state.kind === "error") {
|
||||||
|
return <AttachmentChip attachment={attachment} onDownload={onDownload} tone={tone} />;
|
||||||
|
}
|
||||||
|
if (state.kind === "idle" || state.kind === "loading") {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="rounded-md border border-line/50 bg-surface-card/40 animate-pulse"
|
||||||
|
style={{ width: 280, height: 40 }}
|
||||||
|
aria-label={`Loading ${attachment.name}`}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={`inline-flex flex-col gap-1 rounded-md border px-2 py-1 ${
|
||||||
|
tone === "user" ? "border-blue-400/30 bg-accent-strong/10" : "border-line/50 bg-surface-card/40"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{/* Filename label so the user knows what they're hearing
|
||||||
|
before pressing play. Short, single-line, truncated. */}
|
||||||
|
<span className="text-[10px] text-ink-mid truncate max-w-[280px]" title={attachment.name}>
|
||||||
|
{attachment.name}
|
||||||
|
</span>
|
||||||
|
<audio
|
||||||
|
controls
|
||||||
|
preload="metadata"
|
||||||
|
src={state.src}
|
||||||
|
style={{ width: 280, height: 32 }}
|
||||||
|
onError={() => setState({ kind: "error" })}
|
||||||
|
>
|
||||||
|
{attachment.name}
|
||||||
|
</audio>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTenantSlug(): string | null {
|
||||||
|
if (typeof window === "undefined") return null;
|
||||||
|
const host = window.location.hostname;
|
||||||
|
const m = host.match(/^([^.]+)\.moleculesai\.app$/);
|
||||||
|
return m ? m[1] : null;
|
||||||
|
}
|
||||||
@ -17,6 +17,8 @@
|
|||||||
import type { ChatAttachment } from "./types";
|
import type { ChatAttachment } from "./types";
|
||||||
import { getAttachmentPreviewKind } from "./preview-kind";
|
import { getAttachmentPreviewKind } from "./preview-kind";
|
||||||
import { AttachmentImage } from "./AttachmentImage";
|
import { AttachmentImage } from "./AttachmentImage";
|
||||||
|
import { AttachmentVideo } from "./AttachmentVideo";
|
||||||
|
import { AttachmentAudio } from "./AttachmentAudio";
|
||||||
import { AttachmentChip } from "./AttachmentViews";
|
import { AttachmentChip } from "./AttachmentViews";
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
@ -43,10 +45,25 @@ export function AttachmentPreview({ workspaceId, attachment, onDownload, tone }:
|
|||||||
tone={tone}
|
tone={tone}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
// PR-2 will add cases for video / audio.
|
|
||||||
// PR-3 will add cases for pdf / text.
|
|
||||||
case "video":
|
case "video":
|
||||||
|
return (
|
||||||
|
<AttachmentVideo
|
||||||
|
workspaceId={workspaceId}
|
||||||
|
attachment={attachment}
|
||||||
|
onDownload={onDownload}
|
||||||
|
tone={tone}
|
||||||
|
/>
|
||||||
|
);
|
||||||
case "audio":
|
case "audio":
|
||||||
|
return (
|
||||||
|
<AttachmentAudio
|
||||||
|
workspaceId={workspaceId}
|
||||||
|
attachment={attachment}
|
||||||
|
onDownload={onDownload}
|
||||||
|
tone={tone}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
// PR-3 will add cases for pdf / text.
|
||||||
case "pdf":
|
case "pdf":
|
||||||
case "text":
|
case "text":
|
||||||
case "file":
|
case "file":
|
||||||
|
|||||||
157
canvas/src/components/tabs/chat/AttachmentVideo.tsx
Normal file
157
canvas/src/components/tabs/chat/AttachmentVideo.tsx
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
// AttachmentVideo — inline native HTML5 <video controls> player for
|
||||||
|
// chat attachments (RFC #2991, PR-2).
|
||||||
|
//
|
||||||
|
// Why HTML5-native (vs custom JS player):
|
||||||
|
//
|
||||||
|
// - Browser vendors ship hardware-accelerated decoders, captions,
|
||||||
|
// and fullscreen UI. We get all of it for free.
|
||||||
|
// - Native fullscreen via the <video> element's built-in button
|
||||||
|
// (no AttachmentLightbox needed for video — the browser does it).
|
||||||
|
// - Mobile-friendly: iOS / Android Safari + Chrome handle the
|
||||||
|
// pinch + scrub UX the user already knows.
|
||||||
|
//
|
||||||
|
// Auth model — identical to AttachmentImage:
|
||||||
|
// platform-auth URIs need our cookie/token, so we fetch the bytes,
|
||||||
|
// wrap in a Blob, hand the browser an ObjectURL via <video src=>.
|
||||||
|
// External (http/https) URIs skip the fetch and use the raw URL.
|
||||||
|
//
|
||||||
|
// Memory caveat: a Blob holds the entire video in JS memory until
|
||||||
|
// the bubble unmounts. For multi-hundred-MB videos this is bad. The
|
||||||
|
// server caps single-file uploads at 25MB (chat_files.go), so we're
|
||||||
|
// bounded; if larger files become a real shape, switch to streaming
|
||||||
|
// via MediaSource or just `<video src=…>` with a credentials-aware
|
||||||
|
// fetch via service worker. v2 if measured-needed.
|
||||||
|
|
||||||
|
import { useState, useEffect, useRef } from "react";
|
||||||
|
import type { ChatAttachment } from "./types";
|
||||||
|
import { isPlatformAttachment, resolveAttachmentHref } from "./uploads";
|
||||||
|
import { AttachmentChip } from "./AttachmentViews";
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
workspaceId: string;
|
||||||
|
attachment: ChatAttachment;
|
||||||
|
onDownload: (a: ChatAttachment) => void;
|
||||||
|
tone: "user" | "agent";
|
||||||
|
}
|
||||||
|
|
||||||
|
type FetchState =
|
||||||
|
| { kind: "idle" }
|
||||||
|
| { kind: "loading" }
|
||||||
|
| { kind: "ready"; src: string }
|
||||||
|
| { kind: "error" };
|
||||||
|
|
||||||
|
export function AttachmentVideo({ workspaceId, attachment, onDownload, tone }: Props) {
|
||||||
|
const [state, setState] = useState<FetchState>({ kind: "idle" });
|
||||||
|
const blobUrlRef = useRef<string | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
setState({ kind: "loading" });
|
||||||
|
|
||||||
|
if (!isPlatformAttachment(attachment.uri)) {
|
||||||
|
// External video (http/https) — let the browser stream it
|
||||||
|
// natively without the JS-blob detour.
|
||||||
|
const href = resolveAttachmentHref(workspaceId, attachment.uri);
|
||||||
|
if (!cancelled) setState({ kind: "ready", src: href });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
const href = resolveAttachmentHref(workspaceId, attachment.uri);
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
const adminToken = process.env.NEXT_PUBLIC_ADMIN_TOKEN;
|
||||||
|
if (adminToken) headers["Authorization"] = `Bearer ${adminToken}`;
|
||||||
|
const slug = getTenantSlug();
|
||||||
|
if (slug) headers["X-Molecule-Org-Slug"] = slug;
|
||||||
|
const res = await fetch(href, {
|
||||||
|
headers,
|
||||||
|
credentials: "include",
|
||||||
|
// Videos are larger than images on average; give the request
|
||||||
|
// more headroom. The server's per-request body cap (50MB) is
|
||||||
|
// still the actual ceiling.
|
||||||
|
signal: AbortSignal.timeout(120_000),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
if (!cancelled) setState({ kind: "error" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const blob = await res.blob();
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
blobUrlRef.current = url;
|
||||||
|
if (cancelled) {
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setState({ kind: "ready", src: url });
|
||||||
|
} catch {
|
||||||
|
if (!cancelled) setState({ kind: "error" });
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
if (blobUrlRef.current) {
|
||||||
|
URL.revokeObjectURL(blobUrlRef.current);
|
||||||
|
blobUrlRef.current = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, [workspaceId, attachment.uri]);
|
||||||
|
|
||||||
|
if (state.kind === "error") {
|
||||||
|
return <AttachmentChip attachment={attachment} onDownload={onDownload} tone={tone} />;
|
||||||
|
}
|
||||||
|
if (state.kind === "idle" || state.kind === "loading") {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="rounded-md border border-line/50 bg-surface-card/40 animate-pulse"
|
||||||
|
style={{ width: 320, height: 180 }}
|
||||||
|
aria-label={`Loading ${attachment.name}`}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={`inline-block rounded-lg overflow-hidden border ${
|
||||||
|
tone === "user" ? "border-blue-400/30" : "border-line/50"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<video
|
||||||
|
controls
|
||||||
|
// preload="metadata" so the browser fetches just enough to
|
||||||
|
// show duration + first frame thumbnail without streaming
|
||||||
|
// the whole file before the user clicks play.
|
||||||
|
preload="metadata"
|
||||||
|
// playsInline keeps mobile Safari from auto-fullscreening
|
||||||
|
// on play; the user can still hit the native fullscreen
|
||||||
|
// button (or PiP on Chrome) if they want.
|
||||||
|
playsInline
|
||||||
|
// Native fullscreen via the <video> control bar; no
|
||||||
|
// AttachmentLightbox needed for video.
|
||||||
|
src={state.src}
|
||||||
|
// Cap thumbnail / inline display so the bubble doesn't blow
|
||||||
|
// up vertical layout for tall portrait clips. The native
|
||||||
|
// fullscreen button uses the original aspect ratio.
|
||||||
|
style={{ maxWidth: 320, maxHeight: 240, display: "block" }}
|
||||||
|
// Bytes that aren't actually a valid video (corrupt blob,
|
||||||
|
// wrong Content-Type) fail load → swap to chip.
|
||||||
|
onError={() => setState({ kind: "error" })}
|
||||||
|
>
|
||||||
|
<track kind="captions" />
|
||||||
|
{attachment.name}
|
||||||
|
</video>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Internal helper — same shape as AttachmentImage's. Lifted to a
|
||||||
|
// shared util in PR-2.5 if a third caller needs it (PDF, audio).
|
||||||
|
function getTenantSlug(): string | null {
|
||||||
|
if (typeof window === "undefined") return null;
|
||||||
|
const host = window.location.hostname;
|
||||||
|
const m = host.match(/^([^.]+)\.moleculesai\.app$/);
|
||||||
|
return m ? m[1] : null;
|
||||||
|
}
|
||||||
@ -154,6 +154,73 @@ describe("AttachmentPreview dispatch", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ─── PR-2: video / audio dispatch ───────────────────────────────
|
||||||
|
|
||||||
|
it("kind=video → renders <video controls> after fetch resolves", async () => {
|
||||||
|
fetchMock.mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
blob: async () => new Blob(["fake-mp4"], { type: "video/mp4" }),
|
||||||
|
});
|
||||||
|
preview({ uri: "workspace:/workspace/clip.mp4", name: "clip.mp4", mimeType: "video/mp4" });
|
||||||
|
// Loading placeholder first.
|
||||||
|
expect(await screen.findByLabelText(/Loading clip\.mp4/i)).toBeTruthy();
|
||||||
|
// After the blob resolves, a <video> element with controls=true
|
||||||
|
// is in the DOM. Use a tag query — there's no built-in role for
|
||||||
|
// <video>, but the element is unambiguous in the bubble.
|
||||||
|
await waitFor(() => {
|
||||||
|
const v = document.querySelector("video");
|
||||||
|
expect(v).not.toBeNull();
|
||||||
|
// controls attribute pinned — without it the user can't play.
|
||||||
|
expect(v?.hasAttribute("controls")).toBe(true);
|
||||||
|
// src is the blob URL we minted.
|
||||||
|
expect((v as HTMLVideoElement).src).toBe("blob:test-url");
|
||||||
|
});
|
||||||
|
// Chip MUST NOT render — proves dispatch routed to video, not file.
|
||||||
|
expect(screen.queryByTitle(/Download clip\.mp4/i)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("kind=video fetch fails → falls back to AttachmentChip", async () => {
|
||||||
|
fetchMock.mockResolvedValue({ ok: false, status: 404 });
|
||||||
|
preview({ uri: "workspace:/workspace/missing.mp4", name: "missing.mp4", mimeType: "video/mp4" });
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(screen.getByTitle(/Download missing\.mp4/i)).toBeTruthy();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("kind=video by extension fallback (no mime) → video path", async () => {
|
||||||
|
fetchMock.mockReturnValue(new Promise(() => {}));
|
||||||
|
preview({ uri: "workspace:/workspace/recording.webm", name: "recording.webm" });
|
||||||
|
expect(await screen.findByLabelText(/Loading recording\.webm/i)).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("kind=audio → renders <audio controls> with filename label", async () => {
|
||||||
|
fetchMock.mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
blob: async () => new Blob(["fake-mp3"], { type: "audio/mpeg" }),
|
||||||
|
});
|
||||||
|
preview({ uri: "workspace:/workspace/song.mp3", name: "song.mp3", mimeType: "audio/mpeg" });
|
||||||
|
await waitFor(() => {
|
||||||
|
const a = document.querySelector("audio");
|
||||||
|
expect(a).not.toBeNull();
|
||||||
|
expect(a?.hasAttribute("controls")).toBe(true);
|
||||||
|
expect((a as HTMLAudioElement).src).toBe("blob:test-url");
|
||||||
|
});
|
||||||
|
// Filename label pinned: helps the user know what they're hearing
|
||||||
|
// BEFORE pressing play. Multiple matches — `<span>` text and the
|
||||||
|
// `<audio>`'s fallback `{name}` text node — so getAllByText.
|
||||||
|
expect(screen.getAllByText("song.mp3").length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("kind=audio fetch fails → falls back to chip", async () => {
|
||||||
|
fetchMock.mockResolvedValue({ ok: false, status: 403 });
|
||||||
|
preview({ uri: "workspace:/workspace/locked.wav", name: "locked.wav", mimeType: "audio/wav" });
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(screen.getByTitle(/Download locked\.wav/i)).toBeTruthy();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── universal-fallback regression ─────────────────────────────────
|
||||||
|
|
||||||
it("kind=file is the universal fallback for unknown MIME (regression: don't try to preview a zip)", () => {
|
it("kind=file is the universal fallback for unknown MIME (regression: don't try to preview a zip)", () => {
|
||||||
// Critical safety: agent could attach a misnamed file. Pre-fix
|
// Critical safety: agent could attach a misnamed file. Pre-fix
|
||||||
// the chip path was unconditional; we want unknown MIME to
|
// the chip path was unconditional; we want unknown MIME to
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user