fix: forward display keyboard and clipboard input #1854

Merged
hongming merged 1 commits from fix/display-keyboard-clipboard into main 2026-05-25 19:34:16 +00:00
7 changed files with 170 additions and 7 deletions
+4 -2
View File
@@ -224,12 +224,14 @@ export function Toolbar() {
useEffect(() => {
const handler = (e: KeyboardEvent) => {
if (e.key !== "?") return;
const tag = (e.target as HTMLElement).tagName;
const target = e.target as HTMLElement;
if (target.closest?.('[data-display-stream="true"]')) return;
const tag = target.tagName;
const inInput =
tag === "INPUT" ||
tag === "TEXTAREA" ||
tag === "SELECT" ||
(e.target as HTMLElement).isContentEditable;
target.isContentEditable;
if (inInput) return;
// Don't fire when a modal/dialog is already mounted (canvas modals,
// side panel, etc. use z-50 or above).
@@ -68,7 +68,11 @@ afterEach(() => {
function ShortcutTestComponent() {
useKeyboardShortcuts();
return <div data-testid="canvas-root" />;
return (
<div data-testid="canvas-root">
<div data-testid="display-stream" data-display-stream="true" />
</div>
);
}
function renderWithProvider() {
@@ -78,6 +82,13 @@ function renderWithProvider() {
// ─── Tests ───────────────────────────────────────────────────────────────────
describe("Esc — deselect / close context menu", () => {
it("does not handle keys targeted at the display stream", () => {
mockStoreState.contextMenu = { x: 100, y: 100, nodeId: "n1" };
const { getByTestId } = renderWithProvider();
fireEvent.keyDown(getByTestId("display-stream"), { key: "Escape" });
expect(mockStoreState.closeContextMenu).not.toHaveBeenCalled();
});
it("closes the context menu when one is open", () => {
mockStoreState.contextMenu = { x: 100, y: 100, nodeId: "n1" };
renderWithProvider();
@@ -28,12 +28,14 @@ function hasChildren(nodeId: string, nodes: Node<WorkspaceNodeData>[]): boolean
export function useKeyboardShortcuts() {
useEffect(() => {
const handler = (e: KeyboardEvent) => {
const tag = (e.target as HTMLElement).tagName;
const target = e.target as HTMLElement;
if (target.closest?.('[data-display-stream="true"]')) return;
const tag = target.tagName;
const inInput =
tag === "INPUT" ||
tag === "TEXTAREA" ||
tag === "SELECT" ||
(e.target as HTMLElement).isContentEditable;
target.isContentEditable;
if (e.key === "Escape") {
const state = useCanvasStore.getState();
+98 -1
View File
@@ -313,11 +313,21 @@ function DisplayControlBar({
function DesktopStream({ sessionUrl }: { sessionUrl: string }) {
const containerRef = useRef<HTMLDivElement | null>(null);
const rfbRef = useRef<RFB | null>(null);
const [streamError, setStreamError] = useState<string | null>(null);
const [clipboardStatus, setClipboardStatus] = useState<string | null>(null);
const [remoteClipboardText, setRemoteClipboardText] = useState("");
useEffect(() => {
let cancelled = false;
let rfb: RFB | null = null;
let clipboardTimer: ReturnType<typeof setTimeout> | null = null;
const setTemporaryClipboardStatus = (message: string) => {
setClipboardStatus(message);
if (clipboardTimer) clearTimeout(clipboardTimer);
clipboardTimer = setTimeout(() => setClipboardStatus(null), 2500);
};
async function connect() {
setStreamError(null);
@@ -328,9 +338,19 @@ function DesktopStream({ sessionUrl }: { sessionUrl: string }) {
rfb = new mod.default(containerRef.current, stream.url, {
wsProtocols: ["binary", `molecule-display-token.${stream.token}`],
});
rfbRef.current = rfb;
rfb.scaleViewport = true;
rfb.resizeSession = true;
rfb.focusOnClick = true;
rfb.focus({ preventScroll: true });
rfb.addEventListener("clipboard", (event: Event) => {
const text = (event as CustomEvent<{ text?: string }>).detail?.text ?? "";
if (!text) return;
setRemoteClipboardText(text);
void navigator.clipboard?.writeText(text)
.then(() => setTemporaryClipboardStatus("Copied remote clipboard"))
.catch(() => setTemporaryClipboardStatus("Remote clipboard ready"));
});
rfb.addEventListener("disconnect", (event: Event) => {
const detail = (event as CustomEvent<{ clean?: boolean }>).detail;
if (!cancelled && !detail?.clean) setStreamError("Desktop stream disconnected.");
@@ -343,13 +363,83 @@ function DesktopStream({ sessionUrl }: { sessionUrl: string }) {
connect();
return () => {
cancelled = true;
if (clipboardTimer) clearTimeout(clipboardTimer);
rfbRef.current = null;
rfb?.disconnect();
};
}, [sessionUrl]);
useEffect(() => {
const onPaste = (event: ClipboardEvent) => {
if (!isDisplayEventTarget(containerRef.current, event.target)) return;
const text = event.clipboardData?.getData("text/plain") ?? "";
if (!text) return;
event.preventDefault();
rfbRef.current?.clipboardPasteFrom(text);
rfbRef.current?.focus({ preventScroll: true });
setClipboardStatus("Pasted to desktop");
};
window.addEventListener("paste", onPaste, true);
return () => window.removeEventListener("paste", onPaste, true);
}, []);
const pasteLocalClipboard = async () => {
try {
const text = await navigator.clipboard?.readText();
if (!text) {
setClipboardStatus("Clipboard is empty");
return;
}
rfbRef.current?.clipboardPasteFrom(text);
rfbRef.current?.focus({ preventScroll: true });
setClipboardStatus("Pasted to desktop");
} catch {
setClipboardStatus("Press Ctrl/Cmd+V while the desktop is focused");
}
};
const copyRemoteClipboard = async () => {
if (!remoteClipboardText) {
setClipboardStatus("No remote clipboard yet");
return;
}
try {
await navigator.clipboard.writeText(remoteClipboardText);
setClipboardStatus("Copied remote clipboard");
} catch {
setClipboardStatus("Browser blocked clipboard copy");
}
};
return (
<div className="relative min-h-0 flex-1 bg-black">
<div
data-display-stream="true"
className="relative min-h-0 flex-1 bg-black"
onMouseDown={() => rfbRef.current?.focus({ preventScroll: true })}
>
<div ref={containerRef} title="Workspace desktop" className="h-full w-full overflow-hidden bg-black" />
<div className="absolute right-3 top-3 flex items-center gap-2">
{clipboardStatus && (
<span className="rounded border border-line/50 bg-black/80 px-2 py-1 text-[10px] text-white">
{clipboardStatus}
</span>
)}
<button
type="button"
onClick={pasteLocalClipboard}
className="h-7 rounded border border-line/50 bg-black/75 px-2 text-[10px] font-medium text-white hover:bg-black"
>
Paste
</button>
<button
type="button"
onClick={copyRemoteClipboard}
className="h-7 rounded border border-line/50 bg-black/75 px-2 text-[10px] font-medium text-white hover:bg-black disabled:cursor-not-allowed disabled:opacity-50"
disabled={!remoteClipboardText}
>
Copy
</button>
</div>
{streamError && (
<div className="absolute inset-x-4 top-4 rounded border border-red-500/30 bg-red-950/80 px-3 py-2 text-[11px] text-red-100">
{streamError}
@@ -359,6 +449,13 @@ function DesktopStream({ sessionUrl }: { sessionUrl: string }) {
);
}
function isDisplayEventTarget(container: HTMLElement | null, target: EventTarget | null): boolean {
if (!container) return false;
if (target instanceof Node && container.contains(target)) return true;
const active = document.activeElement;
return active instanceof Node && container.contains(active);
}
function displayWebSocketConnection(sessionUrl: string): { url: string; token: string } {
const url = new URL(sessionUrl, window.location.href);
const token = new URLSearchParams(url.hash.replace(/^#/, "")).get("token") ?? "";
@@ -2,10 +2,12 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
const { mockGet, mockPost, mockRFBConstructor } = vi.hoisted(() => ({
const { mockGet, mockPost, mockRFBConstructor, mockRFBClipboardPasteFrom, mockRFBFocus } = vi.hoisted(() => ({
mockGet: vi.fn(),
mockPost: vi.fn(),
mockRFBConstructor: vi.fn(),
mockRFBClipboardPasteFrom: vi.fn(),
mockRFBFocus: vi.fn(),
}));
vi.mock("@/lib/api", () => ({
@@ -30,6 +32,12 @@ vi.mock("@novnc/novnc", () => ({
this.options = options;
mockRFBConstructor(target, url, options);
}
clipboardPasteFrom(text: string) {
mockRFBClipboardPasteFrom(text);
}
focus(options?: FocusOptions) {
mockRFBFocus(options);
}
disconnect() {}
},
}));
@@ -42,6 +50,8 @@ describe("DisplayTab", () => {
mockGet.mockReset();
mockPost.mockReset();
mockRFBConstructor.mockReset();
mockRFBClipboardPasteFrom.mockReset();
mockRFBFocus.mockReset();
});
it("renders unavailable state for non-display workspaces", async () => {
@@ -157,6 +167,43 @@ describe("DisplayTab", () => {
expect(mockRFBConstructor.mock.calls[0][1]).not.toContain("token=");
});
it("forwards browser paste events into the noVNC clipboard", async () => {
mockGet
.mockResolvedValueOnce({
available: true,
mode: "desktop-control",
protocol: "novnc",
width: 1920,
height: 1080,
})
.mockResolvedValueOnce({
controller: "none",
});
mockPost.mockResolvedValueOnce({
controller: "user",
controlled_by: "admin-token",
expires_at: "2026-05-23T08:48:27Z",
session_url: "/workspaces/ws-display/display/session/websockify#token=signed",
});
render(<DisplayTab workspaceId="ws-display" />);
await waitFor(() => {
expect(screen.getByRole("button", { name: "Take control" })).toBeTruthy();
});
fireEvent.click(screen.getByRole("button", { name: "Take control" }));
const desktop = await screen.findByTitle("Workspace desktop");
fireEvent.paste(desktop, {
clipboardData: {
getData: (type: string) => (type === "text/plain" ? "Paste Me" : ""),
},
});
expect(mockRFBClipboardPasteFrom).toHaveBeenCalledWith("Paste Me");
expect(mockRFBFocus).toHaveBeenCalledWith({ preventScroll: true });
});
it("releases user display control", async () => {
mockGet
.mockResolvedValueOnce({
@@ -15,6 +15,8 @@ export function useKeyboardShortcut(
if (!enabled) return;
function handler(e: KeyboardEvent) {
const target = e.target as HTMLElement;
if (target.closest?.('[data-display-stream="true"]')) return;
if (e.key !== key) return;
if (meta && !e.metaKey) return;
if (ctrl && !e.ctrlKey) return;
+2
View File
@@ -4,6 +4,8 @@ declare module "@novnc/novnc" {
resizeSession: boolean;
focusOnClick: boolean;
constructor(target: HTMLElement, url: string, options?: { wsProtocols?: string[]; [key: string]: unknown });
clipboardPasteFrom(text: string): void;
disconnect(): void;
focus(options?: FocusOptions): void;
}
}