feat(plugins): add clipboard image and media URL APIs

This commit is contained in:
t8y2 committed 2026-09-27 09:52:03 +08:00
1 parent 0ea6598c90
commit 422908e78f
10 files changed
+494 -23

No files matched your search

@@ -3,7 +3,7 @@ import { computed, inject, nextTick, onBeforeUnmount, onMounted, ref, watch } fr
import { AlertTriangle, Loader2 } from "@lucide/vue";
import * as api from "@/lib/backend/api";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { copyToClipboard, readTextFromClipboard } from "@/lib/common/clipboard";
import { copyToClipboard, readImageFromClipboard, readTextFromClipboard } from "@/lib/common/clipboard";
import {
PluginHostBridge,
pluginSandboxDocument,
@@ -419,12 +419,19 @@ function createBridge() {
// Permission-gated in the bridge (host.clipboard:read); the helper
// prefers the Tauri clipboard plugin and falls back to the Web Clipboard.
clipboardRead: (_pluginId) => readTextFromClipboard(),
clipboardReadImage: isTauriRuntime() ? (_pluginId) => readImageFromClipboard() : undefined,
// Session consent for the first clipboard read: a native ask dialog naming
// the plugin, so reads always have a human in the loop. On the web host
// (no dialog surface) the callback is omitted and the bridge denies.
confirmClipboardRead: isTauriRuntime()
? (_pluginId, pluginName) => import("@tauri-apps/plugin-dialog").then(({ ask }) => ask(t("pluginPlatform.clipboardReadConsent", { name: pluginName }), { title: t("pluginPlatform.clipboardReadConsentTitle"), kind: "warning" }).then((allowed) => allowed === true))
: undefined,
openMedia: isTauriRuntime()
? (pluginId, method, params) => tauriFileApi().then(({ openPluginMedia }) => openPluginMedia(pluginId, method, params))
: undefined,
closeMedia: isTauriRuntime()
? (pluginId, token) => tauriFileApi().then(({ closePluginMedia }) => closePluginMedia(pluginId, token))
: undefined,
pickFiles: (pluginId, options) => pickPluginFiles(pluginId, options),
readFileChunk: (pluginId, handleId, offset, length) => readPluginFileChunkById(pluginId, handleId, offset, length),
beginFileSave: (pluginId, request) => beginPluginFileSave(pluginId, request),
+8
View File
@@ -5372,6 +5372,14 @@ export async function closePluginLocalFile(_pluginId: string, _handleId: string)
throw new Error("Plugin local file access is not available in the web backend");
}
export async function openPluginMedia(_pluginId: string, _method: string, _params: Record<string, unknown>): Promise<string> {
throw new Error("Plugin media URLs are not available in the web backend");
}
export async function closePluginMedia(_pluginId: string, _token: string): Promise<void> {
throw new Error("Plugin media URLs are not available in the web backend");
}
// Plugin UI storage goes through the Rust plugin-data tree on native hosts;
// the web workbench host keeps its localStorage fallback in the component, so
// these only exist to satisfy the shared backend surface.
+8
View File
@@ -2673,6 +2673,14 @@ export async function closePluginLocalFile(pluginId: string, handleId: string):
return invoke("plugin_file_close", { pluginId, handleId });
}
export async function openPluginMedia(pluginId: string, method: string, params: Record<string, unknown>): Promise<string> {
return invoke("plugin_media_open", { pluginId, method, params });
}
export async function closePluginMedia(pluginId: string, token: string): Promise<void> {
return invoke("plugin_media_close", { pluginId, token });
}
export async function getPluginUiStorage(pluginId: string, key: string): Promise<unknown> {
return invoke("plugin_ui_storage_get", { pluginId, key });
}
+40
View File
@@ -160,6 +160,46 @@ export async function readTextFromClipboard(env: ClipboardEnvironment = globalTh
throw new Error("Clipboard API is not available");
}
export interface ClipboardImagePayload {
contentType: "image/png";
dataBase64: string;
width: number;
height: number;
}
export async function readImageFromClipboard(): Promise<ClipboardImagePayload> {
if (!isTauriRuntime()) throw new Error("Clipboard image reads require the desktop host");
const { readImage } = await import("@tauri-apps/plugin-clipboard-manager");
const image = await readImage();
try {
const [{ width, height }, rgba] = await Promise.all([image.size(), image.rgba()]);
if (!width || !height || rgba.byteLength !== width * height * 4) throw new Error("Clipboard does not contain a valid image");
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const context = canvas.getContext("2d");
if (!context) throw new Error("Clipboard image conversion is unavailable");
const pixels = new Uint8ClampedArray(rgba.byteLength);
pixels.set(rgba);
context.putImageData(new ImageData(pixels, width, height), 0, 0);
const blob = await new Promise<Blob>((resolve, reject) => {
canvas.toBlob((value) => (value ? resolve(value) : reject(new Error("Clipboard image conversion failed"))), "image/png");
});
return { contentType: "image/png", dataBase64: bytesToBase64(new Uint8Array(await blob.arrayBuffer())), width, height };
} finally {
await image.close();
}
}
function bytesToBase64(bytes: Uint8Array): string {
let binary = "";
const chunkSize = 0x8000;
for (let offset = 0; offset < bytes.length; offset += chunkSize) {
binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
}
return btoa(binary);
}
export async function copyToClipboard(text: string, env: ClipboardEnvironment = globalThis as unknown as ClipboardEnvironment): Promise<void> {
text = clipboardLineEndings(text);
+45 -1
View File
@@ -1116,6 +1116,47 @@ describe("PluginHostBridge", () => {
expect(result?.text).toHaveLength(2 * 1024 * 1024);
});
it("shares clipboard consent with image reads and validates the image payload", async () => {
const messages: unknown[] = [];
const target = { postMessage: (message: unknown) => messages.push(message) } as unknown as Window;
const send = (bridge: PluginHostBridge, id: string, method: string, params?: unknown) => bridge.handleWindowMessage({ source: target, data: { source: "dbx-plugin", version: 1, type: "request", id, method, params } } as MessageEvent);
const clipboardReadImage = vi.fn().mockResolvedValue({ contentType: "image/png", dataBase64: "AQIDBA==", width: 1, height: 1 });
const bridge = new PluginHostBridge(plugin(["host.clipboard:read"]), workbench, {}, () => target, {
invoke: vi.fn(),
notify: vi.fn(),
sendBinary: vi.fn(),
readAsset: vi.fn(),
clipboardReadImage,
confirmClipboardRead: vi.fn().mockResolvedValue(true),
});
send(bridge, "image-ok", "host.clipboardReadImage");
await vi.waitFor(() => expect(messages).toHaveLength(1));
expect(clipboardReadImage).toHaveBeenCalledWith("sample");
expect(messages[0]).toMatchObject({ id: "image-ok", result: { contentType: "image/png", dataBase64: "AQIDBA==", width: 1, height: 1 } });
});
it("opens and revokes plugin-scoped media URLs", async () => {
const messages: unknown[] = [];
const target = { postMessage: (message: unknown) => messages.push(message) } as unknown as Window;
const send = (bridge: PluginHostBridge, id: string, method: string, params?: unknown) => bridge.handleWindowMessage({ source: target, data: { source: "dbx-plugin", version: 1, type: "request", id, method, params } } as MessageEvent);
const openMedia = vi.fn().mockResolvedValue("550e8400-e29b-41d4-a716-446655440000");
const closeMedia = vi.fn().mockResolvedValue(undefined);
const bridge = new PluginHostBridge(plugin(), workbench, {}, () => target, {
invoke: vi.fn(),
notify: vi.fn(),
sendBinary: vi.fn(),
readAsset: vi.fn(),
openMedia,
closeMedia,
});
send(bridge, "media-open", "host.mediaOpen", { method: "filesystem/media/read", params: { uri: "s3://bucket/video.mp4" } });
await vi.waitFor(() => expect(messages).toHaveLength(1));
expect(openMedia).toHaveBeenCalledWith("sample", "filesystem/media/read", { uri: "s3://bucket/video.mp4" });
send(bridge, "media-close", "host.mediaClose", { token: "550e8400-e29b-41d4-a716-446655440000" });
await vi.waitFor(() => expect(messages).toHaveLength(2));
expect(closeMedia).toHaveBeenCalledWith("sample", "550e8400-e29b-41d4-a716-446655440000");
});
it("asks session consent before the first clipboard read and remembers a denial for the bridge lifetime", async () => {
const messages: unknown[] = [];
const target = { postMessage: (message: unknown) => messages.push(message) } as unknown as Window;
@@ -1345,11 +1386,14 @@ describe("PluginHostBridge", () => {
expect(clipboardReadGateAllows(createClipboardReadGate(), 0)).toBe(true);
});
it("exposes the sandbox clipboard namespace mapping writeText to host.copy and readText to host.clipboardRead", () => {
it("exposes clipboard image reads and plugin media URLs in the sandbox SDK", () => {
const source = pluginSdkSource();
expect(source).toContain("clipboard: Object.freeze({");
expect(source).toContain("writeText: (text) => request('host.copy', { text })");
expect(source).toContain("request('host.clipboardRead')");
expect(source).toContain("readImage: async () => request('host.clipboardReadImage')");
expect(source).toContain("request('host.mediaOpen', { method, params })");
expect(source).toContain("request('host.mediaClose', { token })");
});
it("routes host.storage through the owning plugin, caps values, and needs the declared permission", async () => {
@@ -10,6 +10,7 @@ const HOST_MESSAGE_SOURCE = "dbx-host";
const BRIDGE_VERSION = 1;
const MAX_BRIDGE_PAYLOAD_BYTES = 2 * 1024 * 1024;
const MAX_BRIDGE_BINARY_BYTES = 8 * 1024 * 1024;
const MAX_CLIPBOARD_IMAGE_BASE64_BYTES = 24 * 1024 * 1024;
// Distinct from the sidecar binary cap: saved files go straight from the
// plugin iframe to disk and never traverse plugin frames.
const MAX_BRIDGE_SAVE_BYTES = 512 * 1024 * 1024;
@@ -34,7 +35,7 @@ export interface PluginClipboardAuditEntry {
at: number;
/** Request outcome: granted (content returned), denied (user or no consent surface). */
outcome: "granted" | "denied" | "rate-limited";
/** Content length in UTF-16 code units; the content itself is never stored. */
/** Returned payload size (text code units or decoded image bytes); content is never stored. */
length: number;
}
@@ -89,6 +90,13 @@ export interface PluginWorkbenchContext {
[key: string]: unknown;
}
export interface PluginClipboardImage {
contentType: "image/png";
dataBase64: string;
width: number;
height: number;
}
export interface PluginSaveFileRequest {
fileName?: string;
contentType?: string;
@@ -197,6 +205,8 @@ export interface PluginHostBridgeApi {
* further user interaction, so it is permission-gated.
*/
clipboardRead?(pluginId: string): Promise<string>;
/** Read and PNG-encode the current clipboard image. Shares the clipboard-read permission and consent gate. */
clipboardReadImage?(pluginId: string): Promise<PluginClipboardImage>;
/**
* Session consent prompt for the first clipboard read of a bridge lifetime.
* Resolves true to allow (and remember for the workbench session), false to
@@ -204,6 +214,10 @@ export interface PluginHostBridgeApi {
* surface; a host that cannot ask must not silently allow.
*/
confirmClipboardRead?(pluginId: string, pluginName: string): Promise<boolean> | boolean;
/** Register a short-lived, plugin-scoped custom-protocol media source. */
openMedia?(pluginId: string, method: string, params: Record<string, unknown>): Promise<string>;
/** Revoke a media source token previously returned by openMedia. */
closeMedia?(pluginId: string, token: string): Promise<void>;
/** Native open dialog; resolves opened read handles (null selection → empty list). */
pickFiles?(pluginId: string, options: PluginPickFilesOptions): Promise<PluginFileHandleMeta[]>;
/** Stream a chunk from an opened read handle. */
@@ -235,6 +249,7 @@ interface PluginRequestMessage {
export class PluginHostBridge {
private downloads = new Set<string>();
private mediaTokens = new Set<string>();
private context: PluginWorkbenchContext;
private locale: string;
private theme?: PluginBridgeTheme;
@@ -311,6 +326,8 @@ export class PluginHostBridge {
if ((this.initSignals.load && this.initSignals.ready) || (signal === "load" && this.initSignals.load)) {
for (const downloadId of this.downloads) void this.api.cancelDownload?.(this.plugin.manifest.id, downloadId).catch(() => undefined);
this.downloads.clear();
for (const token of this.mediaTokens) void this.api.closeMedia?.(this.plugin.manifest.id, token).catch(() => undefined);
this.mediaTokens.clear();
this.initGeneration += 1;
this.initSignals = { load: false, ready: false };
this.initStarted = false;
@@ -341,6 +358,8 @@ export class PluginHostBridge {
this.disposed = true;
for (const downloadId of this.downloads) void this.api.cancelDownload?.(this.plugin.manifest.id, downloadId).catch(() => undefined);
this.downloads.clear();
for (const token of this.mediaTokens) void this.api.closeMedia?.(this.plugin.manifest.id, token).catch(() => undefined);
this.mediaTokens.clear();
}
private disposed = false;
@@ -404,6 +423,8 @@ export class PluginHostBridge {
// host omits these, and a web host has neither.
clipboardWrite: !!this.api.copyText,
clipboardRead: !!this.api.clipboardRead,
clipboardImageRead: !!this.api.clipboardReadImage,
mediaUrl: !!this.api.openMedia && !!this.api.closeMedia,
},
context: snapshotPluginWorkbenchContext(this.context),
});
@@ -517,6 +538,21 @@ export class PluginHostBridge {
await this.api.sendBinary(this.plugin.manifest.id, channel, requireBase64(input.dataBase64));
return null;
}
if (method === "host.mediaOpen") {
if (!this.api.openMedia || !this.api.closeMedia) throw new Error("Host media URLs are unavailable");
const input = requireRecord(params, "media params");
const backendMethod = requireProtocolName(input.method, "media backend method");
const backendParams = requireRecord(input.params, "media backend params");
const token = await this.api.openMedia(this.plugin.manifest.id, backendMethod, backendParams);
this.mediaTokens.add(token);
return { token };
}
if (method === "host.mediaClose") {
const input = requireRecord(params, "media close params");
const token = requireProtocolName(input.token, "media token");
if (this.mediaTokens.delete(token)) await this.api.closeMedia?.(this.plugin.manifest.id, token);
return null;
}
if (method === "ui.readAsset") {
const input = requireRecord(params, "ui.readAsset params");
return this.api.readAsset(this.plugin.manifest.id, requireSafeAssetPath(input.path));
@@ -612,29 +648,26 @@ export class PluginHostBridge {
// `host.clipboard:read` (writes stay on ungated host.copy).
this.requirePermission("host.clipboard:read");
if (!this.api.clipboardRead) throw new Error("Host clipboard read is unavailable");
const now = Date.now();
if (!clipboardReadGateAllows(this.clipboardReadGate, now)) {
recordClipboardRead(this.clipboardReadGate, now, "rate-limited", 0);
throw new Error("Clipboard read rate limit exceeded; retry in a moment");
}
// Session consent: the first read asks the user through the host's
// dialog surface; a denial is remembered for this workbench session (an
// iframe reload rebuilds the bridge and asks again). A host without a
// consent surface denies rather than silently allowing.
if (this.clipboardReadGate.consented === null) {
const answer = this.api.confirmClipboardRead ? await this.api.confirmClipboardRead(this.plugin.manifest.id, this.plugin.manifest.name) : false;
this.clipboardReadGate.consented = answer === true;
if (!this.clipboardReadGate.consented) {
recordClipboardRead(this.clipboardReadGate, now, "denied", 0);
throw new Error("Clipboard read was denied for this plugin session");
}
}
const now = await this.requireClipboardRead();
const text = await this.api.clipboardRead(this.plugin.manifest.id);
if (typeof text !== "string") throw new Error("Host clipboard read returned a non-string value");
const clamped = text.length > MAX_BRIDGE_PAYLOAD_BYTES ? text.slice(0, MAX_BRIDGE_PAYLOAD_BYTES) : text;
recordClipboardRead(this.clipboardReadGate, now, "granted", clamped.length);
return { text: clamped };
}
if (method === "host.clipboardReadImage") {
this.requirePermission("host.clipboard:read");
if (!this.api.clipboardReadImage) throw new Error("Host clipboard image read is unavailable");
const now = await this.requireClipboardRead();
const image = await this.api.clipboardReadImage(this.plugin.manifest.id);
if (image.contentType !== "image/png" || !Number.isSafeInteger(image.width) || image.width <= 0 || !Number.isSafeInteger(image.height) || image.height <= 0) {
throw new Error("Host clipboard image returned invalid metadata");
}
const dataBase64 = requireBase64(image.dataBase64);
if (dataBase64.length > MAX_CLIPBOARD_IMAGE_BASE64_BYTES) throw new Error("Clipboard image exceeds the 18 MiB upload limit");
recordClipboardRead(this.clipboardReadGate, now, "granted", Math.floor(dataBase64.length * 0.75));
return { ...image, dataBase64 };
}
if (method === "host.pickFiles") {
// Same trust level as host.saveFile: the bytes only flow after the user
// picked the files in the native dialog, so no manifest permission gate.
@@ -709,6 +742,23 @@ export class PluginHostBridge {
throw new Error(`Unsupported plugin host method '${method}'`);
}
private async requireClipboardRead(): Promise<number> {
const now = Date.now();
if (!clipboardReadGateAllows(this.clipboardReadGate, now)) {
recordClipboardRead(this.clipboardReadGate, now, "rate-limited", 0);
throw new Error("Clipboard read rate limit exceeded; retry in a moment");
}
if (this.clipboardReadGate.consented === null) {
const answer = this.api.confirmClipboardRead ? await this.api.confirmClipboardRead(this.plugin.manifest.id, this.plugin.manifest.name) : false;
this.clipboardReadGate.consented = answer === true;
if (!this.clipboardReadGate.consented) {
recordClipboardRead(this.clipboardReadGate, now, "denied", 0);
throw new Error("Clipboard read was denied for this plugin session");
}
}
return now;
}
private requirePermission(permission: string): void {
if (!this.hasPermission(permission)) throw new Error(`Plugin has not declared permission '${permission}'`);
}
@@ -1080,6 +1130,16 @@ export function pluginSdkSource(initialTheme?: PluginBridgeTheme): string {
const result = await request('host.clipboardRead');
return (result && typeof result === 'object' && typeof result.text === 'string') ? result.text : '';
},
readImage: async () => request('host.clipboardReadImage'),
}),
media: Object.freeze({
open: async (method, params) => {
const result = await request('host.mediaOpen', { method, params });
const token = result && typeof result === 'object' && typeof result.token === 'string' ? result.token : '';
if (!token) throw new Error('Host returned an invalid media token');
return { token, url: new URL('__media/' + encodeURIComponent(token), document.baseURI).href };
},
close: (token) => request('host.mediaClose', { token }),
}),
// Persistent per-plugin key-value state; gate on capabilities.storage
// (older hosts omit it) and declare the host.storage permission.