feat(dynamodb): support querying, editing, and exporting

This commit is contained in:
zipg authored and GitHub committed 2026-08-17 03:27:44 +08:00
1 parent 92ab9bb0dc
commit e97c9ab5d7
73 files changed
+4493 -134

No files matched your search

@@ -6,8 +6,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const backend = vi.hoisted(() => ({
getColumns: vi.fn(),
documentFindDocuments: vi.fn(),
documentCountDocuments: vi.fn(),
dynamodbDescribeTable: vi.fn(),
cancelQuery: vi.fn(),
ensureConnected: vi.fn(),
documentInsertDocument: vi.fn(),
documentUpdateDocument: vi.fn(),
documentDeleteDocument: vi.fn(),
documentSaveMeilisearchBatch: vi.fn(),
}));
@@ -24,6 +28,8 @@ const dataGrid = vi.hoisted(() => ({
column_types?: string[];
rows: Array<Array<string | number | boolean | null>>;
mongo_copy_documents?: unknown[];
truncated?: boolean;
has_more?: boolean;
}
| undefined
>)
@@ -31,8 +37,12 @@ const dataGrid = vi.hoisted(() => ({
customSaveHandler: undefined as
| {
save: (changes: { dirtyRows: Map<number, Map<number, unknown>>; deletedRows: Set<number>; newRows: unknown[][]; newRowMeta: unknown[]; columns: string[]; rows: unknown[][] }) => Promise<void>;
preview?: (changes: { dirtyRows: Map<number, Map<number, unknown>>; deletedRows: Set<number>; newRows: unknown[][]; newRowMeta: unknown[]; columns: string[]; rows: unknown[][] }) => Promise<string[]>;
}
| undefined,
countTotalRows: undefined as (() => Promise<number | undefined>) | undefined,
paginate: undefined as ((offset: number, limit: number) => Promise<void>) | undefined,
editable: true,
}));
const settings = vi.hoisted(() => ({
@@ -54,13 +64,19 @@ const settings = vi.hoisted(() => ({
vi.mock("vue-i18n", async (importOriginal) => ({
...(await importOriginal<typeof import("vue-i18n")>()),
useI18n: () => ({ t: (key: string) => key }),
useI18n: () => ({
t: (key: string, params?: Record<string, unknown>) => (key === "dynamodb.items" ? `${params?.count} Items` : key),
}),
}));
vi.mock("@/lib/backend/api", () => ({
getColumns: backend.getColumns,
documentFindDocuments: backend.documentFindDocuments,
documentCountDocuments: backend.documentCountDocuments,
dynamodbDescribeTable: backend.dynamodbDescribeTable,
cancelQuery: backend.cancelQuery,
documentInsertDocument: backend.documentInsertDocument,
documentUpdateDocument: backend.documentUpdateDocument,
documentDeleteDocument: backend.documentDeleteDocument,
documentSaveMeilisearchBatch: backend.documentSaveMeilisearchBatch,
}));
@@ -90,10 +106,14 @@ vi.mock("@/components/grid/DataGrid.vue", async () => {
columnLayoutScopeKey: { type: String, default: "" },
fullExportResult: { type: Function, default: undefined },
customSaveHandler: { type: Object, default: undefined },
countTotalRows: { type: Function, default: undefined },
editable: { type: Boolean, default: false },
},
setup(props, { expose, slots }) {
setup(props, { attrs, expose, slots }) {
dataGrid.fullExportResult = props.fullExportResult as typeof dataGrid.fullExportResult;
dataGrid.customSaveHandler = props.customSaveHandler as typeof dataGrid.customSaveHandler;
dataGrid.countTotalRows = props.countTotalRows as typeof dataGrid.countTotalRows;
dataGrid.paginate = attrs.onPaginate as typeof dataGrid.paginate;
expose({
visibleColumnCount: 2,
displayableColumnCount: 2,
@@ -113,8 +133,9 @@ vi.mock("@/components/grid/DataGrid.vue", async () => {
multiRowTranspose: false,
setMultiRowTranspose: vi.fn(),
});
return () =>
h(
return () => {
dataGrid.editable = props.editable;
return h(
"div",
{
"data-testid": "data-grid",
@@ -124,6 +145,7 @@ vi.mock("@/components/grid/DataGrid.vue", async () => {
"data-result-hidden-column-keys": JSON.stringify((props.result as { local_hidden_column_keys?: string[] }).local_hidden_column_keys ?? []),
"data-result-column-types": JSON.stringify((props.result as { column_types?: string[] }).column_types ?? []),
"data-result-rows": JSON.stringify((props.result as { rows?: unknown[] }).rows ?? []),
"data-editable": String(props.editable),
},
[
slots["search-bar"]?.({
@@ -134,6 +156,7 @@ vi.mock("@/components/grid/DataGrid.vue", async () => {
}),
],
);
};
},
}),
};
@@ -215,8 +238,9 @@ vi.mock("@/components/ui/select", async () => {
props: {
modelValue: { type: String, default: "" },
},
setup(props, { slots }) {
return () => h("div", { "data-testid": "select", "data-model-value": props.modelValue }, slots.default?.());
emits: ["update:modelValue"],
setup(props, { emit, slots }) {
return () => h("div", { "data-testid": "select", "data-model-value": props.modelValue, onClick: () => props.modelValue === "__table__" && emit("update:modelValue", "by_status") }, slots.default?.());
},
});
const passthrough = (name: string) =>
@@ -284,14 +308,23 @@ beforeEach(async () => {
});
backend.getColumns.mockReset();
backend.documentFindDocuments.mockReset();
backend.documentCountDocuments.mockReset();
backend.dynamodbDescribeTable.mockReset();
backend.cancelQuery.mockReset();
backend.ensureConnected.mockReset();
backend.documentInsertDocument.mockReset();
backend.documentUpdateDocument.mockReset();
backend.documentDeleteDocument.mockReset();
backend.documentSaveMeilisearchBatch.mockReset();
dataGrid.fullExportResult = undefined;
dataGrid.customSaveHandler = undefined;
dataGrid.countTotalRows = undefined;
dataGrid.paginate = undefined;
dataGrid.editable = true;
documentJsonEditor.openSearch.mockClear();
backend.documentDeleteDocument.mockResolvedValue(undefined);
backend.documentInsertDocument.mockResolvedValue("created");
backend.documentUpdateDocument.mockResolvedValue(1);
backend.documentSaveMeilisearchBatch.mockResolvedValue(0);
settings.editorSettings.mongoViewMode = "table";
settings.editorSettings.columnWidthDensity = "standard";
@@ -306,6 +339,16 @@ beforeEach(async () => {
settings.updateEditorSettings.mockReset();
settings.updateEditorSettings.mockImplementation((partial: Partial<typeof settings.editorSettings>) => Object.assign(settings.editorSettings, partial));
backend.ensureConnected.mockResolvedValue(undefined);
backend.documentCountDocuments.mockResolvedValue(0);
backend.dynamodbDescribeTable.mockResolvedValue({
name: "orders",
status: "ACTIVE",
itemCount: 0,
sizeBytes: 0,
partitionKey: { name: "tenant_id", attributeType: "S" },
sortKey: { name: "order_id", attributeType: "S" },
indexes: [],
});
backend.getColumns.mockResolvedValue([
{ name: "buyers", data_type: "nested" },
{ name: "buyers.email", data_type: "text" },
@@ -443,6 +486,315 @@ describe("DocumentBrowser Elasticsearch field search", () => {
});
describe("DocumentBrowser MongoDB filter value types", () => {
it("loads DynamoDB table metadata before requesting the first page", async () => {
app?.unmount();
let resolveDescription!: (value: { name: string; status: string; itemCount: number; sizeBytes: number; partitionKey: { name: string; attributeType: string }; sortKey: { name: string; attributeType: string }; indexes: never[] }) => void;
backend.dynamodbDescribeTable.mockReturnValue(
new Promise((resolve) => {
resolveDescription = resolve;
}),
);
backend.documentFindDocuments.mockClear();
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
expect(backend.dynamodbDescribeTable).toHaveBeenCalledOnce();
expect(backend.documentFindDocuments).not.toHaveBeenCalled();
resolveDescription({
name: "orders",
status: "ACTIVE",
itemCount: 0,
sizeBytes: 0,
partitionKey: { name: "tenant_id", attributeType: "S" },
sortKey: { name: "order_id", attributeType: "S" },
indexes: [],
});
await flushUi();
expect(backend.documentFindDocuments).toHaveBeenCalledOnce();
});
it("labels new DynamoDB rows as conditional inserts in the executable preview", async () => {
app?.unmount();
backend.documentFindDocuments.mockResolvedValue({
documents: [{ _id: { tenant_id: "tenant-04", order_id: "ORD-000994" }, tenant_id: "tenant-04", order_id: "ORD-000994", note: "existing" }],
total: 1,
total_is_exact: true,
});
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
const preview = await dataGrid.customSaveHandler!.preview!({
dirtyRows: new Map(),
deletedRows: new Set(),
newRows: [['{"tenant_id":"tenant-04","order_id":"ORD-new"}', "tenant-04", "ORD-new", "new item"]],
newRowMeta: [{}],
columns: ["_id", "tenant_id", "order_id", "note"],
rows: [['{"tenant_id":"tenant-04","order_id":"ORD-000994"}', "tenant-04", "ORD-000994", "existing"]],
});
expect(preview).toHaveLength(1);
expect(preview[0]).toContain('DBX DYNAMODB INSERT ITEM\ntable: "orders"');
expect(preview[0]).toContain('"order_id": "ORD-new"');
});
it("uses one backend update for a DynamoDB key migration and previews the old source key", async () => {
app?.unmount();
backend.documentFindDocuments.mockResolvedValue({
documents: [{ _id: { tenant_id: "tenant-04", order_id: "ORD-old" }, tenant_id: "tenant-04", order_id: "ORD-old", note: "existing" }],
total: 1,
total_is_exact: true,
});
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
const changes = {
dirtyRows: new Map([[0, new Map([[2, "ORD-new"]])]]),
deletedRows: new Set<number>(),
newRows: [],
newRowMeta: [],
columns: ["_id", "tenant_id", "order_id", "note"],
rows: [['{"tenant_id":"tenant-04","order_id":"ORD-old"}', "tenant-04", "ORD-old", "existing"]],
};
const preview = await dataGrid.customSaveHandler!.preview!(changes);
expect(preview).toHaveLength(1);
expect(preview[0]).toContain('key:\n{\n "tenant_id": "tenant-04",\n "order_id": "ORD-old"\n}');
expect(preview[0]).toContain('"order_id": "ORD-new"');
await dataGrid.customSaveHandler!.save(changes);
expect(backend.documentUpdateDocument).toHaveBeenCalledTimes(1);
expect(backend.documentUpdateDocument.mock.calls[0]?.slice(0, 4)).toEqual(["dynamodb-1", "us-east-1", "orders", '{"tenant_id":"tenant-04","order_id":"ORD-old"}']);
expect(JSON.parse(backend.documentUpdateDocument.mock.calls[0]?.[4])).toMatchObject({ tenant_id: "tenant-04", order_id: "ORD-new", note: "existing" });
expect(backend.documentInsertDocument).not.toHaveBeenCalled();
expect(backend.documentDeleteDocument).not.toHaveBeenCalled();
});
it("makes partial-projection DynamoDB index results read-only", async () => {
app?.unmount();
backend.dynamodbDescribeTable.mockResolvedValue({
name: "orders",
status: "ACTIVE",
itemCount: 1,
sizeBytes: 100,
partitionKey: { name: "tenant_id", attributeType: "S" },
sortKey: { name: "order_id", attributeType: "S" },
indexes: [
{
name: "by_status",
kind: "global",
partitionKey: { name: "status", attributeType: "S" },
sortKey: { name: "created_at", attributeType: "N" },
projectionType: "KEYS_ONLY",
nonKeyAttributes: [],
},
],
});
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
expect(dataGrid.editable).toBe(true);
root!.querySelector<HTMLElement>('[data-testid="select"][data-model-value="__table__"]')!.click();
await flushUi();
expect(dataGrid.editable).toBe(false);
expect(root!.textContent).toContain("KEYS_ONLY");
await expect(
dataGrid.customSaveHandler!.save({
dirtyRows: new Map(),
deletedRows: new Set(),
newRows: [],
newRowMeta: [],
columns: ["_id"],
rows: [],
}),
).rejects.toThrow("dynamodb.partialProjectionReadOnly");
});
it("keeps ALL-projection DynamoDB index results editable", async () => {
app?.unmount();
backend.dynamodbDescribeTable.mockResolvedValue({
name: "orders",
status: "ACTIVE",
itemCount: 1,
sizeBytes: 100,
partitionKey: { name: "tenant_id", attributeType: "S" },
sortKey: { name: "order_id", attributeType: "S" },
indexes: [
{
name: "by_status",
kind: "global",
partitionKey: { name: "status", attributeType: "S" },
sortKey: { name: "created_at", attributeType: "N" },
projectionType: "ALL",
nonKeyAttributes: [],
},
],
});
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
root!.querySelector<HTMLElement>('[data-testid="select"][data-model-value="__table__"]')!.click();
await flushUi();
expect(dataGrid.editable).toBe(true);
});
it("keeps an exact DynamoDB count while moving between cursor pages", async () => {
app?.unmount();
backend.documentCountDocuments.mockResolvedValue(250);
backend.documentFindDocuments.mockReset();
backend.documentFindDocuments
.mockResolvedValueOnce({
documents: Array.from({ length: 100 }, (_, index) => ({ _id: { tenant_id: "tenant-a", order_id: String(index) } })),
total: 101,
total_is_exact: false,
next_cursor: "cursor-1",
})
.mockResolvedValueOnce({
documents: Array.from({ length: 100 }, (_, index) => ({ _id: { tenant_id: "tenant-a", order_id: String(index + 100) } })),
total: 101,
total_is_exact: false,
next_cursor: "cursor-2",
});
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
expect(await dataGrid.countTotalRows!()).toBe(250);
await flushUi();
expect(root!.textContent).toContain("250 Items");
await dataGrid.paginate!(100, 100);
await flushUi();
expect(backend.documentFindDocuments.mock.calls.at(-1)?.[10]).toBe("cursor-1");
expect(root!.textContent).toContain("250 Items");
});
it("exports all DynamoDB pages with opaque cursors instead of offsets", async () => {
app?.unmount();
settings.editorSettings.exportBatchSize = 2;
backend.documentFindDocuments.mockReset();
backend.documentFindDocuments
.mockResolvedValueOnce({
documents: [{ _id: { tenant_id: "tenant-a", order_id: "visible" }, tenant_id: "tenant-a", order_id: "visible" }],
total: 2,
total_is_exact: false,
next_cursor: "visible-cursor",
})
.mockResolvedValueOnce({
documents: [
{ _id: { tenant_id: "tenant-a", order_id: "001" }, tenant_id: "tenant-a", order_id: "001", amount: { $dbxDynamoDb: { version: 1, type: "number", value: "9007199254740993" } } },
{ _id: { tenant_id: "tenant-a", order_id: "002" }, tenant_id: "tenant-a", order_id: "002", tags: { $dbxDynamoDb: { version: 1, type: "stringSet", value: ["new", "paid"] } } },
],
total: 3,
total_is_exact: false,
next_cursor: "cursor-1",
})
.mockResolvedValueOnce({
documents: [{ _id: { tenant_id: "tenant-b", order_id: "003" }, tenant_id: "tenant-b", order_id: "003" }],
total: 1,
total_is_exact: true,
});
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
expect(dataGrid.fullExportResult).toBeTypeOf("function");
const progress = vi.fn();
const result = await dataGrid.fullExportResult!(progress);
expect(backend.documentFindDocuments.mock.calls.slice(1).map((call) => [call[3], call[4], call[10]])).toEqual([
[0, 2, undefined],
[0, 2, "cursor-1"],
]);
expect(result?.rows).toHaveLength(3);
expect(result?.rows[0]).toContain('{"$dbxDynamoDb":{"version":1,"type":"number","value":"9007199254740993"}}');
expect(result?.rows[1]).toContain('{"$dbxDynamoDb":{"version":1,"type":"stringSet","value":["new","paid"]}}');
expect(progress).toHaveBeenLastCalledWith({ rowsExported: 3, totalRows: 3 });
});
it("caps an otherwise unlimited DynamoDB export at 10,000 items", async () => {
app?.unmount();
settings.editorSettings.exportBatchSize = 1000;
backend.documentFindDocuments.mockReset();
backend.documentFindDocuments.mockResolvedValueOnce({
documents: [{ _id: { tenant_id: "visible", order_id: "visible" } }],
total: 2,
total_is_exact: false,
next_cursor: "visible-cursor",
});
for (let pageIndex = 0; pageIndex < 10; pageIndex += 1) {
backend.documentFindDocuments.mockResolvedValueOnce({
documents: Array.from({ length: 1000 }, (_, rowIndex) => ({
_id: { tenant_id: `tenant-${pageIndex}`, order_id: String(rowIndex) },
tenant_id: `tenant-${pageIndex}`,
order_id: String(rowIndex),
})),
total: 1001,
total_is_exact: false,
next_cursor: `cursor-${pageIndex + 1}`,
});
}
app = createApp(DocumentBrowser, {
connectionId: "dynamodb-1",
database: "us-east-1",
collection: "orders",
databaseType: "dynamodb",
});
app.mount(root!);
await flushUi();
const progress = vi.fn();
const result = await dataGrid.fullExportResult!(progress);
expect(result?.rows).toHaveLength(10_000);
expect(result?.truncated).toBe(true);
expect(result?.has_more).toBe(true);
expect(backend.documentFindDocuments).toHaveBeenCalledTimes(11);
expect(progress).toHaveBeenLastCalledWith({ rowsExported: 10_000, totalRows: null });
});
it("exports all matching MongoDB documents without changing the visible page", async () => {
app?.unmount();
settings.editorSettings.exportBatchSize = 2;