fix(cassandra): use cursor pagination for table data and exports

Closes #7465
This commit is contained in:
t8y2 committed 2026-09-28 05:59:12 +00:00
1 parent d452346082
commit 4190bab5ad
17 files changed
+1099 -88

No files matched your search

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { databaseConnectionFormKind, databaseDefaultPort, databaseManifestEntry, databaseRuntimeMode, usesAgentCursorForQuery } from "@/lib/database/databaseDriverManifest";
import { databaseConnectionFormKind, databaseDefaultPort, databaseManifestEntry, databaseRuntimeMode, usesAgentCursorForQuery, usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest";
describe("databaseDriverManifest", () => {
it("uses agent cursor only for agent or external runtimes", () => {
@@ -10,6 +10,7 @@ describe("databaseDriverManifest", () => {
expect(usesAgentCursorForQuery("meilisearch")).toBe(false);
expect(usesAgentCursorForQuery("dynamodb")).toBe(false);
expect(usesAgentCursorForQuery("mysql")).toBe(false);
expect(usesAgentCursorForQuery("cassandra")).toBe(true);
expect(usesAgentCursorForQuery("jdbc")).toBe(true);
expect(usesAgentCursorForQuery("prestosql")).toBe(true);
expect(usesAgentCursorForQuery("sqlserver")).toBe(false);
@@ -17,6 +18,14 @@ describe("databaseDriverManifest", () => {
expect(usesAgentCursorForQuery("sqlserver", " SQLSERVER-LEGACY ")).toBe(true);
});
it("limits table-data agent cursors to Cassandra and SQL Server legacy", () => {
expect(usesAgentCursorForTableData("cassandra")).toBe(true);
expect(usesAgentCursorForTableData("sqlserver", "sqlserver-legacy")).toBe(true);
expect(usesAgentCursorForTableData("sqlserver")).toBe(false);
expect(usesAgentCursorForTableData("jdbc")).toBe(false);
expect(usesAgentCursorForTableData("prestosql")).toBe(false);
});
it("exposes connection defaults from the shared manifest", () => {
expect(databaseDefaultPort("mysql")).toBe(3306);
expect(databaseManifestEntry("h2")).toMatchObject({
@@ -159,6 +159,10 @@ export function usesAgentCursorForQuery(dbType?: DatabaseType, driverProfile?: s
return runtimeMode === "agent" || runtimeMode === "external";
}
export function usesAgentCursorForTableData(dbType?: DatabaseType, driverProfile?: string): boolean {
return dbType === "cassandra" || (dbType === "sqlserver" && driverProfile?.trim().toLowerCase() === "sqlserver-legacy");
}
function productCapabilities(overrides: Partial<DatabaseProductCapabilities>): DatabaseProductCapabilities {
const capabilities = Object.fromEntries(DATABASE_PRODUCT_CAPABILITY_KEYS.map((key) => [key, false])) as DatabaseProductCapabilities;
return { ...capabilities, ...overrides };
+63 -28
View File
@@ -2,6 +2,14 @@ import type { DatabaseType, QueryResult } from "@/types/database.ts";
import { buildTableSelectSql } from "@/lib/table/tableSelectSql.ts";
export const TABLE_DATA_EXPORT_PAGE_SIZE = 10_000;
export const TABLE_DATA_EXPORT_MAX_ROWS = 2_147_483_647;
export interface TableDataExportCursorPageOptions {
maxRows: number;
fetchSize: number;
pageSize: number;
resultSessionId?: string;
}
export interface FetchTableDataForExportOptions {
databaseType?: DatabaseType;
@@ -11,8 +19,10 @@ export interface FetchTableDataForExportOptions {
tableType?: string;
columns?: string[];
pageSize?: number;
useAgentCursor?: boolean;
buildPageSql?: (options: { databaseType?: DatabaseType; schema?: string; tableName: string; tableType?: string; columns?: string[]; limit: number; offset: number }) => Promise<string> | string;
executePage: (sql: string) => Promise<QueryResult>;
executePage: (sql: string, cursorOptions?: TableDataExportCursorPageOptions) => Promise<QueryResult>;
closeCursor?: (sessionId?: string) => Promise<void>;
}
export async function fetchTableDataForExport(options: FetchTableDataForExportOptions): Promise<QueryResult> {
@@ -33,35 +43,60 @@ export async function fetchTableDataForExport(options: FetchTableDataForExportOp
const rows: QueryResult["rows"] = [];
let columns: string[] = [];
let executionTimeMs = 0;
let sessionId: string | undefined;
while (true) {
const sql = await (options.buildPageSql ?? buildTableSelectSql)({
databaseType: options.databaseType,
identifierQuote: options.identifierQuote,
schema: options.schema,
tableName: options.tableName,
tableType: options.tableType,
columns: options.columns,
limit: pageSize,
offset,
});
const result = await options.executePage(sql);
if (columns.length === 0) columns = result.columns;
rows.push(...result.rows);
executionTimeMs += result.execution_time_ms ?? 0;
try {
while (true) {
const sql = await (options.buildPageSql ?? buildTableSelectSql)({
databaseType: options.databaseType,
identifierQuote: options.identifierQuote,
schema: options.schema,
tableName: options.tableName,
tableType: options.tableType,
columns: options.columns,
limit: pageSize,
offset,
});
const result = await options.executePage(
sql,
options.useAgentCursor
? {
maxRows: TABLE_DATA_EXPORT_MAX_ROWS,
fetchSize: pageSize,
pageSize,
resultSessionId: sessionId,
}
: undefined,
);
const nextSessionId = result.session_id?.trim() || undefined;
if (options.useAgentCursor) {
sessionId = nextSessionId ?? sessionId;
if (result.truncated || typeof result.has_more !== "boolean") throw new Error("Incomplete cursor result during table export");
if (result.has_more && !nextSessionId) throw new Error("Result session ended before table export completed");
}
if (columns.length === 0) columns = result.columns;
rows.push(...result.rows);
executionTimeMs += result.execution_time_ms ?? 0;
sessionId = nextSessionId;
if (result.rows.length < pageSize) {
return {
columns,
rows,
affected_rows: 0,
execution_time_ms: executionTimeMs,
truncated: false,
session_id: undefined,
has_more: false,
};
if (options.useAgentCursor ? result.has_more !== true : result.rows.length < pageSize) {
return {
columns,
rows,
affected_rows: 0,
execution_time_ms: executionTimeMs,
truncated: false,
session_id: undefined,
has_more: false,
};
}
offset += result.rows.length;
}
} finally {
if (options.useAgentCursor && options.closeCursor) {
try {
await options.closeCursor(sessionId);
} catch {}
}
offset += result.rows.length;
}
}