fix(cassandra): use cursor pagination for table data and exports
Closes #7465
This commit is contained in:
1 parent
d452346082
commit
4190bab5ad
17 files changed
+1099
-88
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { databaseConnectionFormKind, databaseDefaultPort, databaseManifestEntry, databaseRuntimeMode, usesAgentCursorForQuery } from "@/lib/database/databaseDriverManifest";
|
||||
import { databaseConnectionFormKind, databaseDefaultPort, databaseManifestEntry, databaseRuntimeMode, usesAgentCursorForQuery, usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest";
|
||||
|
||||
describe("databaseDriverManifest", () => {
|
||||
it("uses agent cursor only for agent or external runtimes", () => {
|
||||
@@ -10,6 +10,7 @@ describe("databaseDriverManifest", () => {
|
||||
expect(usesAgentCursorForQuery("meilisearch")).toBe(false);
|
||||
expect(usesAgentCursorForQuery("dynamodb")).toBe(false);
|
||||
expect(usesAgentCursorForQuery("mysql")).toBe(false);
|
||||
expect(usesAgentCursorForQuery("cassandra")).toBe(true);
|
||||
expect(usesAgentCursorForQuery("jdbc")).toBe(true);
|
||||
expect(usesAgentCursorForQuery("prestosql")).toBe(true);
|
||||
expect(usesAgentCursorForQuery("sqlserver")).toBe(false);
|
||||
@@ -17,6 +18,14 @@ describe("databaseDriverManifest", () => {
|
||||
expect(usesAgentCursorForQuery("sqlserver", " SQLSERVER-LEGACY ")).toBe(true);
|
||||
});
|
||||
|
||||
it("limits table-data agent cursors to Cassandra and SQL Server legacy", () => {
|
||||
expect(usesAgentCursorForTableData("cassandra")).toBe(true);
|
||||
expect(usesAgentCursorForTableData("sqlserver", "sqlserver-legacy")).toBe(true);
|
||||
expect(usesAgentCursorForTableData("sqlserver")).toBe(false);
|
||||
expect(usesAgentCursorForTableData("jdbc")).toBe(false);
|
||||
expect(usesAgentCursorForTableData("prestosql")).toBe(false);
|
||||
});
|
||||
|
||||
it("exposes connection defaults from the shared manifest", () => {
|
||||
expect(databaseDefaultPort("mysql")).toBe(3306);
|
||||
expect(databaseManifestEntry("h2")).toMatchObject({
|
||||
|
||||
@@ -159,6 +159,10 @@ export function usesAgentCursorForQuery(dbType?: DatabaseType, driverProfile?: s
|
||||
return runtimeMode === "agent" || runtimeMode === "external";
|
||||
}
|
||||
|
||||
export function usesAgentCursorForTableData(dbType?: DatabaseType, driverProfile?: string): boolean {
|
||||
return dbType === "cassandra" || (dbType === "sqlserver" && driverProfile?.trim().toLowerCase() === "sqlserver-legacy");
|
||||
}
|
||||
|
||||
function productCapabilities(overrides: Partial<DatabaseProductCapabilities>): DatabaseProductCapabilities {
|
||||
const capabilities = Object.fromEntries(DATABASE_PRODUCT_CAPABILITY_KEYS.map((key) => [key, false])) as DatabaseProductCapabilities;
|
||||
return { ...capabilities, ...overrides };
|
||||
|
||||
@@ -2,6 +2,14 @@ import type { DatabaseType, QueryResult } from "@/types/database.ts";
|
||||
import { buildTableSelectSql } from "@/lib/table/tableSelectSql.ts";
|
||||
|
||||
export const TABLE_DATA_EXPORT_PAGE_SIZE = 10_000;
|
||||
export const TABLE_DATA_EXPORT_MAX_ROWS = 2_147_483_647;
|
||||
|
||||
export interface TableDataExportCursorPageOptions {
|
||||
maxRows: number;
|
||||
fetchSize: number;
|
||||
pageSize: number;
|
||||
resultSessionId?: string;
|
||||
}
|
||||
|
||||
export interface FetchTableDataForExportOptions {
|
||||
databaseType?: DatabaseType;
|
||||
@@ -11,8 +19,10 @@ export interface FetchTableDataForExportOptions {
|
||||
tableType?: string;
|
||||
columns?: string[];
|
||||
pageSize?: number;
|
||||
useAgentCursor?: boolean;
|
||||
buildPageSql?: (options: { databaseType?: DatabaseType; schema?: string; tableName: string; tableType?: string; columns?: string[]; limit: number; offset: number }) => Promise<string> | string;
|
||||
executePage: (sql: string) => Promise<QueryResult>;
|
||||
executePage: (sql: string, cursorOptions?: TableDataExportCursorPageOptions) => Promise<QueryResult>;
|
||||
closeCursor?: (sessionId?: string) => Promise<void>;
|
||||
}
|
||||
|
||||
export async function fetchTableDataForExport(options: FetchTableDataForExportOptions): Promise<QueryResult> {
|
||||
@@ -33,35 +43,60 @@ export async function fetchTableDataForExport(options: FetchTableDataForExportOp
|
||||
const rows: QueryResult["rows"] = [];
|
||||
let columns: string[] = [];
|
||||
let executionTimeMs = 0;
|
||||
let sessionId: string | undefined;
|
||||
|
||||
while (true) {
|
||||
const sql = await (options.buildPageSql ?? buildTableSelectSql)({
|
||||
databaseType: options.databaseType,
|
||||
identifierQuote: options.identifierQuote,
|
||||
schema: options.schema,
|
||||
tableName: options.tableName,
|
||||
tableType: options.tableType,
|
||||
columns: options.columns,
|
||||
limit: pageSize,
|
||||
offset,
|
||||
});
|
||||
const result = await options.executePage(sql);
|
||||
if (columns.length === 0) columns = result.columns;
|
||||
rows.push(...result.rows);
|
||||
executionTimeMs += result.execution_time_ms ?? 0;
|
||||
try {
|
||||
while (true) {
|
||||
const sql = await (options.buildPageSql ?? buildTableSelectSql)({
|
||||
databaseType: options.databaseType,
|
||||
identifierQuote: options.identifierQuote,
|
||||
schema: options.schema,
|
||||
tableName: options.tableName,
|
||||
tableType: options.tableType,
|
||||
columns: options.columns,
|
||||
limit: pageSize,
|
||||
offset,
|
||||
});
|
||||
const result = await options.executePage(
|
||||
sql,
|
||||
options.useAgentCursor
|
||||
? {
|
||||
maxRows: TABLE_DATA_EXPORT_MAX_ROWS,
|
||||
fetchSize: pageSize,
|
||||
pageSize,
|
||||
resultSessionId: sessionId,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
const nextSessionId = result.session_id?.trim() || undefined;
|
||||
if (options.useAgentCursor) {
|
||||
sessionId = nextSessionId ?? sessionId;
|
||||
if (result.truncated || typeof result.has_more !== "boolean") throw new Error("Incomplete cursor result during table export");
|
||||
if (result.has_more && !nextSessionId) throw new Error("Result session ended before table export completed");
|
||||
}
|
||||
if (columns.length === 0) columns = result.columns;
|
||||
rows.push(...result.rows);
|
||||
executionTimeMs += result.execution_time_ms ?? 0;
|
||||
sessionId = nextSessionId;
|
||||
|
||||
if (result.rows.length < pageSize) {
|
||||
return {
|
||||
columns,
|
||||
rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: executionTimeMs,
|
||||
truncated: false,
|
||||
session_id: undefined,
|
||||
has_more: false,
|
||||
};
|
||||
if (options.useAgentCursor ? result.has_more !== true : result.rows.length < pageSize) {
|
||||
return {
|
||||
columns,
|
||||
rows,
|
||||
affected_rows: 0,
|
||||
execution_time_ms: executionTimeMs,
|
||||
truncated: false,
|
||||
session_id: undefined,
|
||||
has_more: false,
|
||||
};
|
||||
}
|
||||
offset += result.rows.length;
|
||||
}
|
||||
} finally {
|
||||
if (options.useAgentCursor && options.closeCursor) {
|
||||
try {
|
||||
await options.closeCursor(sessionId);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
offset += result.rows.length;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user