fix(cassandra): use cursor pagination for table data and exports
Closes #7465
This commit is contained in:
1 parent
d452346082
commit
4190bab5ad
17 files changed
+1099
-88
No files matched your search
@@ -62,7 +62,7 @@ import { redisCommandResultToQueryResult } from "@/lib/redis/redisQueryResult";
|
||||
import { nextRedisCommandDb } from "@/lib/redis/redisCommandSession";
|
||||
import { isRedisMutatingCommand } from "@/lib/redis/redisCommandTable";
|
||||
import { formatRedisConsoleValue } from "@/lib/redis/redisValuePresentation";
|
||||
import { usesAgentCursorForQuery } from "@/lib/database/databaseDriverManifest";
|
||||
import { usesAgentCursorForQuery, usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest";
|
||||
import { connectionIsDorisFamilyCatalogCapable, defaultAutoCommitForDbType, supportsClearableQuerySchema, supportsTransaction, usesOracleStickyTransactionState, usesProvenReadOnlyStickyTransactionState } from "@/lib/database/databaseFeatureSupport";
|
||||
import { canInsertTableRows, canUseKeylessRowPredicate, DBX_ROWID_COLUMN, editablePrimaryKeys, shouldIncludeSyntheticRowId, usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
|
||||
import { TABLE_DATA_EXPORT_PAGE_SIZE } from "@/lib/table/tableDataExport";
|
||||
@@ -6732,6 +6732,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
const traceId = executionId.slice(0, 8);
|
||||
const startedAt = performance.now();
|
||||
const executionTargetEpoch = manualTransactionTargetEpoch(tab);
|
||||
const cancelRequestCountAtStart = tab.cancelRequestCount ?? 0;
|
||||
const elapsed = () => `${Math.round(performance.now() - startedAt)}ms`;
|
||||
const batchResume = options?.batchResume;
|
||||
const continueOnBatchError = batchResume?.continueOnError ?? settingsStore.editorSettings.continueOnErrorOnBatch;
|
||||
@@ -7602,7 +7603,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
const pagination = limitQueryPagination(requestedPagination, queryResultMaxRows);
|
||||
pageLimit = pagination.limit;
|
||||
pageOffset = pagination.offset;
|
||||
useAgentResultSession = conn?.db_type === "sqlserver" && conn?.driver_profile?.trim().toLowerCase() === "sqlserver-legacy";
|
||||
useAgentResultSession = usesAgentCursorForTableData(conn?.db_type, conn?.driver_profile);
|
||||
}
|
||||
|
||||
errorLocateContext = {
|
||||
@@ -7670,7 +7671,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
});
|
||||
executionDispatched = true;
|
||||
if (tab.mode === "query") clientRequestStartedAt = performance.now();
|
||||
if (useAgentResultSession && tab.mode === "query" && typeof pageOffset === "number" && pageOffset > 0 && !options?.pagination?.sessionId && !(tab.batchSqlExecution && tab.batchSqlExecution.total > 1)) {
|
||||
if (useAgentResultSession && (tab.mode === "query" || tab.mode === "data") && typeof pageOffset === "number" && pageOffset > 0 && !options?.pagination?.sessionId && !(tab.batchSqlExecution && tab.batchSqlExecution.total > 1)) {
|
||||
return (async () => {
|
||||
let sessionId: string | undefined;
|
||||
let skipped = 0;
|
||||
@@ -7678,35 +7679,50 @@ export const useQueryStore = defineStore("query", () => {
|
||||
let executionMs = 0;
|
||||
let completeTimings = true;
|
||||
const timings: Record<string, number> = {};
|
||||
while (true) {
|
||||
const pageResults = await api.executeMulti(executionConnectionId, executionDatabase, sqlToExecute, executionSchema, executionId, {
|
||||
...executionOptions,
|
||||
resultSessionId: sessionId,
|
||||
});
|
||||
const page = pageResults[0];
|
||||
if (!page) return pageResults;
|
||||
pageCount += 1;
|
||||
executionMs += page.execution_time_ms;
|
||||
if (!page.query_timings_ms) completeTimings = false;
|
||||
else for (const [key, value] of Object.entries(page.query_timings_ms)) timings[key] = (timings[key] ?? 0) + value;
|
||||
// Offset jumps consume several cursor pages before publication.
|
||||
// Keep their timings as well, without retaining skipped rows.
|
||||
const timingSummary = { query_timings_ms: completeTimings ? { ...timings } : undefined, execution_time_ms: executionMs, timing_page_count: pageCount };
|
||||
if (skipped + page.rows.length > pageOffset) {
|
||||
const start = pageOffset - skipped;
|
||||
const limit = typeof pageLimit === "number" ? pageLimit : page.rows.length - start;
|
||||
return [{ ...page, ...timingSummary, rows: page.rows.slice(start, start + limit) }];
|
||||
}
|
||||
skipped += page.rows.length;
|
||||
if (!page.has_more || !page.session_id) {
|
||||
if (page.has_more) {
|
||||
// The cursor session ended before the requested offset;
|
||||
// returning the short page would show the wrong rows.
|
||||
throw new Error("Result session ended before the requested page offset");
|
||||
try {
|
||||
while (true) {
|
||||
const pageResults = await api.executeMulti(executionConnectionId, executionDatabase, sqlToExecute, executionSchema, executionId, {
|
||||
...executionOptions,
|
||||
resultSessionId: sessionId,
|
||||
});
|
||||
const page = pageResults[0];
|
||||
if (!page) return pageResults;
|
||||
pageCount += 1;
|
||||
executionMs += page.execution_time_ms;
|
||||
if (!page.query_timings_ms) completeTimings = false;
|
||||
else for (const [key, value] of Object.entries(page.query_timings_ms)) timings[key] = (timings[key] ?? 0) + value;
|
||||
// Offset jumps consume several cursor pages before publication.
|
||||
// Keep their timings as well, without retaining skipped rows.
|
||||
const timingSummary = { query_timings_ms: completeTimings ? { ...timings } : undefined, execution_time_ms: executionMs, timing_page_count: pageCount };
|
||||
if (skipped + page.rows.length > pageOffset) {
|
||||
const start = pageOffset - skipped;
|
||||
const limit = typeof pageLimit === "number" ? pageLimit : page.rows.length - start;
|
||||
return [{ ...page, ...timingSummary, rows: page.rows.slice(start, start + limit) }];
|
||||
}
|
||||
skipped += page.rows.length;
|
||||
if (!page.has_more || !page.session_id) {
|
||||
if (page.has_more) {
|
||||
// The cursor session ended before the requested offset;
|
||||
// returning the short page would show the wrong rows.
|
||||
throw new Error("Result session ended before the requested page offset");
|
||||
}
|
||||
return [{ ...page, ...timingSummary, rows: [] }];
|
||||
}
|
||||
sessionId = page.session_id;
|
||||
const replayTab = findExecutionTab(id);
|
||||
if (replayTab !== tab || replayTab.executionId !== executionId || replayTab.isCancelling || (replayTab.cancelRequestCount ?? 0) !== cancelRequestCountAtStart || manualTransactionTargetEpoch(replayTab) !== executionTargetEpoch) {
|
||||
throw new Error("Query canceled");
|
||||
}
|
||||
return [{ ...page, ...timingSummary, rows: [] }];
|
||||
}
|
||||
sessionId = page.session_id;
|
||||
} catch (error) {
|
||||
if (sessionId) {
|
||||
try {
|
||||
await api.closeQuerySession(executionConnectionId, executionDatabase, sessionId, executionClientSessionId, executionCatalog);
|
||||
} catch (closeError) {
|
||||
queryExecutionLog("warn", "offset-replay-session-close:error", { traceId, sessionId, error: closeError });
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -8937,7 +8953,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
let offset = 0;
|
||||
const clientSessionId = tabClientSessionId(tab, "export");
|
||||
const exportExecutionId = uuid();
|
||||
const useAgentCursor = conn?.db_type === "sqlserver" && conn?.driver_profile?.trim().toLowerCase() === "sqlserver-legacy";
|
||||
const useAgentCursor = usesAgentCursorForTableData(conn?.db_type, conn?.driver_profile);
|
||||
let sessionId: string | undefined;
|
||||
|
||||
try {
|
||||
@@ -8983,19 +8999,33 @@ export const useQueryStore = defineStore("query", () => {
|
||||
},
|
||||
);
|
||||
const result = results[0];
|
||||
if (!result) break;
|
||||
if (!result) {
|
||||
if (useAgentCursor) throw new Error("Missing cursor result during table export");
|
||||
break;
|
||||
}
|
||||
const nextSessionId = result.session_id?.trim() || undefined;
|
||||
if (useAgentCursor) {
|
||||
sessionId = nextSessionId ?? sessionId;
|
||||
if (conn?.db_type === "cassandra" && (result.truncated || typeof result.has_more !== "boolean")) throw new Error("Incomplete cursor result during table export");
|
||||
if (result.has_more === true && !nextSessionId) throw new Error("Result session ended before table export completed");
|
||||
}
|
||||
if (columns.length === 0) columns = result.columns;
|
||||
rows.push(...result.rows);
|
||||
executionTimeMs += result.execution_time_ms ?? 0;
|
||||
onProgress?.({ rowsExported: rows.length, totalRows });
|
||||
sessionId = result.session_id ?? undefined;
|
||||
sessionId = nextSessionId;
|
||||
const shouldFetchNextPage = useAgentCursor ? result.has_more === true : result.rows.length >= pageLimit;
|
||||
if (!shouldFetchNextPage) break;
|
||||
offset += result.rows.length;
|
||||
}
|
||||
} finally {
|
||||
if (sessionId) void api.closeQuerySession(tab.connectionId, executionDatabase, sessionId, clientSessionId, tableMeta.catalog);
|
||||
void closeClientSessionId(tab.connectionId, executionDatabase, clientSessionId, tableMeta.catalog, { tabId: tab.id });
|
||||
try {
|
||||
if (sessionId) await api.closeQuerySession(tab.connectionId, executionDatabase, sessionId, clientSessionId, tableMeta.catalog);
|
||||
} catch (error) {
|
||||
queryExecutionLog("warn", "table-export-session-close:error", { sessionId, error });
|
||||
} finally {
|
||||
await closeClientSessionId(tab.connectionId, executionDatabase, clientSessionId, tableMeta.catalog, { tabId: tab.id });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user