feat(grid): allow load all to exceed the per-request row cap
Closes #10752
This commit is contained in:
1 parent
f8b7ffe516
commit
d58a79d6c8
8 files changed
+167
-12
No files matched your search
@@ -222,6 +222,7 @@ import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid
|
||||
import {
|
||||
canFetchNextDataGridSegment,
|
||||
canGoNextDataGridPage,
|
||||
dataGridLoadAllInitialTarget,
|
||||
dataGridLoadAllNextSegment,
|
||||
dataGridLoadAllSegment,
|
||||
dataGridTotalRowCountLabelKey,
|
||||
@@ -3817,14 +3818,17 @@ function loadAllRowsAndGoToLast() {
|
||||
// against those cursors is untested, so ES/Easysearch grids keep the
|
||||
// reveal-only shortcut instead of loading everything.
|
||||
if (isResultsContext.value && (resolvedDatabaseType.value === "elasticsearch" || resolvedDatabaseType.value === "easysearch")) return;
|
||||
const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value);
|
||||
const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
|
||||
const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
|
||||
const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined);
|
||||
const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore);
|
||||
if (!segment) {
|
||||
loadAllRowsActive.value = true;
|
||||
infiniteScrollAllLoaded = true;
|
||||
selectAndRevealLastLoadedRow();
|
||||
return;
|
||||
}
|
||||
const knownTotal = displayedTotalRowCount.value;
|
||||
const knownTotal = effectiveTotal;
|
||||
const remaining = typeof knownTotal === "number" && Number.isFinite(knownTotal) && knownTotal >= props.result.rows.length ? knownTotal - props.result.rows.length : segment.limit;
|
||||
if (remaining > LOAD_ALL_ROWS_CONFIRM_ROW_THRESHOLD) {
|
||||
pendingLoadAllRows.value = { remaining };
|
||||
@@ -3837,6 +3841,7 @@ function loadAllRowsAndGoToLast() {
|
||||
function startLoadAllRows(segment: { offset: number; limit: number }) {
|
||||
loadAllRowsActive.value = true;
|
||||
loadAllRowsLoopActive = true;
|
||||
infiniteScrollAllLoaded = false;
|
||||
infiniteScrollLoadAllPending = true;
|
||||
infiniteScrollLoading.value = true;
|
||||
isInfiniteScrollPaginating.value = true;
|
||||
@@ -3852,12 +3857,14 @@ function startLoadAllRows(segment: { offset: number; limit: number }) {
|
||||
// The per-request result-row cap bounds each chunk, never the run (#10752).
|
||||
function finishOrContinueLoadAllRun(requestedOffset: number | undefined, requestedLimit: number | undefined): boolean {
|
||||
if (!loadAllRowsLoopActive) return false;
|
||||
const nextSegment = canFetchNextInfiniteScrollSegment.value
|
||||
const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
|
||||
const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
|
||||
const nextSegment = canFetchMore
|
||||
? dataGridLoadAllNextSegment({
|
||||
loadedRowCount: props.result.rows.length,
|
||||
requestedOffset: requestedOffset ?? props.result.rows.length,
|
||||
requestedLimit: requestedLimit ?? pageSize.value,
|
||||
totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined,
|
||||
totalRowCount: totalRowCountIsExact.value ? effectiveTotal : undefined,
|
||||
})
|
||||
: null;
|
||||
if (!nextSegment) {
|
||||
@@ -3875,7 +3882,10 @@ function confirmLoadAllRows() {
|
||||
if (!pending) return;
|
||||
pendingLoadAllRows.value = undefined;
|
||||
loadAllRowsConfirmOpen.value = false;
|
||||
const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value);
|
||||
const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
|
||||
const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
|
||||
const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined);
|
||||
const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore);
|
||||
if (segment) startLoadAllRows(segment);
|
||||
}
|
||||
function checkInfiniteScroll(scroller: HTMLElement) {
|
||||
@@ -10740,10 +10750,10 @@ watch(
|
||||
// While a "load all" run is active the per-request row cap must not be
|
||||
// read as the end of data — only the requested-vs-appended count and an
|
||||
// exact known total end the run (#10752).
|
||||
...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined } } : {}),
|
||||
...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? (paginationTotalRowCount.value ?? displayedTotalRowCount.value) : undefined } } : {}),
|
||||
});
|
||||
if (appendCompletion) {
|
||||
if (infiniteScrollEnabled.value) {
|
||||
if (infiniteScrollEnabled.value || loadAllRowsLoopActive) {
|
||||
currentPage.value = appendCompletion.loadedPage;
|
||||
lastInfiniteScrollPage = Math.max(0, appendCompletion.loadedPage - 1);
|
||||
infiniteScrollAllLoaded = appendCompletion.allLoaded;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { computed, reactive } from "vue";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useDataGridActions } from "@/composables/useDataGridActions";
|
||||
import { clearTableMetadataCache } from "@/lib/metadata/tableMetadataCache";
|
||||
import { MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
|
||||
import { restoredDataTabReloadFilters } from "@/lib/table/tableDataRefresh";
|
||||
import type { IndexInfo, QueryTab } from "@/types/database";
|
||||
|
||||
@@ -618,6 +619,37 @@ describe("useDataGridActions", () => {
|
||||
expect(mocks.executeTabSql).toHaveBeenCalledWith("tab-1", expect.any(String), expect.objectContaining({ pagination: { offset: 100, limit: 99_900, sessionId: undefined, clientSessionId: undefined } }));
|
||||
});
|
||||
|
||||
it("allows load-all to append rows past the continuous query result cap", async () => {
|
||||
mocks.queryResultMaxRows = 100_000;
|
||||
const tab = tableDataTab({
|
||||
id: "tab-large",
|
||||
resultTotalRowCount: 260_000,
|
||||
resultPageLimit: 100,
|
||||
resultPageOffset: 0,
|
||||
result: {
|
||||
columns: ["id"],
|
||||
rows: Array.from({ length: 100_000 }, (_, index) => [index + 1]),
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 1,
|
||||
},
|
||||
});
|
||||
mocks.buildTableSelectSql.mockResolvedValueOnce('SELECT * FROM "public"."users" LIMIT 100000 OFFSET 100000;');
|
||||
mocks.tabs.push(tab);
|
||||
const actions = useDataGridActions(computed(() => tab));
|
||||
|
||||
await actions.onPaginate(tab.id, 100_000, 100_000, "", undefined, true);
|
||||
|
||||
expect(mocks.executeTabSql).toHaveBeenCalledWith(
|
||||
"tab-large",
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
appendResult: {
|
||||
maxRows: MAX_QUERY_RESULT_MAX_ROWS,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("continues the matching Cassandra table cursor and client session", async () => {
|
||||
mocks.infiniteScroll = false;
|
||||
mocks.getConfig.mockReturnValue({ id: "cassandra-1", db_type: "cassandra" });
|
||||
|
||||
@@ -21,7 +21,7 @@ import { applyMongoFindSort } from "@/lib/mongo/mongoShellCommand";
|
||||
import { uuid } from "@/lib/common/utils";
|
||||
import { simpleDataGridOrderByReferencesMissingColumn, type DataGridSortMode } from "@/lib/dataGrid/dataGridSort";
|
||||
import type { DataGridReloadIntent } from "@/lib/dataGrid/dataGridToolbar";
|
||||
import { continuousQueryResultMaxRows } from "@/lib/dataGrid/queryResultRowLimit";
|
||||
import { continuousQueryResultMaxRows, MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
|
||||
import { queryResultBaseSql, queryResultExecutionSql } from "@/lib/tabs/tabPresentation";
|
||||
import { sqlExecutionTargetCapabilities } from "@/lib/database/sqlExecutionTargetCapabilities";
|
||||
import { usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest";
|
||||
@@ -503,7 +503,7 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
|
||||
const appendOptions = appendResult
|
||||
? {
|
||||
appendResult: {
|
||||
maxRows: continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows),
|
||||
maxRows: appendRequested ? MAX_QUERY_RESULT_MAX_ROWS : continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows),
|
||||
},
|
||||
}
|
||||
: {};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { dataGridBottomScrollTop, dataGridInfiniteScrollAppendCompletion, didDataGridInfiniteScrollContextChange, isDataGridAtScrollBottom, restoredDataGridScrollLeft } from "@/lib/dataGrid/dataGridInfiniteScroll";
|
||||
import { dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination";
|
||||
import { dataGridLoadAllInitialTarget, dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination";
|
||||
|
||||
describe("data grid bottom anchoring", () => {
|
||||
it("keeps DOM rows anchored when scrollbar padding increases the scroll height", () => {
|
||||
@@ -145,3 +145,22 @@ describe("dataGridLoadAllNextSegment", () => {
|
||||
expect(dataGridLoadAllNextSegment({ loadedRowCount: 200_000, requestedOffset: 200_000, requestedLimit: 100_000 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("dataGridLoadAllInitialTarget", () => {
|
||||
it("targets the cap or total when below the chunk limit", () => {
|
||||
expect(dataGridLoadAllInitialTarget(100, 100_000, 260_000)).toBe(100_000);
|
||||
expect(dataGridLoadAllInitialTarget(100, 100_000, 50_000)).toBe(50_000);
|
||||
expect(dataGridLoadAllInitialTarget(100, 100_000)).toBe(100_000);
|
||||
});
|
||||
|
||||
it("targets the next chunk increment when already at or past the chunk limit", () => {
|
||||
expect(dataGridLoadAllInitialTarget(100_000, 100_000, 260_000)).toBe(200_000);
|
||||
expect(dataGridLoadAllInitialTarget(200_000, 100_000, 260_000)).toBe(260_000);
|
||||
expect(dataGridLoadAllInitialTarget(260_000, 100_000, 260_000)).toBe(260_000);
|
||||
});
|
||||
|
||||
it("advances by chunk increment when total is unknown", () => {
|
||||
expect(dataGridLoadAllInitialTarget(100_000, 100_000)).toBe(200_000);
|
||||
expect(dataGridLoadAllInitialTarget(200_000, 100_000)).toBe(300_000);
|
||||
});
|
||||
});
|
||||
@@ -163,6 +163,21 @@ export function dataGridLoadAllSegment(loadedRowCount: number, maxRows: number,
|
||||
return { offset, limit: boundedMaxRows - offset };
|
||||
}
|
||||
|
||||
/**
|
||||
* Target cumulative row count for the initial chunk of an explicit "load all"
|
||||
* run. Each chunk is bounded by `chunkLimit` (the per-request result-row cap,
|
||||
* e.g. 100,000), but an existing dataset that has already reached or exceeded
|
||||
* `chunkLimit` must still be able to start loading subsequent chunks up to
|
||||
* `totalRowCount` (if known) or by another chunk increment (#10752).
|
||||
*/
|
||||
export function dataGridLoadAllInitialTarget(loadedRowCount: number, chunkLimit: number, totalRowCount?: number): number {
|
||||
const loaded = Number.isFinite(loadedRowCount) ? Math.max(0, Math.trunc(loadedRowCount)) : 0;
|
||||
const chunk = Number.isFinite(chunkLimit) ? Math.max(1, Math.trunc(chunkLimit)) : 1;
|
||||
const chunkCap = loaded < chunk ? chunk : loaded + chunk;
|
||||
const total = typeof totalRowCount === "number" && Number.isFinite(totalRowCount) && totalRowCount >= 0 ? Math.trunc(totalRowCount) : undefined;
|
||||
return total !== undefined ? Math.min(chunkCap, total) : chunkCap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Next chunk of an explicit "load all" run, issued after the previous chunk
|
||||
* completed. The per-request result-row cap bounds each request, not the run:
|
||||
|
||||
@@ -7724,7 +7724,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
return producedResult;
|
||||
}
|
||||
|
||||
const queryResultMaxRows = tab.mode === "query" ? effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows) : undefined;
|
||||
const queryResultMaxRows = tab.mode === "query" ? (options?.appendResult ? options.appendResult.maxRows : effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows)) : undefined;
|
||||
|
||||
if (tab.mode === "query") {
|
||||
const prepared = await prepareEditableQueryExecution(tab, sqlToExecute, conn, effectiveDbType, executionDatabase, traceId, elapsed);
|
||||
|
||||
Reference in new issue
Block a user