feat(es): paginate past 10k documents via pit and search_after
This commit is contained in:
1 parent
1d71a5fc23
commit
73a7b3f2c7
35 files changed
+1622
-153
No files matched your search
@@ -22,7 +22,7 @@ import type { DynamoDbIndexInfo, DynamoDbTableDescription } from "@/lib/backend/
|
||||
import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import { clampSearchSplitWidth } from "@/lib/dataGrid/dataGridSearchSplit";
|
||||
import { documentViewerFontStyle } from "@/lib/document/documentViewerFontStyle";
|
||||
import { clampDocumentPage, clampElasticsearchRequestLimit, documentPageRequestLimit, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
|
||||
import { ELASTICSEARCH_DEFAULT_MAX_RESULT_WINDOW, clampDocumentPage, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
|
||||
import { canGoNextDocumentPage, isSameDocumentQueryTotalCountRequest, resolveDocumentQueryTotals, type DocumentQueryTotalCountRequest } from "@/lib/document/documentQueryTotals";
|
||||
import {
|
||||
arrayObjectAncestorPathForDocumentField,
|
||||
@@ -238,6 +238,7 @@ const pageTotal = computed(() => paginationTotal.value);
|
||||
const documentPageCount = computed(() => (pageTotal.value === undefined ? undefined : Math.max(1, Math.ceil(pageTotal.value / pageSize.value))));
|
||||
const canGoNextPage = computed(() => {
|
||||
if (documentStoreProvider.value.kind === "dynamodb") return dynamodbHasNextCursor.value;
|
||||
if (documentStoreProvider.value.kind === "elasticsearch") return elasticsearchHasNextCursor.value;
|
||||
return canGoNextDocumentPage({
|
||||
page: page.value,
|
||||
pageSize: pageSize.value,
|
||||
@@ -247,7 +248,7 @@ const canGoNextPage = computed(() => {
|
||||
});
|
||||
const documentRequestLimit = computed(() => {
|
||||
if (documentStoreProvider.value.kind !== "elasticsearch") return pageSize.value;
|
||||
return documentPageRequestLimit(page.value, pageSize.value, paginationTotal.value);
|
||||
return Math.min(pageSize.value, ELASTICSEARCH_DEFAULT_MAX_RESULT_WINDOW);
|
||||
});
|
||||
|
||||
const tableFindPaneStyle = computed(() => {
|
||||
@@ -289,6 +290,8 @@ const dynamodbIndexName = ref("__table__");
|
||||
const dynamodbPageCursors = ref<Array<string | undefined>>([undefined]);
|
||||
const dynamodbHasNextCursor = ref(false);
|
||||
const dynamodbExactTotal = ref<number | undefined>();
|
||||
const elasticsearchPageCursors = ref<Array<string | undefined>>([undefined]);
|
||||
const elasticsearchHasNextCursor = ref(false);
|
||||
let dynamodbExactCountKey: string | null = null;
|
||||
|
||||
const dynamodbIndexOptions = computed<Array<{ value: string; label: string; index?: DynamoDbIndexInfo }>>(() => [
|
||||
@@ -440,7 +443,7 @@ const gridResult = computed<QueryResult>(() => {
|
||||
|
||||
async function exportAllDocumentStoreDocuments(onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void): Promise<QueryResult | undefined> {
|
||||
const kind = documentStoreProvider.value.kind;
|
||||
if (kind !== "mongodb" && kind !== "dynamodb") return undefined;
|
||||
if (kind !== "mongodb" && kind !== "dynamodb" && kind !== "elasticsearch") return undefined;
|
||||
|
||||
const connectionId = props.connectionId;
|
||||
const database = props.database;
|
||||
@@ -456,44 +459,53 @@ async function exportAllDocumentStoreDocuments(onProgress?: (info: { rowsExporte
|
||||
let exportedCopyDocuments: JsonRecord[] | undefined = kind === "mongodb" ? [] : undefined;
|
||||
let totalRows: number | null = null;
|
||||
let cursor: string | undefined;
|
||||
let lastCursor: string | undefined;
|
||||
const seenCursors = new Set<string>();
|
||||
|
||||
while (exportedDocuments.length < rowLimit) {
|
||||
const requestLimit = Math.min(batchSize, kind === "dynamodb" ? 1000 : Number.POSITIVE_INFINITY, rowLimit - exportedDocuments.length);
|
||||
if (requestLimit <= 0) break;
|
||||
const result = await api.documentFindDocuments(connectionId, database, collection, kind === "dynamodb" ? 0 : exportedDocuments.length, requestLimit, filter, undefined, sort, undefined, exportExecutionId, cursor);
|
||||
const pageDocuments = result.documents.slice(0, requestLimit).map(asRecord);
|
||||
exportedDocuments.push(...pageDocuments);
|
||||
try {
|
||||
while (exportedDocuments.length < rowLimit) {
|
||||
const requestLimit = Math.min(batchSize, kind === "dynamodb" ? 1000 : kind === "elasticsearch" ? ELASTICSEARCH_DEFAULT_MAX_RESULT_WINDOW : Number.POSITIVE_INFINITY, rowLimit - exportedDocuments.length);
|
||||
if (requestLimit <= 0) break;
|
||||
if (cursor) lastCursor = cursor;
|
||||
const result = await api.documentFindDocuments(connectionId, database, collection, kind === "dynamodb" || kind === "elasticsearch" ? 0 : exportedDocuments.length, requestLimit, filter, undefined, sort, undefined, exportExecutionId, cursor, kind === "elasticsearch");
|
||||
const pageDocuments = result.documents.slice(0, requestLimit).map(asRecord);
|
||||
exportedDocuments.push(...pageDocuments);
|
||||
|
||||
if (kind === "mongodb" && exportedCopyDocuments) {
|
||||
if (result.extended_documents?.length === result.documents.length) {
|
||||
exportedCopyDocuments.push(...result.extended_documents.slice(0, pageDocuments.length).map(asRecord));
|
||||
} else {
|
||||
exportedCopyDocuments = undefined;
|
||||
if (kind === "mongodb" && exportedCopyDocuments) {
|
||||
if (result.extended_documents?.length === result.documents.length) {
|
||||
exportedCopyDocuments.push(...result.extended_documents.slice(0, pageDocuments.length).map(asRecord));
|
||||
} else {
|
||||
exportedCopyDocuments = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if ((kind === "mongodb" || kind === "elasticsearch") && result.total_is_exact !== false) totalRows = Math.min(result.total, rowLimit);
|
||||
onProgress?.({ rowsExported: exportedDocuments.length, totalRows });
|
||||
|
||||
if (kind === "dynamodb" || kind === "elasticsearch") {
|
||||
cursor = result.next_cursor;
|
||||
if (!cursor) break;
|
||||
if (seenCursors.has(cursor)) throw new Error(kind === "dynamodb" ? t("dynamodb.repeatedCursor") : "Elasticsearch cursor repeated during export");
|
||||
seenCursors.add(cursor);
|
||||
continue;
|
||||
}
|
||||
|
||||
const reachedExactTotal = result.total_is_exact !== false && exportedDocuments.length >= result.total;
|
||||
if (pageDocuments.length === 0 || pageDocuments.length < requestLimit || reachedExactTotal) break;
|
||||
}
|
||||
|
||||
if (kind === "mongodb" && result.total_is_exact !== false) totalRows = Math.min(result.total, rowLimit);
|
||||
onProgress?.({ rowsExported: exportedDocuments.length, totalRows });
|
||||
|
||||
if (kind === "dynamodb") {
|
||||
cursor = result.next_cursor;
|
||||
if (!cursor) break;
|
||||
if (seenCursors.has(cursor)) throw new Error(t("dynamodb.repeatedCursor"));
|
||||
seenCursors.add(cursor);
|
||||
continue;
|
||||
} finally {
|
||||
if (kind === "elasticsearch") {
|
||||
const cursorToClose = lastCursor ?? cursor;
|
||||
if (cursorToClose) void closeElasticsearchCursor(cursorToClose);
|
||||
}
|
||||
|
||||
const reachedExactTotal = result.total_is_exact !== false && exportedDocuments.length >= result.total;
|
||||
if (pageDocuments.length === 0 || pageDocuments.length < requestLimit || reachedExactTotal) break;
|
||||
}
|
||||
|
||||
if (kind === "dynamodb") {
|
||||
if (kind === "dynamodb" || kind === "elasticsearch") {
|
||||
const truncatedByLimit = !!cursor && exportedDocuments.length >= rowLimit;
|
||||
totalRows = truncatedByLimit ? null : exportedDocuments.length;
|
||||
onProgress?.({ rowsExported: exportedDocuments.length, totalRows });
|
||||
if (truncatedByLimit) {
|
||||
toast(t("dynamodb.exportLimitReached", { count: rowLimit }), 6000);
|
||||
toast(kind === "dynamodb" ? t("dynamodb.exportLimitReached", { count: rowLimit }) : `Elasticsearch export limit reached (${rowLimit})`, 6000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -501,7 +513,7 @@ async function exportAllDocumentStoreDocuments(onProgress?: (info: { rowsExporte
|
||||
if (result.columns.length === 0) result.columns = gridResult.value.columns;
|
||||
result.column_types = kind === "mongodb" ? mongoDocumentGridColumnTypes(exportedDocuments, result.columns) : undefined;
|
||||
result.affected_rows = exportedDocuments.length;
|
||||
result.truncated = kind === "dynamodb" && !!cursor && exportedDocuments.length >= rowLimit;
|
||||
result.truncated = (kind === "dynamodb" || kind === "elasticsearch") && !!cursor && exportedDocuments.length >= rowLimit;
|
||||
result.has_more = result.truncated;
|
||||
return result;
|
||||
}
|
||||
@@ -744,6 +756,22 @@ function resetDynamoDbPagination(options: { preserveExactCount?: boolean } = {})
|
||||
if (!options.preserveExactCount) resetDynamoDbExactCount();
|
||||
}
|
||||
|
||||
async function closeElasticsearchCursor(cursor?: string) {
|
||||
if (!cursor) return;
|
||||
try {
|
||||
await api.closeQuerySession(props.connectionId, props.database, cursor);
|
||||
} catch (error) {
|
||||
console.warn("[DBX] failed to close Elasticsearch cursor", error);
|
||||
}
|
||||
}
|
||||
|
||||
function resetElasticsearchPagination() {
|
||||
const cursor = [...elasticsearchPageCursors.value].reverse().find((candidate): candidate is string => !!candidate);
|
||||
if (cursor) void closeElasticsearchCursor(cursor);
|
||||
elasticsearchPageCursors.value = [undefined];
|
||||
elasticsearchHasNextCursor.value = false;
|
||||
}
|
||||
|
||||
function currentDocumentFilter(): string | undefined {
|
||||
const filter = currentDocumentFilterJson(filterInput.value, appliedDocumentFilter.value, documentStoreProvider.value.kind);
|
||||
if (documentStoreProvider.value.kind !== "dynamodb" || dynamodbIndexName.value === "__table__") return filter;
|
||||
@@ -808,7 +836,7 @@ const documentQueryPreview = computed(() => {
|
||||
collection: props.collection,
|
||||
filterJson: filter,
|
||||
sortJson: sortInput.value.trim(),
|
||||
skip: page.value * pageSize.value,
|
||||
skip: documentStoreProvider.value.kind === "elasticsearch" ? 0 : page.value * pageSize.value,
|
||||
limit: documentRequestLimit.value,
|
||||
});
|
||||
});
|
||||
@@ -1248,6 +1276,7 @@ function resetElasticsearchTotals(options: { preservePaginationTotal?: boolean }
|
||||
paginationTotal.value = nextTotals.paginationTotal;
|
||||
total.value = nextTotals.total;
|
||||
totalIsExact.value = nextTotals.totalIsExact;
|
||||
resetElasticsearchPagination();
|
||||
}
|
||||
|
||||
function clampPageToPaginationTotal(): number | undefined {
|
||||
@@ -1352,16 +1381,18 @@ async function load(options: { page?: number; append?: boolean; offset?: number;
|
||||
resetElasticsearchTotals();
|
||||
}
|
||||
const sort = currentDocumentSortJson(sortInput.value);
|
||||
const cursor = storeKind === "dynamodb" ? dynamodbPageCursors.value[requestPage] : undefined;
|
||||
if (storeKind === "dynamodb" && requestPage > 0 && !cursor) {
|
||||
throw new Error(t("dynamodb.pageCursorUnavailable"));
|
||||
const cursor = storeKind === "dynamodb" ? dynamodbPageCursors.value[requestPage] : storeKind === "elasticsearch" ? elasticsearchPageCursors.value[requestPage] : undefined;
|
||||
if ((storeKind === "dynamodb" || storeKind === "elasticsearch") && requestPage > 0 && !cursor) {
|
||||
throw new Error(storeKind === "dynamodb" ? t("dynamodb.pageCursorUnavailable") : "Elasticsearch page cursor unavailable; go back to the first page and page forward again");
|
||||
}
|
||||
const skip = storeKind === "dynamodb" ? 0 : (options.offset ?? requestPage * pageSize.value);
|
||||
const requestedLimit = options.limit ?? (storeKind === "elasticsearch" ? documentPageRequestLimit(requestPage, pageSize.value, paginationTotal.value) : pageSize.value);
|
||||
// Elasticsearch rejects from+size beyond index.max_result_window even when an explicit limit
|
||||
// (e.g. from the per-tab page size control) bypassed the default-path clamp above.
|
||||
const requestLimit = storeKind === "elasticsearch" ? clampElasticsearchRequestLimit(skip, requestedLimit) : requestedLimit;
|
||||
const result = await api.documentFindDocuments(connectionId, database, collection, skip, requestLimit, filter, undefined, sort, undefined, executionId, cursor);
|
||||
// Starting a fresh ES first page invalidates any previous PIT cursor stack.
|
||||
if (storeKind === "elasticsearch" && !cursor && requestPage === 0) {
|
||||
resetElasticsearchPagination();
|
||||
}
|
||||
const skip = storeKind === "dynamodb" || storeKind === "elasticsearch" ? 0 : (options.offset ?? requestPage * pageSize.value);
|
||||
const requestedLimit = options.limit ?? pageSize.value;
|
||||
const requestLimit = storeKind === "elasticsearch" ? Math.min(requestedLimit, ELASTICSEARCH_DEFAULT_MAX_RESULT_WINDOW) : requestedLimit;
|
||||
const result = await api.documentFindDocuments(connectionId, database, collection, skip, requestLimit, filter, undefined, sort, undefined, executionId, cursor, storeKind === "elasticsearch");
|
||||
if (documentLoadExecutionId.value !== executionId) return;
|
||||
if (connectionId !== props.connectionId || database !== props.database || collection !== props.collection || storeKind !== documentStoreProvider.value.kind) return;
|
||||
const nextDocuments =
|
||||
@@ -1387,6 +1418,10 @@ async function load(options: { page?: number; append?: boolean; offset?: number;
|
||||
dynamodbHasNextCursor.value = !!result.next_cursor;
|
||||
}
|
||||
if (storeKind === "elasticsearch") {
|
||||
const nextCursors = elasticsearchPageCursors.value.slice(0, requestPage + 1);
|
||||
nextCursors[requestPage + 1] = result.next_cursor ?? undefined;
|
||||
elasticsearchPageCursors.value = nextCursors;
|
||||
elasticsearchHasNextCursor.value = !!result.next_cursor;
|
||||
applyElasticsearchSearchTotal(result.total, result.total_is_exact !== false, filter);
|
||||
} else if (storeKind === "dynamodb") {
|
||||
cancelElasticsearchCount();
|
||||
@@ -1501,28 +1536,34 @@ async function paginate(offset: number, limit: number) {
|
||||
}
|
||||
const pageSizeChanged = normalizedLimit !== pageSize.value;
|
||||
pageSize.value = normalizedLimit;
|
||||
if (documentStoreProvider.value.kind === "dynamodb" && pageSizeChanged) {
|
||||
if (pageSizeChanged && (documentStoreProvider.value.kind === "dynamodb" || documentStoreProvider.value.kind === "elasticsearch")) {
|
||||
page.value = 0;
|
||||
resetDynamoDbPagination({ preserveExactCount: true });
|
||||
if (documentStoreProvider.value.kind === "dynamodb") {
|
||||
resetDynamoDbPagination({ preserveExactCount: true });
|
||||
} else {
|
||||
resetElasticsearchPagination();
|
||||
}
|
||||
await load({ page: 0 });
|
||||
return;
|
||||
}
|
||||
const requestedPage = Math.floor(normalizedOffset / normalizedLimit);
|
||||
const nextPage = clampDocumentPage(requestedPage, normalizedLimit, paginationTotal.value);
|
||||
if (documentStoreProvider.value.kind !== "dynamodb") {
|
||||
if (documentStoreProvider.value.kind !== "dynamodb" && documentStoreProvider.value.kind !== "elasticsearch") {
|
||||
await load({ page: nextPage, offset: nextPage * normalizedLimit, limit: normalizedLimit });
|
||||
return;
|
||||
}
|
||||
for (let cursorPage = 0; cursorPage <= nextPage; cursorPage += 1) {
|
||||
if (cursorPage > 0 && !dynamodbPageCursors.value[cursorPage]) {
|
||||
error.value = t("dynamodb.pageCursorUnavailable");
|
||||
const pageCursor = documentStoreProvider.value.kind === "dynamodb" ? dynamodbPageCursors.value[cursorPage] : elasticsearchPageCursors.value[cursorPage];
|
||||
if (cursorPage > 0 && !pageCursor) {
|
||||
error.value = documentStoreProvider.value.kind === "dynamodb" ? t("dynamodb.pageCursorUnavailable") : "Elasticsearch page cursor unavailable; go back to the first page and page forward again";
|
||||
return;
|
||||
}
|
||||
if (cursorPage === nextPage) {
|
||||
await load({ page: cursorPage });
|
||||
return;
|
||||
}
|
||||
if (!dynamodbPageCursors.value[cursorPage + 1]) {
|
||||
const nextCursor = documentStoreProvider.value.kind === "dynamodb" ? dynamodbPageCursors.value[cursorPage + 1] : elasticsearchPageCursors.value[cursorPage + 1];
|
||||
if (!nextCursor) {
|
||||
await load({ page: cursorPage });
|
||||
}
|
||||
}
|
||||
@@ -2041,6 +2082,7 @@ onBeforeUnmount(() => {
|
||||
documentRequestGeneration++;
|
||||
loadedDocumentQueryTotalCountRequest = undefined;
|
||||
cancelElasticsearchCount();
|
||||
resetElasticsearchPagination();
|
||||
stopDocumentLoadingTimer();
|
||||
endTableSearchSplitResize();
|
||||
});
|
||||
@@ -2317,7 +2359,7 @@ defineExpose({ focusSearch });
|
||||
:inexact-total-row-count-mode="documentStoreProvider.kind === 'mongodb' ? 'estimated' : 'at-least'"
|
||||
:pagination-total-row-count="pageTotal"
|
||||
:count-total-rows="countExactDocumentTotal"
|
||||
:full-export-result="documentStoreProvider.kind === 'mongodb' || documentStoreProvider.kind === 'dynamodb' ? exportAllDocumentStoreDocuments : undefined"
|
||||
:full-export-result="documentStoreProvider.kind === 'mongodb' || documentStoreProvider.kind === 'dynamodb' || documentStoreProvider.kind === 'elasticsearch' ? exportAllDocumentStoreDocuments : undefined"
|
||||
@sort="onSort"
|
||||
@reload="refreshDocuments"
|
||||
@paginate="(offset: number, limit: number) => paginate(offset, limit)"
|
||||
|
||||
+36
-2
@@ -14,6 +14,7 @@ const backend = vi.hoisted(() => ({
|
||||
documentUpdateDocument: vi.fn(),
|
||||
documentDeleteDocument: vi.fn(),
|
||||
documentSaveMeilisearchBatch: vi.fn(),
|
||||
closeQuerySession: vi.fn(),
|
||||
}));
|
||||
|
||||
const documentJsonEditor = vi.hoisted(() => ({
|
||||
@@ -79,6 +80,7 @@ vi.mock("@/lib/backend/api", () => ({
|
||||
documentUpdateDocument: backend.documentUpdateDocument,
|
||||
documentDeleteDocument: backend.documentDeleteDocument,
|
||||
documentSaveMeilisearchBatch: backend.documentSaveMeilisearchBatch,
|
||||
closeQuerySession: backend.closeQuerySession,
|
||||
}));
|
||||
|
||||
vi.mock("@/stores/connectionStore", () => ({
|
||||
@@ -315,6 +317,7 @@ beforeEach(async () => {
|
||||
backend.documentUpdateDocument.mockReset();
|
||||
backend.documentDeleteDocument.mockReset();
|
||||
backend.documentSaveMeilisearchBatch.mockReset();
|
||||
backend.closeQuerySession.mockReset();
|
||||
dataGrid.fullExportResult = undefined;
|
||||
dataGrid.customSaveHandler = undefined;
|
||||
dataGrid.countTotalRows = undefined;
|
||||
@@ -889,8 +892,39 @@ describe("DocumentBrowser MongoDB filter value types", () => {
|
||||
expect(progress).toHaveBeenLastCalledWith({ rowsExported: 1, totalRows: null });
|
||||
});
|
||||
|
||||
it("does not offer the MongoDB full export callback to Elasticsearch viewers", () => {
|
||||
expect(dataGrid.fullExportResult).toBeUndefined();
|
||||
it("exports Elasticsearch documents with cursor pagination", async () => {
|
||||
settings.editorSettings.exportBatchSize = 2;
|
||||
backend.documentFindDocuments.mockReset();
|
||||
backend.documentFindDocuments
|
||||
.mockResolvedValueOnce({
|
||||
documents: [
|
||||
{ _id: "one", title: "First" },
|
||||
{ _id: "two", title: "Second" },
|
||||
],
|
||||
total: 3,
|
||||
total_is_exact: true,
|
||||
next_cursor: "cursor-1",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
documents: [{ _id: "three", title: "Third" }],
|
||||
total: 3,
|
||||
total_is_exact: true,
|
||||
});
|
||||
|
||||
expect(dataGrid.fullExportResult).toBeTypeOf("function");
|
||||
const result = await dataGrid.fullExportResult!();
|
||||
await flushUi();
|
||||
|
||||
expect(backend.documentFindDocuments.mock.calls.map((call) => [call[3], call[4], call[10], call[11]])).toEqual([
|
||||
[0, 2, undefined, true],
|
||||
[0, 2, "cursor-1", true],
|
||||
]);
|
||||
expect(result?.rows).toEqual([
|
||||
["one", "First"],
|
||||
["two", "Second"],
|
||||
["three", "Third"],
|
||||
]);
|
||||
expect(backend.closeQuerySession).toHaveBeenCalledWith("connection-1", "", "cursor-1");
|
||||
});
|
||||
|
||||
it("identifies consistently numeric MongoDB columns for shared grid alignment", async () => {
|
||||
|
||||
+4
@@ -260,6 +260,10 @@ describe("DocumentBrowser infinite scroll (issue #6455)", () => {
|
||||
await mountBrowser(databaseType);
|
||||
|
||||
expect(backend.documentFindDocuments.mock.calls[0]!.slice(0, 5)).toEqual(["mongo-1", "test", "docs", 0, 5]);
|
||||
if (databaseType === "elasticsearch") {
|
||||
expect(backend.documentFindDocuments.mock.calls[0]![10]).toBeUndefined();
|
||||
expect(backend.documentFindDocuments.mock.calls[0]![11]).toBe(true);
|
||||
}
|
||||
expect(dataGrid.pageLimit).toBe(5);
|
||||
expect(dataGrid.pageSizePreference).toBe("table-open");
|
||||
});
|
||||
|
||||
@@ -49,6 +49,7 @@ import {
|
||||
Settings2,
|
||||
WandSparkles,
|
||||
Camera,
|
||||
AlertTriangle,
|
||||
} from "@lucide/vue";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
@@ -79,7 +80,7 @@ import TemporalCellEditor from "@/components/grid/TemporalCellEditor.vue";
|
||||
import EnumCellEditor from "@/components/grid/EnumCellEditor.vue";
|
||||
import DataGridReadonlyTextSelection from "@/components/grid/DataGridReadonlyTextSelection.vue";
|
||||
import GridSnapshotDialog from "@/components/grid/GridSnapshotDialog.vue";
|
||||
import type { QueryResult, ColumnInfo, ConstraintInfo, DatabaseType, ForeignKeyInfo, IndexInfo, ReferenceKeyInfo, TriggerInfo, TableInfoTab, QueryResultSourceColumnRef } from "@/types/database";
|
||||
import type { QueryResult, ColumnInfo, ConstraintInfo, DatabaseType, ForeignKeyInfo, IndexInfo, ReferenceKeyInfo, TriggerInfo, TableInfoTab, QueryResultSourceColumnRef, QueryPageJumpProgress } from "@/types/database";
|
||||
import { isQueryExecutionErrorResult } from "@/lib/query/queryResultError";
|
||||
import { tableObjectSourceKind } from "@/lib/table/tableObjectSourceKind";
|
||||
import { tableColumnDefaultDisplayValue } from "@/lib/table/tableColumnDefaultPresentation";
|
||||
@@ -205,7 +206,18 @@ import {
|
||||
isToggleTransposeShortcut,
|
||||
} from "@/lib/editor/keyboardShortcuts";
|
||||
import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter";
|
||||
import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, showDataGridRerunTotalCountAction, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination";
|
||||
import {
|
||||
canFetchNextDataGridSegment,
|
||||
canGoNextDataGridPage,
|
||||
dataGridTotalRowCountLabelKey,
|
||||
dataGridTruncationHintKey,
|
||||
ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS,
|
||||
elasticsearchCursorPageJumpRequestCount,
|
||||
hasCompleteLocalDataGridResult,
|
||||
resolveDataGridPaginationTotal,
|
||||
showDataGridRerunTotalCountAction,
|
||||
type DataGridInexactTotalRowCountMode,
|
||||
} from "@/lib/dataGrid/dataGridPagination";
|
||||
import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions";
|
||||
import {
|
||||
createResultScopedPendingRequests,
|
||||
@@ -481,6 +493,7 @@ interface DataGridProps {
|
||||
paginationTotalRowCount?: number;
|
||||
paginationEnabled?: boolean;
|
||||
totalRowCountLoading?: boolean;
|
||||
pageJumpProgress?: QueryPageJumpProgress;
|
||||
/** Document stores (e.g. MongoDB) count exactly on demand without SQL tableMeta/countSql. */
|
||||
countTotalRows?: () => Promise<number | undefined>;
|
||||
loading?: boolean;
|
||||
@@ -3548,6 +3561,13 @@ watch(
|
||||
);
|
||||
const manualTotalRowCount = ref<number | undefined>(undefined);
|
||||
const manualTotalRowCountLoading = ref(false);
|
||||
const esDeepPageJumpConfirmOpen = ref(false);
|
||||
const pendingEsDeepPageJump = ref<{ targetPage: number; requestCount: number; updateCurrentPage: boolean }>();
|
||||
watch(esDeepPageJumpConfirmOpen, (open) => {
|
||||
if (!open) {
|
||||
pendingEsDeepPageJump.value = undefined;
|
||||
}
|
||||
});
|
||||
const showTruncationWarning = computed(() => props.result.truncated === true && typeof props.pageLimit !== "number" && props.result.has_more !== true);
|
||||
const truncationHintKey = computed(() => dataGridTruncationHintKey(resolvedDatabaseType.value));
|
||||
// affected_rows reported by the backend can be larger than the rows we
|
||||
@@ -3624,8 +3644,14 @@ const canFetchNextInfiniteScrollSegment = computed(() =>
|
||||
);
|
||||
const canJumpLastPage = computed(() => canGoNextPage.value && (hasKnownPaginationTotalRowCount.value || allRowsLoaded.value || !!props.tableMeta || !!props.countSql || !!props.countTotalRows));
|
||||
const totalRowCountBusy = computed(() => props.totalRowCountLoading === true || manualTotalRowCountLoading.value);
|
||||
const pageJumpBusy = computed(() => !!props.pageJumpProgress && props.pageJumpProgress.totalRequests > 1);
|
||||
const pageJumpProgressPercent = computed(() => {
|
||||
const progress = props.pageJumpProgress;
|
||||
if (!progress || progress.totalRequests <= 0) return 0;
|
||||
return Math.min(100, Math.round((progress.completedRequests / progress.totalRequests) * 100));
|
||||
});
|
||||
/** Automatic background counts keep rows interactive; explicit count navigation still blocks the surface. */
|
||||
const gridSurfaceBusy = computed(() => isRefreshingData.value || props.loading === true || manualTotalRowCountLoading.value);
|
||||
const gridSurfaceBusy = computed(() => isRefreshingData.value || props.loading === true || manualTotalRowCountLoading.value || pageJumpBusy.value);
|
||||
const gridPaginationBusy = computed(() => gridSurfaceBusy.value || totalRowCountBusy.value);
|
||||
const dataGridNativeSelectionBlockOwner = {};
|
||||
watch(
|
||||
@@ -3740,6 +3766,40 @@ function currentOrderBy(): string | undefined {
|
||||
return orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined);
|
||||
}
|
||||
|
||||
function executeServerPageJump(targetPage: number, updateCurrentPage = false) {
|
||||
const offset = (targetPage - 1) * pageSize.value;
|
||||
if (!Number.isSafeInteger(offset)) {
|
||||
return;
|
||||
}
|
||||
if (updateCurrentPage) {
|
||||
currentPage.value = targetPage;
|
||||
}
|
||||
resetGridVerticalScroll(true);
|
||||
emit("paginate", offset, pageSize.value, currentWhereInput(), currentOrderBy());
|
||||
}
|
||||
|
||||
function requestServerPageJump(targetPage: number, updateCurrentPage = false) {
|
||||
const usesElasticsearchCursor = isResultsContext.value && (resolvedDatabaseType.value === "elasticsearch" || resolvedDatabaseType.value === "easysearch");
|
||||
const requestCount = elasticsearchCursorPageJumpRequestCount(currentPage.value, targetPage);
|
||||
if (usesElasticsearchCursor && requestCount >= ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS) {
|
||||
pendingEsDeepPageJump.value = { targetPage, requestCount, updateCurrentPage };
|
||||
esDeepPageJumpConfirmOpen.value = true;
|
||||
return;
|
||||
}
|
||||
|
||||
executeServerPageJump(targetPage, updateCurrentPage);
|
||||
}
|
||||
|
||||
function confirmEsDeepPageJump() {
|
||||
const pending = pendingEsDeepPageJump.value;
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
pendingEsDeepPageJump.value = undefined;
|
||||
esDeepPageJumpConfirmOpen.value = false;
|
||||
executeServerPageJump(pending.targetPage, pending.updateCurrentPage);
|
||||
}
|
||||
|
||||
watch(
|
||||
() => [props.countSql ?? "", props.tableMeta?.schema ?? "", props.tableMeta?.tableName ?? "", currentWhereInput() ?? "", props.database ?? "", props.connectionId ?? ""],
|
||||
(values, previousValues) => {
|
||||
@@ -3779,10 +3839,10 @@ function firstPage() {
|
||||
emit("paginate", 0, pageSize.value, currentWhereInput(), currentOrderBy());
|
||||
}
|
||||
function prevPage() {
|
||||
if (currentPage.value <= 1) return;
|
||||
currentPage.value--;
|
||||
resetGridVerticalScroll(true);
|
||||
emit("paginate", (currentPage.value - 1) * pageSize.value, pageSize.value, currentWhereInput(), currentOrderBy());
|
||||
if (currentPage.value <= 1) {
|
||||
return;
|
||||
}
|
||||
requestServerPageJump(currentPage.value - 1, true);
|
||||
}
|
||||
function nextPage() {
|
||||
if (!canGoNextPage.value) return;
|
||||
@@ -3792,18 +3852,19 @@ function nextPage() {
|
||||
}
|
||||
|
||||
function jumpPage(page: number) {
|
||||
if (gridPaginationBusy.value || infiniteScrollEnabled.value || !Number.isSafeInteger(page) || page < 1) return;
|
||||
if (gridPaginationBusy.value || infiniteScrollEnabled.value || !Number.isSafeInteger(page) || page < 1) {
|
||||
return;
|
||||
}
|
||||
const targetPage = Math.min(page, maximumPage.value ?? page);
|
||||
if (targetPage === currentPage.value) return;
|
||||
if (targetPage === currentPage.value) {
|
||||
return;
|
||||
}
|
||||
if (allRowsLoaded.value) {
|
||||
currentPage.value = targetPage;
|
||||
resetGridVerticalScroll(true);
|
||||
return;
|
||||
}
|
||||
const offset = (targetPage - 1) * pageSize.value;
|
||||
if (!Number.isSafeInteger(offset)) return;
|
||||
resetGridVerticalScroll(true);
|
||||
emit("paginate", offset, pageSize.value, currentWhereInput(), currentOrderBy());
|
||||
requestServerPageJump(targetPage);
|
||||
}
|
||||
|
||||
function infiniteScrollNextPage() {
|
||||
@@ -3861,13 +3922,16 @@ function applyCustomPageSize() {
|
||||
|
||||
function jumpToCountedLastPage(total: number) {
|
||||
const paginationTotal = resolveDataGridPaginationTotal({ paginationTotalRowCount: total, totalRowCountIsExact: true, maxRows: paginationMaxRows.value });
|
||||
if (paginationTotal === undefined || paginationTotal <= 0) return;
|
||||
if (paginationTotal === undefined || paginationTotal <= 0) {
|
||||
return;
|
||||
}
|
||||
const lastPageNum = Math.max(1, Math.ceil(paginationTotal / pageSize.value));
|
||||
if (lastPageNum <= currentPage.value) return;
|
||||
if (lastPageNum <= currentPage.value) {
|
||||
return;
|
||||
}
|
||||
// Do not bump currentPage before the new page loads — otherwise stale rows
|
||||
// briefly render with last-page indexes (e.g. 12001-13000) and flash a fake full page.
|
||||
resetGridVerticalScroll(true);
|
||||
emit("paginate", (lastPageNum - 1) * pageSize.value, pageSize.value, currentWhereInput(), currentOrderBy());
|
||||
requestServerPageJump(lastPageNum);
|
||||
}
|
||||
|
||||
async function beginManualTotalRowCount(): Promise<boolean> {
|
||||
@@ -13705,8 +13769,25 @@ function openGridSnapshot() {
|
||||
<div v-if="hasGridVerticalOverflow" ref="gridVerticalScrollbarTrackRef" class="data-grid-vertical-scrollbar" @pointerdown="startGridVerticalScrollbarDrag">
|
||||
<div ref="gridVerticalScrollbarThumbRef" class="data-grid-vertical-scrollbar__thumb" />
|
||||
</div>
|
||||
<div v-if="gridSurfaceBusy" class="absolute inset-0 z-20 bg-background/50 flex items-center justify-center">
|
||||
<div class="flex items-center gap-2 rounded-md border bg-background px-3 py-1.5 text-xs text-muted-foreground shadow-sm">
|
||||
<div v-if="gridSurfaceBusy" class="absolute inset-0 z-20 flex items-center justify-center" :class="pageJumpProgress ? 'bg-background/35 backdrop-blur-[1px]' : 'bg-background/50'">
|
||||
<div v-if="pageJumpProgress" class="w-72 max-w-[calc(100%-2rem)] rounded-lg border bg-background/95 p-3.5 shadow-lg">
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="flex h-8 w-8 shrink-0 items-center justify-center rounded-full bg-primary/10 text-primary">
|
||||
<Loader2 class="h-4 w-4 animate-spin" />
|
||||
</div>
|
||||
<div class="min-w-0 flex-1">
|
||||
<div class="truncate text-sm font-medium text-foreground">{{ t("grid.pageJumpLoading", { page: pageJumpProgress.targetPage }) }}</div>
|
||||
<div class="mt-0.5 flex items-center justify-between gap-3 text-[11px] text-muted-foreground">
|
||||
<span>{{ t("grid.pageJumpProgress", { current: pageJumpProgress.completedRequests, total: pageJumpProgress.totalRequests }) }}</span>
|
||||
<span class="shrink-0 tabular-nums">{{ formatElapsedSeconds(loadingElapsed) }}s</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-3 h-1.5 overflow-hidden rounded-full bg-muted" role="progressbar" :aria-valuemin="0" :aria-valuemax="pageJumpProgress.totalRequests" :aria-valuenow="pageJumpProgress.completedRequests">
|
||||
<div class="h-full rounded-full bg-primary transition-[width] duration-200 ease-out" :style="{ width: `${pageJumpProgressPercent}%` }" />
|
||||
</div>
|
||||
</div>
|
||||
<div v-else class="flex items-center gap-2 rounded-md border bg-background px-3 py-1.5 text-xs text-muted-foreground shadow-sm">
|
||||
<Loader2 class="w-3.5 h-3.5 animate-spin" />
|
||||
<span class="tabular-nums">{{ formatElapsedSeconds(loadingElapsed) }}s</span>
|
||||
</div>
|
||||
@@ -14337,6 +14418,29 @@ function openGridSnapshot() {
|
||||
<DataGridCopyColumnNamesDialog v-if="copyColumnNamesDialogMounted" v-model:open="copyColumnNamesDialogOpen" :column-names="copyColumnNamesDialogColumns" :database-type="resolvedDatabaseType" :column-comments="columnCommentMap" @copy="copyText" />
|
||||
<GridSnapshotDialog v-model:open="gridSnapshotOpen" :source="gridSnapshotSource" />
|
||||
|
||||
<Dialog v-model:open="esDeepPageJumpConfirmOpen">
|
||||
<DialogContent class="sm:max-w-[480px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle class="flex items-center gap-2 text-amber-600 dark:text-amber-400">
|
||||
<AlertTriangle class="h-5 w-5" />
|
||||
{{ t("grid.esDeepPageJumpConfirmTitle") }}
|
||||
</DialogTitle>
|
||||
</DialogHeader>
|
||||
<p class="py-3 text-sm leading-6 text-muted-foreground">
|
||||
{{
|
||||
t("grid.esDeepPageJumpConfirmMessage", {
|
||||
page: pendingEsDeepPageJump?.targetPage,
|
||||
requests: pendingEsDeepPageJump?.requestCount,
|
||||
})
|
||||
}}
|
||||
</p>
|
||||
<DialogFooter>
|
||||
<Button variant="outline" @click="esDeepPageJumpConfirmOpen = false">{{ t("dangerDialog.cancel") }}</Button>
|
||||
<Button @click="confirmEsDeepPageJump">{{ t("grid.esDeepPageJumpContinue") }}</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
<Dialog v-model:open="generateIncrementDialogOpen">
|
||||
<DialogContent class="sm:max-w-[380px]">
|
||||
<DialogHeader>
|
||||
|
||||
@@ -135,6 +135,19 @@ beforeEach(() => {
|
||||
localStorage.removeItem("dbx-filter-builder-value-shortcut-hint-days");
|
||||
});
|
||||
describe("DataGrid canvas surfaces", () => {
|
||||
it("asks before an expensive Elasticsearch cursor jump", () => {
|
||||
expect(dataGridSource).toContain("requestCount >= ELASTICSEARCH_PAGE_JUMP_WARNING_REQUESTS");
|
||||
expect(dataGridSource).toContain('t("grid.esDeepPageJumpConfirmMessage"');
|
||||
expect(dataGridSource).toContain('@click="confirmEsDeepPageJump"');
|
||||
});
|
||||
|
||||
it("shows stable request progress while an Elasticsearch page jump is running", () => {
|
||||
expect(dataGridSource).toContain('t("grid.pageJumpLoading", { page: pageJumpProgress.targetPage })');
|
||||
expect(dataGridSource).toContain('t("grid.pageJumpProgress", { current: pageJumpProgress.completedRequests, total: pageJumpProgress.totalRequests })');
|
||||
expect(dataGridSource).toContain('role="progressbar"');
|
||||
expect(dataGridSource).toContain(':style="{ width: `${pageJumpProgressPercent}%` }"');
|
||||
});
|
||||
|
||||
it("uses the stable overlay for viewport and device-pixel measurement", () => {
|
||||
expect(dataGridSource).toContain("function canvasMeasurementSurface(): HTMLElement | null");
|
||||
expect(dataGridSource).toContain("return canvasOverlayRef.value ?? null;");
|
||||
|
||||
@@ -1791,6 +1791,7 @@ defineExpose({
|
||||
:total-row-count="activeTab.resultTotalRowCount"
|
||||
:total-row-count-is-exact="activeTab.resultTotalRowCount !== undefined || activeTab.result.total_is_exact !== false"
|
||||
:total-row-count-loading="activeTab.resultTotalRowCountLoading"
|
||||
:page-jump-progress="activeTab.resultPageJumpProgress"
|
||||
:on-execute-sql="async (sql: string) => emit('executeSql', sql)"
|
||||
:full-export-result="(onProgress?: (info: { rowsExported: number; totalRows: number | null }) => void) => queryStore.fetchTabResultForExport(activeTab.id, onProgress)"
|
||||
:query-result-export-request="
|
||||
|
||||
Reference in new issue
Block a user