fix(elasticsearch): correct totals and CAT JSON output

apps/desktop/src/components/common/ElasticsearchJsonResponsePanel.vue

@@ -111,7 +111,7 @@ onMounted(() => {
       </Button>
     </header>
     <div class="min-h-0 flex-1 overflow-hidden bg-background p-4">
-      <pre v-show="responseView === 'raw' || !parsedBody.valid" class="m-0 h-full overflow-auto bg-transparent p-0 font-mono text-sm leading-6 whitespace-pre-wrap break-words">{{ body }}</pre>
+      <pre v-show="responseView === 'raw' || !parsedBody.valid" class="m-0 h-full overflow-auto bg-transparent p-0 font-mono text-sm leading-6 whitespace-pre">{{ body }}</pre>
       <div v-if="parsedBody.valid" v-show="responseView === 'json'" class="h-full min-h-0">
         <JsonTree ref="jsonTreeRef" :value="parsedBody.value" :highlight-json="highlightJson" :virtualized="true" class="dbx-editor-font-family text-sm leading-6" />
       </div>

apps/desktop/src/components/document/DocumentBrowser.vue

@@ -16,6 +16,7 @@ import * as api from "@/lib/backend/api";
 import { useConnectionStore } from "@/stores/connectionStore";
 import { clampSearchSplitWidth } from "@/lib/dataGrid/dataGridSearchSplit";
 import { documentViewerFontStyle } from "@/lib/document/documentViewerFontStyle";
+import { clampDocumentPage, documentPageRequestLimit, resetElasticsearchDocumentTotals, resolveElasticsearchDocumentTotals } from "@/lib/document/elasticsearchDocumentTotals";
 import {
   arrayObjectAncestorPathForDocumentField,
   buildDocumentFilterCondition,
@@ -84,7 +85,9 @@ type ViewMode = "document" | "table";
 const documents = ref<JsonRecord[]>([]);
 const copyDocuments = ref<JsonRecord[]>([]);
 const lastGridColumns = ref<string[]>([]);
-const total = ref(0);
+const total = ref<number | undefined>(undefined);
+const totalIsExact = ref(true);
+const paginationTotal = ref<number | undefined>(undefined);
 const loading = ref(false);
 const documentLoadExecutionId = ref("");
 const documentLoadCancelling = ref(false);
@@ -123,15 +126,26 @@ const tableFindPaneWidth = ref<number | null>(null);
 const isResizingTableSearchSplit = ref(false);
 let tableSearchSplitStartX = 0;
 let tableSearchSplitStartWidth = 0;
+let elasticsearchCountKey: string | null = null;
+let elasticsearchExactTotal: number | undefined;
+let elasticsearchPaginationLowerBound: number | undefined;
+let elasticsearchCountExecutionId = "";
+let elasticsearchCountGeneration = 0;
 const documentStoreProvider = computed(() => documentStoreProviderFor(props.databaseType));
 
+const pageTotal = computed(() => paginationTotal.value ?? total.value ?? 0);
+const documentRequestLimit = computed(() => {
+  if (documentStoreProvider.value.kind !== "elasticsearch" || paginationTotal.value === undefined) return pageSize.value;
+  return documentPageRequestLimit(page.value, pageSize.value, paginationTotal.value);
+});
+
 const tableFindPaneStyle = computed(() => {
   if (tableFindPaneWidth.value == null) return {};
   return { flex: `0 0 ${tableFindPaneWidth.value}px` };
 });
 const documentFontStyle = computed(() => documentViewerFontStyle(settingsStore.editorSettings));
 const documentStoreLabels = computed(() => ({
-  documentsLabel: documentStoreProvider.value.documentsLabel({ total: total.value, t }),
+  documentsLabel: documentStoreProvider.value.documentsLabel({ total: total.value ?? 0, t }),
   queryPreview: documentQueryPreview.value,
 }));
 
@@ -436,7 +450,7 @@ const documentQueryPreview = computed(() => {
     filterJson: filter,
     sortJson: sortInput.value.trim(),
     skip: page.value * pageSize.value,
-    limit: pageSize.value,
+    limit: documentRequestLimit.value,
   });
 });
 
@@ -580,6 +594,7 @@ async function gridSave(changes: DocumentGridChanges) {
     await api.documentInsertDocument(props.connectionId, props.database, props.collection, JSON.stringify(doc));
   }
 
+  if (isEs) resetElasticsearchTotals({ preservePaginationTotal: true });
   await load();
 }
 
@@ -684,6 +699,100 @@ function startDocumentLoadingTimer() {
   }, 100);
 }
 
+function elasticsearchCountFilterKey(filter: string | undefined): string {
+  return JSON.stringify([props.connectionId, props.database, props.collection, filter ?? ""]);
+}
+
+function cancelElasticsearchCount() {
+  elasticsearchCountGeneration++;
+  const executionId = elasticsearchCountExecutionId;
+  elasticsearchCountExecutionId = "";
+  if (executionId) void api.cancelQuery(executionId);
+}
+
+function resetElasticsearchTotals(options: { preservePaginationTotal?: boolean } = {}) {
+  const nextTotals = resetElasticsearchDocumentTotals(paginationTotal.value, options.preservePaginationTotal);
+  cancelElasticsearchCount();
+  elasticsearchCountKey = null;
+  elasticsearchExactTotal = undefined;
+  elasticsearchPaginationLowerBound = undefined;
+  paginationTotal.value = nextTotals.paginationTotal;
+  total.value = nextTotals.total;
+  totalIsExact.value = nextTotals.totalIsExact;
+}
+
+function clampPageToPaginationTotal(): boolean {
+  const cap = paginationTotal.value;
+  if (cap === undefined) return false;
+  const nextPage = clampDocumentPage(page.value, pageSize.value, cap);
+  if (page.value === nextPage) return false;
+  page.value = nextPage;
+  return true;
+}
+
+function startElasticsearchExactCount(filter: string | undefined) {
+  if (elasticsearchCountExecutionId || elasticsearchExactTotal !== undefined || !elasticsearchCountKey) return;
+  const key = elasticsearchCountKey;
+  const executionId = uuid();
+  const generation = elasticsearchCountGeneration;
+  elasticsearchCountExecutionId = executionId;
+
+  void api
+    .elasticsearchCountDocuments(props.connectionId, props.collection, filter, executionId)
+    .then((exactCount) => {
+      if (generation !== elasticsearchCountGeneration || key !== elasticsearchCountKey || executionId !== elasticsearchCountExecutionId || !Number.isFinite(exactCount) || exactCount < 0) {
+        return;
+      }
+      elasticsearchExactTotal = exactCount;
+      const totals = resolveElasticsearchDocumentTotals(elasticsearchPaginationLowerBound ?? exactCount, false, exactCount);
+      total.value = totals.total;
+      totalIsExact.value = totals.totalIsExact;
+      paginationTotal.value = totals.paginationTotal;
+      if (clampPageToPaginationTotal()) void load();
+    })
+    .catch(() => {
+      // The lower-bound result remains truthful when a background count fails.
+    })
+    .finally(() => {
+      if (generation === elasticsearchCountGeneration && executionId === elasticsearchCountExecutionId) {
+        elasticsearchCountExecutionId = "";
+      }
+    });
+}
+
+function applyElasticsearchSearchTotal(searchTotal: number, isExact: boolean, filter: string | undefined) {
+  const key = elasticsearchCountFilterKey(filter);
+  if (key !== elasticsearchCountKey) {
+    cancelElasticsearchCount();
+    elasticsearchCountKey = key;
+    elasticsearchExactTotal = undefined;
+    elasticsearchPaginationLowerBound = undefined;
+  }
+
+  elasticsearchPaginationLowerBound = searchTotal;
+  const totals = resolveElasticsearchDocumentTotals(searchTotal, isExact, elasticsearchExactTotal);
+  if (isExact) {
+    cancelElasticsearchCount();
+    elasticsearchExactTotal = searchTotal;
+    total.value = totals.total;
+    totalIsExact.value = totals.totalIsExact;
+    paginationTotal.value = totals.paginationTotal;
+    return;
+  }
+
+  if (elasticsearchExactTotal !== undefined) {
+    total.value = totals.total;
+    totalIsExact.value = totals.totalIsExact;
+    paginationTotal.value = totals.paginationTotal;
+    return;
+  }
+
+  total.value = totals.total;
+  totalIsExact.value = totals.totalIsExact;
+  paginationTotal.value = totals.paginationTotal;
+  startElasticsearchExactCount(filter);
+}
+
 async function load() {
   if (documentLoadExecutionId.value) void api.cancelQuery(documentLoadExecutionId.value);
   const executionId = uuid();
@@ -696,8 +805,12 @@ async function load() {
   const previousSelectedId = previousSelectedIdx === null ? null : documentIdentity(documents.value[previousSelectedIdx]);
   try {
     const filter = currentDocumentFilter();
+    if (documentStoreProvider.value.kind === "elasticsearch" && elasticsearchCountKey !== null && elasticsearchCountKey !== elasticsearchCountFilterKey(filter)) {
+      resetElasticsearchTotals();
+    }
     const sort = currentDocumentSortJson(sortInput.value);
-    const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, page.value * pageSize.value, pageSize.value, filter, undefined, sort, executionId);
+    const skip = page.value * pageSize.value;
+    const result = await api.documentFindDocuments(props.connectionId, props.database, props.collection, skip, documentRequestLimit.value, filter, undefined, sort, executionId);
     if (documentLoadExecutionId.value !== executionId) return;
     const nextDocuments =
       documentStoreProvider.value.kind === "elasticsearch" && result.raw_documents?.length === result.documents.length
@@ -722,7 +835,14 @@ async function load() {
       }
       lastGridColumns.value = [...keySet];
     }
-    total.value = result.total;
+    if (documentStoreProvider.value.kind === "elasticsearch") {
+      applyElasticsearchSearchTotal(result.total, result.total_is_exact !== false, filter);
+    } else {
+      cancelElasticsearchCount();
+      total.value = result.total;
+      totalIsExact.value = true;
+      paginationTotal.value = result.total;
+    }
     syncSelectedDocumentAfterLoad(previousSelectedIdx, previousSelectedId);
   } catch (e: unknown) {
     if (documentLoadExecutionId.value === executionId) error.value = e instanceof Error ? e.message : String(e);
@@ -736,6 +856,11 @@ async function load() {
   }
 }
 
+async function refreshDocuments() {
+  if (documentStoreProvider.value.kind === "elasticsearch") resetElasticsearchTotals({ preservePaginationTotal: true });
+  await load();
+}
+
 async function cancelDocumentLoad() {
   const executionId = documentLoadExecutionId.value;
   if (!executionId || documentLoadCancelling.value) return;
@@ -754,20 +879,22 @@ async function cancelDocumentLoad() {
 
 function applyFilter() {
   page.value = 0;
-  load();
+  if (documentStoreProvider.value.kind === "elasticsearch") resetElasticsearchTotals();
+  void load();
 }
 
 function paginate(offset: number, limit: number) {
   const normalizedLimit = normalizeResultPageSize(limit, pageSize.value);
   pageSize.value = normalizedLimit;
-  page.value = Math.floor(Math.max(0, offset) / normalizedLimit);
-  load();
+  const requestedPage = Math.floor(Math.max(0, offset) / normalizedLimit);
+  page.value = clampDocumentPage(requestedPage, normalizedLimit, paginationTotal.value);
+  void load();
 }
 
 function onSort(column: string, _columnIndex: number, direction: "asc" | "desc" | null) {
   sortInput.value = documentStoreProvider.value.sortInputForColumn(column, direction);
   page.value = 0;
-  load();
+  void load();
 }
 
 function asRecord(value: unknown): JsonRecord {
@@ -1129,6 +1256,7 @@ async function saveDoc() {
     isNew.value = false;
     documentEditMode.value = "fields";
     editFields.value = [];
+    if (kind === "elasticsearch") resetElasticsearchTotals({ preservePaginationTotal: true });
     await load();
     if (selectedIdx.value !== null && documents.value[selectedIdx.value]) {
       editJson.value = stringifyDocumentStoreValue(documents.value[selectedIdx.value], documentStoreProvider.value.kind, 2);
@@ -1151,6 +1279,7 @@ async function applyDeleteDoc(idx: number) {
       selectedIdx.value = null;
       editJson.value = "";
     }
+    if (documentStoreProvider.value.kind === "elasticsearch") resetElasticsearchTotals({ preservePaginationTotal: true });
     await load();
   } catch (e: unknown) {
     error.value = e instanceof Error ? e.message : String(e);
@@ -1176,13 +1305,13 @@ async function confirmDelete() {
 function prevPage() {
   if (page.value <= 0) return;
   page.value--;
-  load();
+  void load();
 }
 
 function nextPage() {
-  if ((page.value + 1) * pageSize.value >= total.value) return;
+  if ((page.value + 1) * pageSize.value >= pageTotal.value) return;
   page.value++;
-  load();
+  void load();
 }
 
 function docPreview(doc: JsonRecord): string {
@@ -1274,6 +1403,7 @@ onMounted(async () => {
 onBeforeUnmount(() => {
   window.removeEventListener("pointerdown", handleDocumentBrowserPointerDown, true);
   if (documentLoadExecutionId.value) void api.cancelQuery(documentLoadExecutionId.value);
+  cancelElasticsearchCount();
   stopDocumentLoadingTimer();
   endTableSearchSplitResize();
 });
@@ -1339,14 +1469,14 @@ defineExpose({ focusSearch });
       <span class="shrink-0 ml-1">{{ documentStoreLabels.documentsLabel }}</span>
 
       <Button v-if="viewMode === 'document'" variant="ghost" size="icon" class="h-5 w-5" @click="startNew"><Plus class="h-3 w-3" /></Button>
-      <Button v-if="viewMode === 'document'" variant="ghost" size="icon" class="h-5 w-5" @click="load"><RefreshCw class="h-3 w-3" :class="{ 'animate-spin': loading }" /></Button>
+      <Button v-if="viewMode === 'document'" variant="ghost" size="icon" class="h-5 w-5" @click="refreshDocuments"><RefreshCw class="h-3 w-3" :class="{ 'animate-spin': loading }" /></Button>
 
       <div v-if="viewMode === 'document'" class="flex items-center gap-1 ml-1">
         <Button variant="ghost" size="icon" class="h-5 w-5" :disabled="page <= 0" @click="prevPage">
           <ChevronLeft class="h-3 w-3" />
         </Button>
-        <span>{{ page + 1 }} / {{ Math.max(1, Math.ceil(total / pageSize)) }}</span>
-        <Button variant="ghost" size="icon" class="h-5 w-5" :disabled="(page + 1) * pageSize >= total" @click="nextPage">
+        <span>{{ page + 1 }} / {{ Math.max(1, Math.ceil(pageTotal / pageSize)) }}</span>
+        <Button variant="ghost" size="icon" class="h-5 w-5" :disabled="(page + 1) * pageSize >= pageTotal" @click="nextPage">
           <ChevronRight class="h-3 w-3" />
         </Button>
       </div>
@@ -1446,8 +1576,10 @@ defineExpose({ focusSearch });
       :page-offset="page * pageSize"
       :page-limit="pageSize"
       :total-row-count="total"
+      :total-row-count-is-exact="totalIsExact"
+      :pagination-total-row-count="pageTotal"
       @sort="onSort"
-      @reload="load"
+      @reload="refreshDocuments"
       @paginate="(offset: number, limit: number) => paginate(offset, limit)"
     >
       <template #search-bar="{ localFilterCount, hasLocalColumnFilters, localFilterSummaries, clearLocalFilter }: { localFilterCount: number; hasLocalColumnFilters: boolean; localFilterSummaries: LocalFilterSummary[]; clearLocalFilter: (columnIndex?: number) => void }">

apps/desktop/src/components/grid/DataGrid.vue

@@ -287,6 +287,8 @@ interface DataGridProps {
   pageLimit?: number;
   countSql?: string;
   totalRowCount?: number;
+  totalRowCountIsExact?: boolean;
+  paginationTotalRowCount?: number;
   totalRowCountLoading?: boolean;
   loading?: boolean;
   cacheKey?: string;
@@ -2363,9 +2365,13 @@ const inferredBackendTotalRowCount = computed(() => {
 });
 const serverKnownTotalRowCount = computed(() => props.totalRowCount ?? manualTotalRowCount.value);
 const displayedTotalRowCount = computed(() => serverKnownTotalRowCount.value ?? inferredBackendTotalRowCount.value);
+const totalRowCountIsExact = computed(() => props.totalRowCountIsExact !== false);
+// A backend can expose an exact display total while deliberately restricting
+// offset pagination to a smaller safe range.
+const paginationTotalRowCount = computed(() => props.paginationTotalRowCount ?? serverKnownTotalRowCount.value);
 // Only a server-confirmed total drives pagination — an inferred total means
 // rows exist that we never fetched, so navigation must stay inside rows.length.
-const hasKnownTotalRowCount = computed(() => typeof serverKnownTotalRowCount.value === "number" && serverKnownTotalRowCount.value >= 0);
+const hasKnownPaginationTotalRowCount = computed(() => typeof paginationTotalRowCount.value === "number" && paginationTotalRowCount.value >= 0);
 // When context=results and the caller hasn't configured server-side
 // pagination (no pageLimit), the backend handed us every row up-front and
 // rowCount IS the total. Without this hint, the "page is full → assume more"
@@ -2391,11 +2397,11 @@ const canGoNextPage = computed(() => {
     pageSize: pageSize.value,
     pageOffset: props.pageOffset,
     currentPage: currentPage.value,
-    totalRowCount: hasKnownTotalRowCount.value ? displayedTotalRowCount.value : undefined,
+    totalRowCount: hasKnownPaginationTotalRowCount.value ? paginationTotalRowCount.value : undefined,
     allRowsLoaded: allRowsLoaded.value,
   });
 });
-const canJumpLastPage = computed(() => canGoNextPage.value && (hasKnownTotalRowCount.value || allRowsLoaded.value || !!props.tableMeta || !!props.countSql));
+const canJumpLastPage = computed(() => canGoNextPage.value && (hasKnownPaginationTotalRowCount.value || allRowsLoaded.value || !!props.tableMeta || !!props.countSql));
 const totalRowCountBusy = computed(() => props.totalRowCountLoading === true || manualTotalRowCountLoading.value);
 const canCalculateTotalRowCount = computed(() => !!props.connectionId && (!!props.tableMeta || !!props.countSql));
 // When a refresh/rollback completes and the current page exceeds the last
@@ -2408,7 +2414,7 @@ watch(
     // and the completion was triggered by a refresh/rollback.
     if (!loading && prevLoading && isRefreshingData.value) {
       isRefreshingData.value = false;
-      const total = displayedTotalRowCount.value;
+      const total = paginationTotalRowCount.value;
       if (!total || total <= 0) return;
       const lastPageNum = Math.max(1, Math.ceil(total / pageSize.value));
       if (currentPage.value <= lastPageNum) return;
@@ -2571,8 +2577,8 @@ function applyCustomPageSize() {
 
 async function lastPage() {
   if (infiniteScrollEnabled.value) return;
-  if (hasKnownTotalRowCount.value) {
-    const total = displayedTotalRowCount.value ?? 0;
+  if (hasKnownPaginationTotalRowCount.value) {
+    const total = paginationTotalRowCount.value ?? 0;
     if (total <= 0) return;
     const lastPageNum = Math.ceil(total / pageSize.value);
     if (lastPageNum <= currentPage.value) return;
@@ -8916,7 +8922,7 @@ const gridContextMenuItems = computed<ContextMenuItem[]>(() => {
       <div class="flex min-w-0 items-center gap-2 overflow-hidden">
         <span v-if="hasData" class="shrink-0">
           {{ t(showTruncationWarning ? "grid.loadedRows" : "grid.totalRows", { count: result.rows.length }) }}
-          <span v-if="typeof displayedTotalRowCount === 'number' && displayedTotalRowCount >= 0" class="text-muted-foreground/70">{{ t("grid.totalRowCount", { count: displayedTotalRowCount }) }}</span>
+          <span v-if="typeof displayedTotalRowCount === 'number' && displayedTotalRowCount >= 0" class="text-muted-foreground/70">{{ t(totalRowCountIsExact === false ? "grid.totalRowCountAtLeast" : "grid.totalRowCount", { count: displayedTotalRowCount }) }}</span>
           <span v-else-if="totalRowCountBusy" class="text-muted-foreground/70">
             {{ t("grid.totalRowCountLoading") }}
           </span>