feat(meilisearch): import JSON and enhance document export

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-09-20 13:50:39 +08:00
1 parent 853a408d30
commit 908eab3c3a
15 files changed
+297 -34

No files matched your search

@@ -2,7 +2,7 @@
import { computed, onBeforeUnmount, onMounted, ref, watch } from "vue";
import type { Component } from "vue";
import { useI18n } from "vue-i18n";
import { ArrowDown, ArrowUp, Braces, ChevronLeft, ChevronRight, Copy, Download, LayoutGrid, LoaderCircle, Pencil, Save, Search, Table2, Trash2 } from "@lucide/vue";
import { ArrowDown, ArrowUp, Braces, ChevronDown, ChevronLeft, ChevronRight, Copy, Download, LayoutGrid, LoaderCircle, Pencil, Save, Search, Table2, Trash2, Upload } from "@lucide/vue";
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import { Input } from "@/components/ui/input";
@@ -10,6 +10,7 @@ import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import { Popover, PopoverAnchor, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu";
import DangerConfirmDialog from "@/components/editor/DangerConfirmDialog.vue";
import ErrorBanner from "@/components/ui/ErrorBanner.vue";
import QueryLoadingState from "@/components/common/QueryLoadingState.vue";
@@ -18,7 +19,7 @@ import RedisJsonEditor from "@/components/redis/RedisJsonEditor.vue";
import * as api from "@/lib/backend/api";
import { compactLocalTimestamp, sanitizeExportBaseName, saveTextFile } from "@/lib/export/saveTextFile";
import { parseDocumentStoreJsonDocument, serializeDocumentStoreId, stringifyDocumentStoreValue } from "@/lib/app/documentJsonValues";
import { parseJsonPreservingLargeNumbers, safeJsonFormat } from "@/lib/common/safeJsonFormat";
import { parseJsonPreservingLargeNumbers, safeJsonFormat, stringifyJsonPreservingLargeNumbers } from "@/lib/common/safeJsonFormat";
import { useToast } from "@/composables/useToast";
const props = defineProps<{
@@ -149,9 +150,15 @@ function renderMarkHtml(value: string): string {
* user payload — no name-based stripping, so user fields named `_formatted` or
* `_rankingScore` survive untouched.
*/
const displayValueCache = new WeakMap<Hit, Record<string, any>>();
function displayValue(hit: Hit): Record<string, any> {
const cached = displayValueCache.get(hit);
if (cached) return cached;
const formatted = hit?.formatted;
return formatted && typeof formatted === "object" ? { ...formatted } : { ...hit.document };
const value = formatted && typeof formatted === "object" ? { ...formatted } : { ...hit.document };
displayValueCache.set(hit, value);
return value;
}
/** Deep-remove `<mark>` search highlight tags to reconstruct the stored document. */
@@ -287,41 +294,96 @@ async function startEdit(hit: Hit) {
/** Download every hit matching the current search as a JSON file of the stored documents. */
const exporting = ref(false);
const importing = ref(false);
const importFileInput = ref<HTMLInputElement | null>(null);
const EXPORT_BATCH_SIZE = 1000;
async function exportResults() {
if (exporting.value || totalHits.value === 0) return;
if (q.value.trim() || hybridEnabled.value || rankingScoreThreshold.value > 0) {
toast(t("meilisearch.exportSearchUnsupported"), 5000);
type ExportScope = "filtered" | "all";
async function exportResults(scope: ExportScope) {
if (exporting.value) return;
if (scope === "filtered" && totalHits.value === 0) {
toast(t("grid.exportFailed", { message: t("meilisearch.empty") }), 5000);
return;
}
exporting.value = true;
try {
const useSearchPagination = scope === "filtered" && (q.value.trim() || hybridEnabled.value || rankingScoreThreshold.value > 0);
if (useSearchPagination && !indexSettingsLoaded.value) await loadIndexSettings();
if (useSearchPagination && searchMaxTotalHits.value === null) {
throw new Error(t("meilisearch.exportSearchUnsupported"));
}
const documents: Record<string, unknown>[] = [];
let batchOffset = 0;
let expectedTotal: number | undefined;
while (true) {
const page = await api.meilisearchFetchDocuments(props.connectionId, props.index, {
filter: filter.value.trim() || null,
sort: sort.value.trim() || null,
limit: EXPORT_BATCH_SIZE,
offset: batchOffset,
});
const batch = page.documents ?? [];
let batch: Record<string, unknown>[];
if (useSearchPagination) {
const page = await api.meilisearchSearchDocuments(props.connectionId, props.index, buildSearchParams(EXPORT_BATCH_SIZE, batchOffset));
batch = (page.hits ?? []).map((hit) => hit.document);
expectedTotal = page.totalHits;
} else {
const page = await api.meilisearchFetchDocuments(props.connectionId, props.index, {
filter: scope === "filtered" ? filter.value.trim() || null : null,
sort: scope === "filtered" ? sort.value.trim() || null : null,
limit: EXPORT_BATCH_SIZE,
offset: batchOffset,
});
batch = page.documents ?? [];
expectedTotal = page.total;
}
documents.push(...batch);
if (batch.length < EXPORT_BATCH_SIZE) break;
if (batch.length === 0 || batch.length < EXPORT_BATCH_SIZE || (expectedTotal !== undefined && documents.length >= expectedTotal)) break;
batchOffset += batch.length;
}
const reachedSearchLimit = useSearchPagination && searchMaxTotalHits.value !== null && expectedTotal !== undefined && expectedTotal >= searchMaxTotalHits.value && documents.length >= searchMaxTotalHits.value;
if (useSearchPagination && expectedTotal !== undefined && (documents.length < expectedTotal || reachedSearchLimit)) {
toast(t("grid.exportFailed", { message: t("meilisearch.exportLimitReached", { count: documents.length, total: expectedTotal }) }), 5000);
return;
}
if (documents.length === 0) {
toast(t("grid.exportFailed", { message: t("meilisearch.empty") }), 5000);
return;
}
const content = stringifyDocumentStoreValue(documents, "meilisearch", 2);
const baseName = sanitizeExportBaseName(props.index) || "search-results";
await saveTextFile(content, `${baseName}-${compactLocalTimestamp()}.json`, "JSON", "json");
const saved = await saveTextFile(content, `${baseName}-${compactLocalTimestamp()}.json`, "JSON", "json");
if (saved !== false) toast(t("grid.exported"));
} catch (e: any) {
toast(e?.message || String(e), 5000);
toast(t("grid.exportFailed", { message: e?.message || String(e) }), 5000);
} finally {
exporting.value = false;
}
}
async function importDocuments(event: Event) {
const input = event.target as HTMLInputElement;
const file = input.files?.[0];
input.value = "";
if (!file) return;
importing.value = true;
try {
const parsed = parseJsonPreservingLargeNumbers(await file.text());
const documents = Array.isArray(parsed) ? parsed : [parsed];
if (!documents.length || documents.some((document) => !document || typeof document !== "object" || Array.isArray(document))) {
throw new Error(t("meilisearch.importInvalidJson"));
}
const batchSize = 100;
for (let offset = 0; offset < documents.length; offset += batchSize) {
const batch = documents.slice(offset, offset + batchSize).map((document) => stringifyJsonPreservingLargeNumbers(document));
await api.documentSaveMeilisearchBatch(props.connectionId, props.index, [], [], batch);
}
toast(t("meilisearch.importSuccess", { count: documents.length }));
emit("refresh-stats");
await runSearch();
} catch (e: any) {
toast(t("meilisearch.importFailed", { message: e?.message || String(e) }), 5000);
} finally {
importing.value = false;
}
}
async function copyDocument(hit: Hit) {
try {
await navigator.clipboard.writeText(stringifyDocumentStoreValue(rawDocument(hit), "meilisearch", 2));
@@ -414,6 +476,8 @@ watch(autoRefresh, (enabled) => {
/** Sortable attributes advertised by the index settings, for the sort picker. */
const sortableAttributes = ref<string[]>([]);
const searchMaxTotalHits = ref<number | null>(null);
const indexSettingsLoaded = ref(false);
async function loadIndexSettings() {
try {
@@ -425,9 +489,14 @@ async function loadIndexSettings() {
}
const sortable = settings?.sortableAttributes;
sortableAttributes.value = Array.isArray(sortable) ? sortable.filter((item): item is string => typeof item === "string") : [];
const maxTotalHits = settings?.pagination?.maxTotalHits;
searchMaxTotalHits.value = typeof maxTotalHits === "number" && Number.isFinite(maxTotalHits) ? maxTotalHits : null;
} catch {
// Settings discovery is best-effort; the sort/filter inputs stay free-form.
sortableAttributes.value = [];
searchMaxTotalHits.value = null;
} finally {
indexSettingsLoaded.value = true;
}
}
@@ -630,9 +699,25 @@ onBeforeUnmount(() => {
</button>
</div>
<div class="flex-1" />
<Button v-if="totalHits > 0" variant="ghost" size="sm" class="h-6 gap-1 px-2 text-xs text-muted-foreground hover:text-foreground" :disabled="exporting" @click="exportResults">
<Download class="h-3.5 w-3.5" />
{{ t("meilisearch.exportResults") }} ({{ totalHits }})
<DropdownMenu v-if="!loading && !error">
<DropdownMenuTrigger as-child>
<Button variant="ghost" size="sm" class="h-6 gap-1 px-2 text-xs text-muted-foreground hover:text-foreground" :disabled="exporting || importing">
<LoaderCircle v-if="exporting" class="h-3.5 w-3.5 animate-spin" />
<Download v-else class="h-3.5 w-3.5" />
{{ t("meilisearch.exportResults") }}
<ChevronDown class="h-3 w-3" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" class="w-max min-w-32 max-w-[calc(100vw-1rem)]">
<DropdownMenuItem class="w-full whitespace-nowrap px-2" :disabled="totalHits === 0" @select="void exportResults('filtered')">{{ t("meilisearch.exportFiltered") }} ({{ totalHits }})</DropdownMenuItem>
<DropdownMenuItem class="w-full whitespace-nowrap px-2" @select="void exportResults('all')">{{ t("meilisearch.exportAll") }}</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<input ref="importFileInput" type="file" accept="application/json,.json" class="hidden" @change="importDocuments" />
<Button variant="ghost" size="sm" class="h-6 gap-1 px-2 text-xs text-muted-foreground hover:text-foreground" :disabled="loading || exporting || importing" @click="importFileInput?.click()">
<LoaderCircle v-if="importing" class="h-3.5 w-3.5 animate-spin" />
<Upload v-else class="h-3.5 w-3.5" />
{{ t("meilisearch.importResults") }}
</Button>
<span v-if="!loading && !error" class="text-muted-foreground text-xs tabular-nums">{{ resultSummary }}</span>
</div>
@@ -6,9 +6,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
searchDocuments: vi.fn(),
fetchDocuments: vi.fn(),
saveMeilisearchBatch: vi.fn(),
getIndexSettings: vi.fn(),
saveTextFile: vi.fn(),
toast: vi.fn(),
jsonTreeValues: [] as unknown[],
}));
function passthrough(tag: string): Component {
@@ -27,6 +29,7 @@ vi.mock("@lucide/vue", () => ({
ArrowDown: passthrough("span"),
ArrowUp: passthrough("span"),
Braces: passthrough("span"),
ChevronDown: passthrough("span"),
ChevronLeft: passthrough("span"),
ChevronRight: passthrough("span"),
Copy: passthrough("span"),
@@ -38,6 +41,7 @@ vi.mock("@lucide/vue", () => ({
Search: passthrough("span"),
Table2: passthrough("span"),
Trash2: passthrough("span"),
Upload: passthrough("span"),
}));
vi.mock("@/components/ui/button", () => ({ Button: passthrough("button") }));
vi.mock("@/components/ui/badge", () => ({ Badge: passthrough("span") }));
@@ -57,10 +61,26 @@ vi.mock("@/components/ui/dialog", () => ({
DialogHeader: passthrough("div"),
DialogTitle: passthrough("div"),
}));
vi.mock("@/components/ui/dropdown-menu", () => ({
DropdownMenu: passthrough("div"),
DropdownMenuContent: passthrough("div"),
DropdownMenuItem: passthrough("button"),
DropdownMenuTrigger: passthrough("div"),
}));
vi.mock("@/components/editor/DangerConfirmDialog.vue", () => ({ default: passthrough("div") }));
vi.mock("@/components/ui/ErrorBanner.vue", () => ({ default: passthrough("div") }));
vi.mock("@/components/common/QueryLoadingState.vue", () => ({ default: passthrough("div") }));
vi.mock("@/components/common/JsonTree.vue", () => ({ default: passthrough("div") }));
vi.mock("@/components/common/JsonTree.vue", () => ({
default: defineComponent({
inheritAttrs: false,
setup(_, { attrs }) {
return () => {
mocks.jsonTreeValues.push(attrs.value);
return h("div", attrs);
};
},
}),
}));
vi.mock("@/components/redis/RedisJsonEditor.vue", () => ({ default: passthrough("div") }));
vi.mock("@/composables/useToast", () => ({ useToast: () => ({ toast: mocks.toast }) }));
vi.mock("@/lib/export/saveTextFile", () => ({
@@ -71,6 +91,7 @@ vi.mock("@/lib/export/saveTextFile", () => ({
vi.mock("@/lib/backend/api", () => ({
meilisearchSearchDocuments: mocks.searchDocuments,
meilisearchFetchDocuments: mocks.fetchDocuments,
documentSaveMeilisearchBatch: mocks.saveMeilisearchBatch,
meilisearchGetIndexSettings: mocks.getIndexSettings,
}));
@@ -90,9 +111,19 @@ afterEach(() => {
app = undefined;
root = undefined;
vi.clearAllMocks();
mocks.jsonTreeValues.length = 0;
});
describe("MeilisearchDocumentsPage export", () => {
async function mountPage() {
root = document.createElement("div");
document.body.append(root);
app = createApp(MeilisearchDocumentsPage, { connectionId: "c1", index: "movies" });
app.mount(root);
await vi.waitFor(() => expect(mocks.searchDocuments).toHaveBeenCalledTimes(1));
await nextTick();
}
it("uses full-document pagination instead of capped displayed search hits", async () => {
const batch = (size: number, from: number) => Array.from({ length: size }, (_, index) => ({ movie_id: from + index, title: `Movie ${from + index}`, internal_notes: `note ${from + index}` }));
mocks.searchDocuments.mockResolvedValue({
@@ -106,19 +137,15 @@ describe("MeilisearchDocumentsPage export", () => {
return Promise.resolve({ documents: [], total: 1300 });
});
root = document.createElement("div");
document.body.append(root);
app = createApp(MeilisearchDocumentsPage, { connectionId: "c1", index: "movies" });
app.mount(root);
await vi.waitFor(() => expect(mocks.searchDocuments).toHaveBeenCalledTimes(1));
await nextTick();
await mountPage();
const exportButton = Array.from(root.querySelectorAll("button")).find((button) => button.textContent?.includes("meilisearch.exportResults"));
expect(exportButton?.textContent).toContain("1000");
exportButton?.dispatchEvent(new MouseEvent("click", { bubbles: true }));
expect(exportButton).not.toBeUndefined();
const filteredItem = Array.from(root.querySelectorAll("button")).find((button) => button.textContent?.includes("meilisearch.exportFiltered"));
filteredItem?.dispatchEvent(new Event("select", { bubbles: true }));
await vi.waitFor(() => expect(mocks.saveTextFile).toHaveBeenCalledTimes(1));
expect(mocks.toast).toHaveBeenCalledWith("grid.exported");
expect(mocks.searchDocuments).toHaveBeenCalledTimes(1);
const offsets = mocks.fetchDocuments.mock.calls.map((call) => (call[2] as { offset: number }).offset);
@@ -132,4 +159,77 @@ describe("MeilisearchDocumentsPage export", () => {
expect(exported[0]).toEqual({ movie_id: 0, title: "Movie 0", internal_notes: "note 0" });
expect(exported[1299]).toEqual({ movie_id: 1299, title: "Movie 1299", internal_notes: "note 1299" });
});
it("keeps JSON tree values stable while export state changes", async () => {
mocks.searchDocuments.mockResolvedValue({
hits: [{ id: 1, document: { movie_id: 1, details: { status: "ready" } } }],
totalHits: 1,
processingTimeMs: 1,
});
mocks.fetchDocuments.mockResolvedValue({ documents: [{ movie_id: 1, details: { status: "ready" } }], total: 1 });
await mountPage();
const beforeExport = mocks.jsonTreeValues.at(-1);
const filteredItem = Array.from(root!.querySelectorAll("button")).find((button) => button.textContent?.includes("meilisearch.exportFiltered"));
filteredItem?.dispatchEvent(new Event("select", { bubbles: true }));
await vi.waitFor(() => expect(mocks.saveTextFile).toHaveBeenCalledTimes(1));
expect(mocks.jsonTreeValues.some((value) => value === beforeExport)).toBe(true);
});
it("does not save an empty export file", async () => {
mocks.searchDocuments.mockResolvedValue({
hits: [{ id: 1, document: { movie_id: 1 } }],
totalHits: 1,
processingTimeMs: 1,
});
mocks.fetchDocuments.mockResolvedValue({ documents: [], total: 1 });
await mountPage();
const filteredItem = Array.from(root!.querySelectorAll("button")).find((button) => button.textContent?.includes("meilisearch.exportFiltered"));
filteredItem?.dispatchEvent(new Event("select", { bubbles: true }));
await vi.waitFor(() => expect(mocks.toast).toHaveBeenCalledWith("grid.exportFailed", 5000));
expect(mocks.saveTextFile).not.toHaveBeenCalled();
});
it("keeps all-document export available when the current filter has no hits", async () => {
mocks.searchDocuments.mockResolvedValue({ hits: [], totalHits: 0, processingTimeMs: 1 });
mocks.fetchDocuments.mockResolvedValue({ documents: [{ movie_id: 1, title: "Movie 1" }], total: 1 });
await mountPage();
const allItem = Array.from(root!.querySelectorAll("button")).find((button) => button.textContent?.includes("meilisearch.exportAll"));
expect(allItem).not.toBeUndefined();
allItem?.dispatchEvent(new Event("select", { bubbles: true }));
await vi.waitFor(() => expect(mocks.saveTextFile).toHaveBeenCalledTimes(1));
expect(mocks.fetchDocuments.mock.calls[0]?.[2]).toMatchObject({ filter: null, sort: null, offset: 0 });
});
it("imports JSON objects in write batches", async () => {
mocks.searchDocuments.mockResolvedValue({ hits: [], totalHits: 0, processingTimeMs: 1 });
await mountPage();
const fileInput = root!.querySelector<HTMLInputElement>('input[type="file"]');
const file = new File([JSON.stringify([{ movie_id: 1 }, { movie_id: 2 }])], "movies.json", { type: "application/json" });
Object.defineProperty(fileInput, "files", { value: [file] });
fileInput?.dispatchEvent(new Event("change", { bubbles: true }));
await vi.waitFor(() => expect(mocks.saveMeilisearchBatch).toHaveBeenCalledTimes(1));
expect(mocks.saveMeilisearchBatch.mock.calls[0]?.[4]).toEqual([JSON.stringify({ movie_id: 1 }), JSON.stringify({ movie_id: 2 })]);
expect(mocks.toast).toHaveBeenCalledWith("meilisearch.importSuccess");
});
it("imports decimal fields as raw JSON literals, not lossless number objects", async () => {
mocks.searchDocuments.mockResolvedValue({ hits: [], totalHits: 0, processingTimeMs: 1 });
await mountPage();
const fileInput = root!.querySelector<HTMLInputElement>('input[type="file"]');
const file = new File([JSON.stringify([{ movie_id: 1, rating: 8.5 }])], "movies.json", { type: "application/json" });
Object.defineProperty(fileInput, "files", { value: [file] });
fileInput?.dispatchEvent(new Event("change", { bubbles: true }));
await vi.waitFor(() => expect(mocks.saveMeilisearchBatch).toHaveBeenCalledTimes(1));
const batch = mocks.saveMeilisearchBatch.mock.calls[0]?.[4] as string[];
expect(batch).toEqual([JSON.stringify({ movie_id: 1, rating: 8.5 })]);
expect(JSON.parse(batch[0])).toEqual({ movie_id: 1, rating: 8.5 });
});
});
@@ -48,6 +48,7 @@ vi.mock("@lucide/vue", () => ({
ArrowDown: passthrough("span"),
ArrowUp: passthrough("span"),
Braces: passthrough("span"),
ChevronDown: passthrough("span"),
ChevronLeft: passthrough("span"),
ChevronRight: passthrough("span"),
Copy: passthrough("span"),
@@ -59,6 +60,7 @@ vi.mock("@lucide/vue", () => ({
Search: passthrough("span"),
Table2: passthrough("span"),
Trash2: passthrough("span"),
Upload: passthrough("span"),
}));
vi.mock("@/components/ui/button", () => ({ Button: passthrough("button") }));
vi.mock("@/components/ui/badge", () => ({ Badge: passthrough("span") }));