mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-19 16:16:45 +00:00
feat(mongodb): allow editing inferred CSV import column types
This commit is contained in:
onenewcode
@@ -8,6 +8,7 @@ import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import LightDropdown, { type LightDropdownItem } from "@/components/ui/LightDropdown.vue";
|
||||
import { AlertTriangle, ArrowLeft, ArrowRight, FileUp, Loader2, Square, Upload, X } from "@lucide/vue";
|
||||
import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import { connectionIsEffectivelyReadOnly } from "@/lib/database/readOnlyWriteAccess";
|
||||
@@ -42,6 +43,7 @@ const hasHeader = ref(true);
|
||||
const trim = ref(false);
|
||||
const emptyAsNull = ref(true);
|
||||
const typeMode = ref<api.MongoImportTypeMode>("auto");
|
||||
const columnTypeOverrides = ref<Record<string, api.MongoImportInferredType>>({});
|
||||
const recognizeObjectIdHex = ref(false);
|
||||
const skipErrorRows = ref(false);
|
||||
const batchSize = ref(500);
|
||||
@@ -66,8 +68,18 @@ const fileLabel = computed(() => sourceName.value || t("tableImport.noFileSelect
|
||||
const encodingOptions = TABLE_IMPORT_ENCODING_OPTIONS;
|
||||
const targetLabel = computed(() => `${props.database}.${props.collection}`);
|
||||
|
||||
const parseOptions = computed(
|
||||
(): api.MongoImportParseOptions => ({
|
||||
const columnTypeOptions: api.MongoImportInferredType[] = ["string", "boolean", "integer", "decimal", "date", "objectId", "object", "array", "mixed"];
|
||||
const columnTypeItems = computed<LightDropdownItem[]>(() =>
|
||||
columnTypeOptions.map((value) => ({
|
||||
value,
|
||||
label: t(`mongo.import.inferredType.${value}`),
|
||||
})),
|
||||
);
|
||||
const showColumnTypeEditors = computed(() => format.value === "csv" && typeMode.value !== "extendedJson");
|
||||
|
||||
const parseOptions = computed((): api.MongoImportParseOptions => {
|
||||
const overrides = columnTypeOverrides.value;
|
||||
return {
|
||||
encoding: encoding.value,
|
||||
delimiter: delimiter.value,
|
||||
hasHeader: hasHeader.value,
|
||||
@@ -76,8 +88,9 @@ const parseOptions = computed(
|
||||
typeMode: typeMode.value,
|
||||
recognizeObjectIdHex: recognizeObjectIdHex.value,
|
||||
skipErrorRows: skipErrorRows.value,
|
||||
}),
|
||||
);
|
||||
columnTypes: Object.keys(overrides).length ? { ...overrides } : null,
|
||||
};
|
||||
});
|
||||
|
||||
function formatFromName(name: string): api.MongoImportFormat {
|
||||
const lower = name.toLowerCase();
|
||||
@@ -99,6 +112,7 @@ function assignSource(source: string | File) {
|
||||
sourceName.value = importSourceDisplayName(source);
|
||||
format.value = formatFromName(sourceName.value);
|
||||
typeMode.value = format.value === "csv" ? "auto" : "extendedJson";
|
||||
columnTypeOverrides.value = {};
|
||||
delimiter.value = importTextDelimiterForName(sourceName.value);
|
||||
wizardStep.value = "source";
|
||||
queuePreview();
|
||||
@@ -137,8 +151,11 @@ async function loadPreview() {
|
||||
const source = selectedSource.value;
|
||||
if (!source) return;
|
||||
const requestId = ++previewRequestId;
|
||||
loadingPreview.value = true;
|
||||
previewError.value = "";
|
||||
const keepExistingPreview = !!preview.value;
|
||||
if (!keepExistingPreview) {
|
||||
loadingPreview.value = true;
|
||||
previewError.value = "";
|
||||
}
|
||||
try {
|
||||
const input = importPreviewInput(uploadedSource, source);
|
||||
const next = await api.previewMongodbImportFile(input.fileOrPath, {
|
||||
@@ -149,6 +166,7 @@ async function loadPreview() {
|
||||
});
|
||||
if (requestId !== previewRequestId) return;
|
||||
preview.value = next;
|
||||
previewError.value = "";
|
||||
uploadedSource = uploadedImportSourceFromPreview(next) ?? uploadedSource;
|
||||
} catch (error) {
|
||||
if (requestId !== previewRequestId) return;
|
||||
@@ -159,7 +177,11 @@ async function loadPreview() {
|
||||
}
|
||||
}
|
||||
|
||||
watch([format, encoding, delimiter, hasHeader, trim, emptyAsNull, typeMode, recognizeObjectIdHex, previewLimit], () => {
|
||||
watch([format, hasHeader, typeMode], () => {
|
||||
if (Object.keys(columnTypeOverrides.value).length) columnTypeOverrides.value = {};
|
||||
});
|
||||
|
||||
watch([format, encoding, delimiter, hasHeader, trim, emptyAsNull, typeMode, recognizeObjectIdHex, previewLimit, columnTypeOverrides], () => {
|
||||
if (selectedSource.value) queuePreview();
|
||||
});
|
||||
|
||||
@@ -178,6 +200,7 @@ function reset() {
|
||||
selectedSource.value = null;
|
||||
sourceName.value = "";
|
||||
previewError.value = "";
|
||||
columnTypeOverrides.value = {};
|
||||
wizardStep.value = "source";
|
||||
running.value = false;
|
||||
cancelling.value = false;
|
||||
@@ -274,6 +297,23 @@ function setTypeMode(value: unknown) {
|
||||
if (value === "string" || value === "auto" || value === "extendedJson") typeMode.value = value;
|
||||
}
|
||||
|
||||
function isImportInferredType(value: unknown): value is api.MongoImportInferredType {
|
||||
return columnTypeOptions.includes(value as api.MongoImportInferredType);
|
||||
}
|
||||
|
||||
function columnType(column: api.MongoImportColumn): api.MongoImportInferredType {
|
||||
return columnTypeOverrides.value[column.name] ?? column.inferredType;
|
||||
}
|
||||
|
||||
function setColumnType(name: string, value: unknown) {
|
||||
if (!isImportInferredType(value)) return;
|
||||
const inferred = preview.value?.columns.find((column) => column.name === name)?.inferredType;
|
||||
const next = { ...columnTypeOverrides.value };
|
||||
if (value === inferred) delete next[name];
|
||||
else next[name] = value;
|
||||
columnTypeOverrides.value = next;
|
||||
}
|
||||
|
||||
function requestClose() {
|
||||
if (running.value) {
|
||||
closeBlocked.value = true;
|
||||
@@ -319,7 +359,7 @@ function requestClose() {
|
||||
<div class="grid grid-cols-3 gap-3 rounded-md border p-3">
|
||||
<div class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.sourceFormat") }}</Label>
|
||||
<Select :model-value="format" @update:model-value="setFormat">
|
||||
<Select :model-value="format" :aria-label="t('tableImport.sourceFormat')" @update:model-value="setFormat">
|
||||
<SelectTrigger class="h-8 text-xs"><SelectValue /></SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="csv">CSV</SelectItem>
|
||||
@@ -339,11 +379,11 @@ function requestClose() {
|
||||
</div>
|
||||
<div v-if="format === 'csv'" class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.delimiter") }}</Label>
|
||||
<Input v-model="delimiter" class="h-8 text-xs font-mono" />
|
||||
<Input v-model="delimiter" class="h-8 text-xs font-mono" :aria-label="t('tableImport.delimiter')" />
|
||||
</div>
|
||||
<div class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("mongo.import.typeMode") }}</Label>
|
||||
<Select :model-value="typeMode" @update:model-value="setTypeMode">
|
||||
<Select :model-value="typeMode" :aria-label="t('mongo.import.typeMode')" @update:model-value="setTypeMode">
|
||||
<SelectTrigger class="h-8 text-xs"><SelectValue /></SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="string">{{ t("mongo.import.typeString") }}</SelectItem>
|
||||
@@ -365,28 +405,39 @@ function requestClose() {
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-2 gap-x-4 gap-y-2 rounded-md border p-3">
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="hasHeader" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("tableImport.hasHeader") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="hasHeader" type="checkbox" class="h-3.5 w-3.5 accent-primary" :aria-label="t('tableImport.hasHeader')" /> {{ t("tableImport.hasHeader") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="trim" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("tableImport.trimValues") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="emptyAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.emptyAsNull") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="recognizeObjectIdHex" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.recognizeObjectIdHex") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="skipErrorRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.skipErrorRows") }}</label>
|
||||
</div>
|
||||
|
||||
<div v-if="loadingPreview" class="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<div v-if="loadingPreview && !preview" class="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Loader2 class="h-3.5 w-3.5 animate-spin" />
|
||||
{{ t("mongo.import.previewing") }}
|
||||
</div>
|
||||
<div v-else-if="previewError" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">{{ previewError }}</div>
|
||||
<div v-else-if="preview" class="space-y-2">
|
||||
<div v-if="previewError" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">{{ previewError }}</div>
|
||||
<div v-if="preview" class="space-y-2">
|
||||
<div class="text-xs text-muted-foreground">{{ t("mongo.import.estimatedRows", { count: preview.estimatedRows ?? 0 }) }}</div>
|
||||
<div class="max-h-56 overflow-auto rounded-md border">
|
||||
<table class="w-full text-xs">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="border-b px-2 py-1 text-left">#</th>
|
||||
<th v-for="column in preview.columns" :key="column.name" class="border-b px-2 py-1 text-left">
|
||||
{{ column.name }}
|
||||
<span class="text-muted-foreground">({{ column.inferredType }})</span>
|
||||
<th v-for="column in preview.columns" :key="column.name" class="border-b px-2 py-1 text-left align-top">
|
||||
<div class="font-medium">{{ column.name }}</div>
|
||||
<LightDropdown
|
||||
v-if="showColumnTypeEditors"
|
||||
:model-value="columnType(column)"
|
||||
:items="columnTypeItems"
|
||||
:aria-label="t('mongo.import.columnType', { name: column.name })"
|
||||
:disabled="running"
|
||||
trigger-class="mt-1 inline-flex h-6 w-full min-w-[6.5rem] max-w-[8.5rem] items-center justify-between gap-1 rounded-md border bg-background px-1.5 text-[11px] font-normal hover:bg-muted/30 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
content-class="pointer-events-auto !z-[80] ring-1 ring-foreground/10"
|
||||
check-position="right"
|
||||
@update:model-value="(value) => setColumnType(column.name, value)"
|
||||
/>
|
||||
<span v-else-if="format !== 'csv'" class="text-muted-foreground">({{ column.inferredType }})</span>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
|
||||
@@ -27,6 +27,71 @@ vi.mock("@/lib/database/productionExecutionGuard", () => ({
|
||||
executeWithProductionContextGuard: async ({ execute }: { execute: () => Promise<unknown> }) => execute(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/ui/select", async () => {
|
||||
const { defineComponent, h } = await import("vue");
|
||||
|
||||
function collectOptions(nodes: unknown[] | undefined): ReturnType<typeof h>[] {
|
||||
if (!nodes) return [];
|
||||
const options: ReturnType<typeof h>[] = [];
|
||||
for (const node of nodes) {
|
||||
if (!node || typeof node !== "object") continue;
|
||||
const vnode = node as { type?: { name?: string }; props?: { value?: unknown }; children?: unknown };
|
||||
if (vnode.type?.name === "SelectItemStub") {
|
||||
const children = vnode.children;
|
||||
const label = typeof children === "object" && children && "default" in children && typeof (children as { default: unknown }).default === "function" ? (children as { default: () => unknown }).default() : children;
|
||||
options.push(h("option", { value: String(vnode.props?.value ?? "") }, label as never));
|
||||
continue;
|
||||
}
|
||||
const children = vnode.children;
|
||||
if (typeof children === "object" && children && "default" in children && typeof (children as { default: unknown }).default === "function") {
|
||||
options.push(...collectOptions((children as { default: () => unknown[] }).default()));
|
||||
} else if (Array.isArray(children)) {
|
||||
options.push(...collectOptions(children));
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
return {
|
||||
Select: defineComponent({
|
||||
props: { modelValue: { type: [String, Number, Boolean], default: undefined } },
|
||||
emits: ["update:modelValue"],
|
||||
setup(props, { emit, slots, attrs }) {
|
||||
return () =>
|
||||
h(
|
||||
"select",
|
||||
{
|
||||
...attrs,
|
||||
value: props.modelValue,
|
||||
onChange: (event: Event) => emit("update:modelValue", (event.target as HTMLSelectElement).value),
|
||||
},
|
||||
collectOptions(slots.default?.() as unknown[] | undefined),
|
||||
);
|
||||
},
|
||||
}),
|
||||
SelectItem: defineComponent({
|
||||
name: "SelectItemStub",
|
||||
props: { value: { type: [String, Number, Boolean], required: true } },
|
||||
setup: () => () => null,
|
||||
}),
|
||||
SelectContent: defineComponent({
|
||||
setup:
|
||||
(_, { slots }) =>
|
||||
() =>
|
||||
h("div", slots.default?.()),
|
||||
}),
|
||||
SelectTrigger: defineComponent({
|
||||
setup:
|
||||
(_, { slots }) =>
|
||||
() =>
|
||||
h("div", slots.default?.()),
|
||||
}),
|
||||
SelectValue: defineComponent({
|
||||
setup: () => () => null,
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
import MongoImportDialog from "../MongoImportDialog.vue";
|
||||
|
||||
function previewResult(overrides: Partial<MongoImportPreview> = {}): MongoImportPreview {
|
||||
@@ -128,4 +193,191 @@ describe("MongoImportDialog", () => {
|
||||
expect(api.releaseMongodbImportSource).toHaveBeenCalledWith("src-1");
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
async function mountWithFile(fileName: string, preview: MongoImportPreview) {
|
||||
api.previewMongodbImportFile.mockResolvedValue(preview);
|
||||
const app = createApp(
|
||||
defineComponent({
|
||||
setup() {
|
||||
return () =>
|
||||
h(MongoImportDialog, {
|
||||
open: true,
|
||||
connectionId: "c1",
|
||||
database: "shop",
|
||||
collection: "orders",
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
app.mount(document.body);
|
||||
await nextTick();
|
||||
const input = document.querySelector("input[type='file']") as HTMLInputElement;
|
||||
const file = new File(["id\n1"], fileName, { type: "text/csv" });
|
||||
Object.defineProperty(input, "files", { configurable: true, value: [file] });
|
||||
input.dispatchEvent(new Event("change"));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
return app;
|
||||
}
|
||||
|
||||
function columnTypeTrigger() {
|
||||
return document.querySelector('button[aria-label="mongo.import.columnType"]') as HTMLButtonElement | null;
|
||||
}
|
||||
|
||||
async function setColumnTypeValue(value: string) {
|
||||
const trigger = columnTypeTrigger();
|
||||
expect(trigger).not.toBeNull();
|
||||
trigger!.click();
|
||||
await nextTick();
|
||||
const item = [...document.querySelectorAll('[role="menuitem"]')].find((element) => (element.textContent ?? "").includes(`inferredType.${value}`)) as HTMLButtonElement | undefined;
|
||||
expect(item).toBeTruthy();
|
||||
item!.click();
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
}
|
||||
|
||||
it("defaults csv column types to inference and lets the user override them", async () => {
|
||||
const app = await mountWithFile("orders.csv", previewResult({ columns: [{ name: "id", inferredType: "integer" }], rows: [{ id: 1 }] }));
|
||||
const trigger = columnTypeTrigger();
|
||||
expect(trigger).not.toBeNull();
|
||||
expect(trigger?.textContent).toContain("inferredType.integer");
|
||||
expect(trigger?.getAttribute("aria-label")).toBe("mongo.import.columnType");
|
||||
|
||||
await setColumnTypeValue("string");
|
||||
|
||||
expect(api.previewMongodbImportFile.mock.calls.at(-1)?.[1]).toMatchObject({
|
||||
parseOptions: expect.objectContaining({ columnTypes: { id: "string" } }),
|
||||
});
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("selects a column type on pointerdown so dialogs do not swallow the click", async () => {
|
||||
const app = await mountWithFile("orders.csv", previewResult({ columns: [{ name: "id", inferredType: "integer" }], rows: [{ id: 1 }] }));
|
||||
const trigger = columnTypeTrigger();
|
||||
expect(trigger).not.toBeNull();
|
||||
trigger!.click();
|
||||
await nextTick();
|
||||
const item = [...document.querySelectorAll('[role="menuitem"]')].find((element) => (element.textContent ?? "").includes("inferredType.boolean")) as HTMLButtonElement | undefined;
|
||||
expect(item).toBeTruthy();
|
||||
item!.dispatchEvent(new PointerEvent("pointerdown", { bubbles: true, cancelable: true }));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
expect(api.previewMongodbImportFile.mock.calls.at(-1)?.[1]).toMatchObject({
|
||||
parseOptions: expect.objectContaining({ columnTypes: { id: "boolean" } }),
|
||||
});
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("does not replace the preview table with a loading state when column types change", async () => {
|
||||
const app = await mountWithFile("orders.csv", previewResult({ columns: [{ name: "id", inferredType: "integer" }], rows: [{ id: 1 }] }));
|
||||
expect(columnTypeTrigger()).not.toBeNull();
|
||||
api.previewMongodbImportFile.mockImplementation(() => new Promise(() => {}));
|
||||
|
||||
const trigger = columnTypeTrigger();
|
||||
trigger!.click();
|
||||
await nextTick();
|
||||
const item = [...document.querySelectorAll('[role="menuitem"]')].find((element) => (element.textContent ?? "").includes("inferredType.string")) as HTMLButtonElement | undefined;
|
||||
expect(item).toBeTruthy();
|
||||
item!.click();
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
|
||||
expect(columnTypeTrigger()).not.toBeNull();
|
||||
expect(document.body.textContent).not.toContain("mongo.import.previewing");
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("clears column type overrides when the file changes", async () => {
|
||||
const app = await mountWithFile("orders.csv", previewResult({ columns: [{ name: "id", inferredType: "integer" }] }));
|
||||
await setColumnTypeValue("string");
|
||||
|
||||
const input = document.querySelector("input[type='file']") as HTMLInputElement;
|
||||
const file = new File(["id\n2"], "other.csv", { type: "text/csv" });
|
||||
Object.defineProperty(input, "files", { configurable: true, value: [file] });
|
||||
input.dispatchEvent(new Event("change"));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
|
||||
expect(api.previewMongodbImportFile.mock.calls.at(-1)?.[1]).toMatchObject({
|
||||
parseOptions: expect.objectContaining({ columnTypes: null }),
|
||||
});
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("does not show column type editors for json imports", async () => {
|
||||
const app = await mountWithFile(
|
||||
"orders.json",
|
||||
previewResult({
|
||||
format: "json",
|
||||
fileName: "orders.json",
|
||||
columns: [{ name: "id", inferredType: "string" }],
|
||||
}),
|
||||
);
|
||||
expect(columnTypeTrigger()).toBeNull();
|
||||
expect(document.body.textContent).toContain("(string)");
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
function lastParseOptions() {
|
||||
return api.previewMongodbImportFile.mock.calls.at(-1)?.[1]?.parseOptions as Record<string, unknown> | undefined;
|
||||
}
|
||||
|
||||
async function overrideIdToString() {
|
||||
await setColumnTypeValue("string");
|
||||
}
|
||||
|
||||
async function setLabeledSelect(label: string, value: string) {
|
||||
const select = document.querySelector(`select[aria-label="${label}"]`) as HTMLSelectElement;
|
||||
expect(select).not.toBeNull();
|
||||
select.value = value;
|
||||
select.dispatchEvent(new Event("change"));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
}
|
||||
|
||||
it("keeps column type overrides when only the delimiter changes", async () => {
|
||||
const app = await mountWithFile("orders.csv", previewResult({ columns: [{ name: "id", inferredType: "integer" }] }));
|
||||
await overrideIdToString();
|
||||
expect(lastParseOptions()).toMatchObject({ columnTypes: { id: "string" } });
|
||||
|
||||
const delimiter = document.querySelector('input[aria-label="tableImport.delimiter"]') as HTMLInputElement;
|
||||
expect(delimiter).not.toBeNull();
|
||||
delimiter.value = ";";
|
||||
delimiter.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
|
||||
expect(lastParseOptions()).toMatchObject({ columnTypes: { id: "string" }, delimiter: ";" });
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("clears column type overrides when format, header row, or type mode changes", async () => {
|
||||
const app = await mountWithFile("orders.csv", previewResult({ columns: [{ name: "id", inferredType: "integer" }] }));
|
||||
|
||||
await overrideIdToString();
|
||||
await setLabeledSelect("tableImport.sourceFormat", "json");
|
||||
expect(lastParseOptions()).toMatchObject({ columnTypes: null });
|
||||
|
||||
await setLabeledSelect("tableImport.sourceFormat", "csv");
|
||||
await overrideIdToString();
|
||||
const header = document.querySelector('input[aria-label="tableImport.hasHeader"]') as HTMLInputElement;
|
||||
header.click();
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
expect(lastParseOptions()).toMatchObject({ columnTypes: null });
|
||||
|
||||
await overrideIdToString();
|
||||
await setLabeledSelect("mongo.import.typeMode", "string");
|
||||
expect(lastParseOptions()).toMatchObject({ columnTypes: null });
|
||||
|
||||
app.unmount();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -168,6 +168,13 @@ function selectItem(item: LightDropdownItem) {
|
||||
if (props.closeOnSelect) close();
|
||||
}
|
||||
|
||||
function selectItemOnKeyboard(item: LightDropdownItem, event: MouseEvent) {
|
||||
// Keyboard activation (Enter/Space) emits a click with detail 0; mouse
|
||||
// clicks are already handled by pointerdown, so only pick up keyboard here
|
||||
// to avoid selecting twice for one press.
|
||||
if (event.detail === 0) selectItem(item);
|
||||
}
|
||||
|
||||
onBeforeUnmount(close);
|
||||
</script>
|
||||
|
||||
@@ -180,7 +187,7 @@ onBeforeUnmount(close);
|
||||
<ChevronDown v-if="showChevron" class="h-3 w-3 opacity-50" />
|
||||
</button>
|
||||
<Teleport to="body">
|
||||
<div v-if="open" ref="menuRef" class="fixed z-50 min-w-32 overflow-x-hidden overflow-y-auto rounded-md p-1 cn-menu-translucent text-popover-foreground" :class="contentClass" :style="menuStyle" role="menu">
|
||||
<div v-if="open" ref="menuRef" class="pointer-events-auto fixed z-50 min-w-32 overflow-x-hidden overflow-y-auto rounded-md p-1 cn-menu-translucent text-popover-foreground" :class="contentClass" :style="menuStyle" role="menu" @pointerdown.prevent>
|
||||
<div v-if="label" :class="labelClass">{{ label }}</div>
|
||||
<div v-if="label" class="bg-border -mx-1 my-1 h-px" />
|
||||
<template v-for="item in items" :key="item.value">
|
||||
@@ -194,7 +201,8 @@ onBeforeUnmount(close);
|
||||
:title="item.title"
|
||||
:style="{ paddingInlineStart: `${0.375 + (item.indentLevel ?? 0) * 0.75}rem` }"
|
||||
role="menuitem"
|
||||
@click="selectItem(item)"
|
||||
@pointerdown.prevent="selectItem(item)"
|
||||
@click="selectItemOnKeyboard(item, $event)"
|
||||
>
|
||||
<Check v-if="checkPosition === 'left'" class="h-3 w-3 shrink-0" :class="[isItemSelected(item) ? selectedCheckClass : 'opacity-0']" />
|
||||
<span v-if="item.leadingText" class="inline-flex h-5 w-6 shrink-0 items-center justify-center text-sm font-medium leading-none">
|
||||
|
||||
@@ -32,6 +32,10 @@ const forwarded = useForwardPropsEmits(delegatedProps, emits);
|
||||
(event) => {
|
||||
const originalEvent = event.detail.originalEvent;
|
||||
const target = originalEvent.target as HTMLElement;
|
||||
if (target.closest('[role=menu], [role=listbox], [data-slot=select-content]')) {
|
||||
event.preventDefault();
|
||||
return;
|
||||
}
|
||||
if (originalEvent.offsetX > target.clientWidth || originalEvent.offsetY > target.clientHeight) {
|
||||
event.preventDefault();
|
||||
}
|
||||
|
||||
@@ -5372,6 +5372,18 @@ export default withEnglishFallback({
|
||||
typeString: "Hamısı sətir",
|
||||
typeAuto: "Avtomatik aşkarlama",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "{name} tipi",
|
||||
inferredType: {
|
||||
string: "Sətir",
|
||||
boolean: "Məntiqi",
|
||||
integer: "Tam ədəd",
|
||||
decimal: "Onluq",
|
||||
date: "Tarix",
|
||||
objectId: "ObjectId",
|
||||
object: "Obyekt",
|
||||
array: "Massiv",
|
||||
mixed: "Qarışıq",
|
||||
},
|
||||
hasHeader: "Birinci sətir başlıqdır",
|
||||
trim: "Dəyərləri kəs",
|
||||
emptyAsNull: "Boş xanaları null yaz",
|
||||
|
||||
@@ -5522,6 +5522,18 @@ export default {
|
||||
typeString: "All strings",
|
||||
typeAuto: "Auto detect",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "Type for {name}",
|
||||
inferredType: {
|
||||
string: "String",
|
||||
boolean: "Boolean",
|
||||
integer: "Integer",
|
||||
decimal: "Decimal",
|
||||
date: "Date",
|
||||
objectId: "ObjectId",
|
||||
object: "Object",
|
||||
array: "Array",
|
||||
mixed: "Mixed",
|
||||
},
|
||||
hasHeader: "First row is header",
|
||||
trim: "Trim values",
|
||||
emptyAsNull: "Empty cells as null",
|
||||
|
||||
@@ -5224,6 +5224,18 @@ export default withEnglishFallback({
|
||||
typeString: "Todo como texto",
|
||||
typeAuto: "Detectar automáticamente",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "Tipo de {name}",
|
||||
inferredType: {
|
||||
string: "Cadena",
|
||||
boolean: "Booleano",
|
||||
integer: "Entero",
|
||||
decimal: "Decimal",
|
||||
date: "Fecha",
|
||||
objectId: "ObjectId",
|
||||
object: "Objeto",
|
||||
array: "Matriz",
|
||||
mixed: "Mixto",
|
||||
},
|
||||
hasHeader: "La primera fila es el encabezado",
|
||||
trim: "Recortar valores",
|
||||
emptyAsNull: "Celdas vacías como null",
|
||||
|
||||
@@ -5222,6 +5222,18 @@ export default withEnglishFallback({
|
||||
typeString: "Tutto come stringa",
|
||||
typeAuto: "Rilevamento automatico",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "Tipo di {name}",
|
||||
inferredType: {
|
||||
string: "Stringa",
|
||||
boolean: "Booleano",
|
||||
integer: "Intero",
|
||||
decimal: "Decimale",
|
||||
date: "Data",
|
||||
objectId: "ObjectId",
|
||||
object: "Oggetto",
|
||||
array: "Array",
|
||||
mixed: "Misto",
|
||||
},
|
||||
hasHeader: "La prima riga è l'intestazione",
|
||||
trim: "Taglia gli spazi",
|
||||
emptyAsNull: "Celle vuote come null",
|
||||
|
||||
@@ -5251,6 +5251,18 @@ export default withEnglishFallback({
|
||||
typeString: "すべて文字列",
|
||||
typeAuto: "自動検出",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "{name} の型",
|
||||
inferredType: {
|
||||
string: "文字列",
|
||||
boolean: "ブール",
|
||||
integer: "整数",
|
||||
decimal: "小数",
|
||||
date: "日付",
|
||||
objectId: "ObjectId",
|
||||
object: "オブジェクト",
|
||||
array: "配列",
|
||||
mixed: "混合",
|
||||
},
|
||||
hasHeader: "先頭行をヘッダーにする",
|
||||
trim: "値をトリム",
|
||||
emptyAsNull: "空セルを null にする",
|
||||
|
||||
@@ -4859,6 +4859,18 @@ export default withEnglishFallback({
|
||||
typeString: "모두 문자열",
|
||||
typeAuto: "자동 감지",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "{name} 유형",
|
||||
inferredType: {
|
||||
string: "문자열",
|
||||
boolean: "불리언",
|
||||
integer: "정수",
|
||||
decimal: "소수",
|
||||
date: "날짜",
|
||||
objectId: "ObjectId",
|
||||
object: "객체",
|
||||
array: "배열",
|
||||
mixed: "혼합",
|
||||
},
|
||||
hasHeader: "첫 행을 헤더로 사용",
|
||||
trim: "값 앞뒤 공백 제거",
|
||||
emptyAsNull: "빈 셀을 null로 기록",
|
||||
|
||||
@@ -5224,6 +5224,18 @@ export default withEnglishFallback({
|
||||
typeString: "Tudo como texto",
|
||||
typeAuto: "Detectar automaticamente",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "Tipo de {name}",
|
||||
inferredType: {
|
||||
string: "Texto",
|
||||
boolean: "Booleano",
|
||||
integer: "Inteiro",
|
||||
decimal: "Decimal",
|
||||
date: "Data",
|
||||
objectId: "ObjectId",
|
||||
object: "Objeto",
|
||||
array: "Array",
|
||||
mixed: "Misto",
|
||||
},
|
||||
hasHeader: "A primeira linha é o cabeçalho",
|
||||
trim: "Remover espaços",
|
||||
emptyAsNull: "Células vazias como null",
|
||||
|
||||
@@ -5266,6 +5266,18 @@ export default withEnglishFallback({
|
||||
typeString: "Tümü metin",
|
||||
typeAuto: "Otomatik algıla",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "{name} türü",
|
||||
inferredType: {
|
||||
string: "Metin",
|
||||
boolean: "Mantıksal",
|
||||
integer: "Tam sayı",
|
||||
decimal: "Ondalık",
|
||||
date: "Tarih",
|
||||
objectId: "ObjectId",
|
||||
object: "Nesne",
|
||||
array: "Dizi",
|
||||
mixed: "Karışık",
|
||||
},
|
||||
hasHeader: "İlk satır başlıktır",
|
||||
trim: "Değerleri kırp",
|
||||
emptyAsNull: "Boş hücreleri null yaz",
|
||||
|
||||
@@ -5497,6 +5497,18 @@ export default withEnglishFallback({
|
||||
typeString: "全部按字符串",
|
||||
typeAuto: "自动推断",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "{name} 的类型",
|
||||
inferredType: {
|
||||
string: "字符串",
|
||||
boolean: "布尔",
|
||||
integer: "整数",
|
||||
decimal: "小数",
|
||||
date: "日期",
|
||||
objectId: "ObjectId",
|
||||
object: "对象",
|
||||
array: "数组",
|
||||
mixed: "混合",
|
||||
},
|
||||
hasHeader: "首行是表头",
|
||||
trim: "去除首尾空白",
|
||||
emptyAsNull: "空单元格写入 null",
|
||||
|
||||
@@ -4539,6 +4539,18 @@ export default withEnglishFallback({
|
||||
typeString: "全部按字串",
|
||||
typeAuto: "自動推斷",
|
||||
typeExtendedJson: "Extended JSON",
|
||||
columnType: "{name} 的類型",
|
||||
inferredType: {
|
||||
string: "字串",
|
||||
boolean: "布林",
|
||||
integer: "整數",
|
||||
decimal: "小數",
|
||||
date: "日期",
|
||||
objectId: "ObjectId",
|
||||
object: "物件",
|
||||
array: "陣列",
|
||||
mixed: "混合",
|
||||
},
|
||||
hasHeader: "首列是表頭",
|
||||
trim: "去除首尾空白",
|
||||
emptyAsNull: "空儲存格寫入 null",
|
||||
|
||||
@@ -1033,6 +1033,8 @@ export type {
|
||||
TableImportProgress,
|
||||
MongoImportFormat,
|
||||
MongoImportTypeMode,
|
||||
MongoImportInferredType,
|
||||
MongoImportColumn,
|
||||
MongoImportIssue,
|
||||
MongoImportParseOptions,
|
||||
MongoImportPreviewRequest,
|
||||
|
||||
@@ -5047,7 +5047,7 @@ export async function releaseTableImportSource(_sourceRef: string): Promise<bool
|
||||
|
||||
export type MongoImportFormat = "csv" | "json" | "ndjson";
|
||||
export type MongoImportTypeMode = "string" | "auto" | "extendedJson";
|
||||
export type MongoImportInferredType = "boolean" | "integer" | "decimal" | "date" | "object" | "array" | "string";
|
||||
export type MongoImportInferredType = "boolean" | "integer" | "decimal" | "date" | "objectId" | "object" | "array" | "mixed" | "string";
|
||||
export type MongoImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type MongoImportPhase = "preparing" | "parsing" | "writing" | "done";
|
||||
export type MongoExportFormat = "csv" | "ndjson";
|
||||
@@ -5072,6 +5072,7 @@ export interface MongoImportParseOptions {
|
||||
typeMode?: MongoImportTypeMode | null;
|
||||
recognizeObjectIdHex?: boolean | null;
|
||||
skipErrorRows?: boolean | null;
|
||||
columnTypes?: Partial<Record<string, MongoImportInferredType>> | null;
|
||||
}
|
||||
|
||||
export interface MongoImportPreviewRequest {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#![allow(clippy::result_large_err)]
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::io::{BufRead, BufReader, BufWriter, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -62,8 +62,10 @@ pub enum MongoImportInferredType {
|
||||
Integer,
|
||||
Decimal,
|
||||
Date,
|
||||
ObjectId,
|
||||
Object,
|
||||
Array,
|
||||
Mixed,
|
||||
String,
|
||||
}
|
||||
|
||||
@@ -174,6 +176,8 @@ pub struct MongoImportParseOptions {
|
||||
pub recognize_object_id_hex: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub skip_error_rows: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub column_types: Option<HashMap<String, MongoImportInferredType>>,
|
||||
}
|
||||
|
||||
impl Default for MongoImportParseOptions {
|
||||
@@ -187,6 +191,7 @@ impl Default for MongoImportParseOptions {
|
||||
type_mode: Some(MongoImportTypeMode::Auto),
|
||||
recognize_object_id_hex: Some(false),
|
||||
skip_error_rows: Some(false),
|
||||
column_types: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -219,6 +224,10 @@ impl MongoImportParseOptions {
|
||||
fn skip_error_rows(&self) -> bool {
|
||||
self.skip_error_rows.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn column_types(&self) -> HashMap<String, MongoImportInferredType> {
|
||||
self.column_types.clone().unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -327,6 +336,7 @@ struct CsvParseConfig {
|
||||
empty_as_null: bool,
|
||||
type_mode: MongoImportTypeMode,
|
||||
recognize_object_id_hex: bool,
|
||||
column_types: HashMap<String, MongoImportInferredType>,
|
||||
}
|
||||
|
||||
pub fn format_from_path(path: &str) -> Result<MongoImportFormat, String> {
|
||||
@@ -387,6 +397,7 @@ fn csv_config(path: &str, options: &MongoImportParseOptions) -> Result<CsvParseC
|
||||
empty_as_null: options.empty_as_null(),
|
||||
type_mode: options.type_mode(),
|
||||
recognize_object_id_hex: options.recognize_object_id_hex(),
|
||||
column_types: options.column_types(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -535,6 +546,16 @@ fn intersect_column_types(existing: Option<u8>, cell_mask: u8) -> u8 {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_object_id_cell(value: &str, column: &str, row: u64) -> Result<Bson, MongoImportIssue> {
|
||||
let oid = ObjectId::parse_str(value).map_err(|error| {
|
||||
MongoImportIssue::new("TYPE_CONVERSION", format!("Invalid ObjectId: {error}"))
|
||||
.with_row(row)
|
||||
.with_column(column)
|
||||
.with_value(value)
|
||||
})?;
|
||||
Ok(Bson::ObjectId(oid))
|
||||
}
|
||||
|
||||
fn convert_cell(
|
||||
value: Option<&str>,
|
||||
column: &str,
|
||||
@@ -545,21 +566,38 @@ fn convert_cell(
|
||||
let Some(value) = value.filter(|value| !value.is_empty()) else {
|
||||
return Ok(if config.empty_as_null { Bson::Null } else { Bson::String(String::new()) });
|
||||
};
|
||||
if let Some(override_type) = config.column_types.get(column).copied() {
|
||||
return convert_auto_cell(value, column, row, override_type, config);
|
||||
}
|
||||
// Without an explicit override, keep the legacy per-cell ObjectId behavior:
|
||||
// any 24-hex cell converts (mixed columns included), and a non-hex cell in
|
||||
// a uniform-hex column degrades to its own cell type instead of failing the
|
||||
// row with TYPE_CONVERSION.
|
||||
if (config.recognize_object_id_hex || (column == "_id" && config.type_mode != MongoImportTypeMode::String))
|
||||
&& is_object_id_hex(value)
|
||||
{
|
||||
let oid = ObjectId::parse_str(value).map_err(|error| {
|
||||
MongoImportIssue::new("TYPE_CONVERSION", format!("Invalid ObjectId: {error}"))
|
||||
.with_row(row)
|
||||
.with_column(column)
|
||||
.with_value(value)
|
||||
})?;
|
||||
return Ok(Bson::ObjectId(oid));
|
||||
return parse_object_id_cell(value, column, row);
|
||||
}
|
||||
let inferred =
|
||||
if inferred == MongoImportInferredType::ObjectId { MongoImportInferredType::Mixed } else { inferred };
|
||||
match config.type_mode {
|
||||
MongoImportTypeMode::String => Ok(Bson::String(value.to_string())),
|
||||
MongoImportTypeMode::Auto => convert_auto_cell(value, column, row, inferred),
|
||||
MongoImportTypeMode::ExtendedJson => convert_extended_json_cell(value, column, row),
|
||||
MongoImportTypeMode::Auto => convert_auto_cell(value, column, row, inferred, config),
|
||||
MongoImportTypeMode::ExtendedJson => {
|
||||
if (config.recognize_object_id_hex || column == "_id") && is_object_id_hex(value) {
|
||||
parse_object_id_cell(value, column, row)
|
||||
} else {
|
||||
convert_extended_json_cell(value, column, row)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn mixed_cell_type(value: &str, column: &str, config: &CsvParseConfig) -> MongoImportInferredType {
|
||||
if column_should_infer_object_id(column, config) && is_object_id_hex(value) {
|
||||
MongoImportInferredType::ObjectId
|
||||
} else {
|
||||
inferred_type_from_mask(classify_cell(value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -568,10 +606,15 @@ fn convert_auto_cell(
|
||||
column: &str,
|
||||
row: u64,
|
||||
inferred: MongoImportInferredType,
|
||||
config: &CsvParseConfig,
|
||||
) -> Result<Bson, MongoImportIssue> {
|
||||
let conversion_error = |message: String| {
|
||||
MongoImportIssue::new("TYPE_CONVERSION", message).with_row(row).with_column(column).with_value(value)
|
||||
};
|
||||
let inferred = match inferred {
|
||||
MongoImportInferredType::Mixed => mixed_cell_type(value, column, config),
|
||||
other => other,
|
||||
};
|
||||
match inferred {
|
||||
MongoImportInferredType::Boolean => {
|
||||
if is_boolean(value) {
|
||||
@@ -598,10 +641,11 @@ fn convert_auto_cell(
|
||||
let date = parse_date(value).ok_or_else(|| conversion_error(format!("Expected date, got {value}")))?;
|
||||
Ok(Bson::DateTime(date))
|
||||
}
|
||||
MongoImportInferredType::ObjectId => parse_object_id_cell(value, column, row),
|
||||
MongoImportInferredType::Object | MongoImportInferredType::Array => {
|
||||
parse_json_bson(value).map_err(conversion_error)
|
||||
}
|
||||
MongoImportInferredType::String => Ok(Bson::String(value.to_string())),
|
||||
MongoImportInferredType::Mixed | MongoImportInferredType::String => Ok(Bson::String(value.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -762,6 +806,10 @@ fn parsed_document(row: u64, document: Document, with_extended_json: bool) -> Pa
|
||||
/// Reads at most [`TYPE_SAMPLE_ROWS`] data rows to decide each column's type. Preview and
|
||||
/// execution both call this with the same bound, so the types shown in the wizard are the
|
||||
/// types the import actually writes.
|
||||
fn column_should_infer_object_id(name: &str, config: &CsvParseConfig) -> bool {
|
||||
config.recognize_object_id_hex || name == "_id"
|
||||
}
|
||||
|
||||
fn infer_csv_types(
|
||||
path: &str,
|
||||
config: &CsvParseConfig,
|
||||
@@ -772,21 +820,30 @@ fn infer_csv_types(
|
||||
let (reader, _) = open_transcoded_text_file(path, encoding).map_err(encoding_issue)?;
|
||||
let mut csv_reader = csv_reader(reader, config.delimiter);
|
||||
let mut record = csv::StringRecord::new();
|
||||
let mut headers = Vec::new();
|
||||
let mut masks: Vec<Option<u8>> = Vec::new();
|
||||
let mut all_object_id: Vec<bool> = Vec::new();
|
||||
let mut header_seen = false;
|
||||
let mut sampled = 0usize;
|
||||
while sampled < sample_rows && csv_reader.read_record(&mut record).map_err(csv_read_issue)? {
|
||||
if config.has_header && !header_seen {
|
||||
header_seen = true;
|
||||
masks = vec![None; unique_headers(&record_strings(&record))?.len()];
|
||||
headers = unique_headers(&record_strings(&record))?;
|
||||
masks = vec![None; headers.len()];
|
||||
all_object_id = vec![true; headers.len()];
|
||||
continue;
|
||||
}
|
||||
if masks.is_empty() {
|
||||
masks = vec![None; record.len().max(1)];
|
||||
all_object_id = vec![true; masks.len()];
|
||||
headers = generated_field_names(masks.len());
|
||||
}
|
||||
for (index, value) in record.iter().enumerate().take(masks.len()) {
|
||||
if let Some(text) = csv_cell_text(value, config) {
|
||||
masks[index] = Some(intersect_column_types(masks[index], classify_cell(&text)));
|
||||
if !is_object_id_hex(&text) {
|
||||
all_object_id[index] = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
sampled += 1;
|
||||
@@ -794,7 +851,22 @@ fn infer_csv_types(
|
||||
if !auto {
|
||||
return Ok(vec![MongoImportInferredType::String; masks.len()]);
|
||||
}
|
||||
Ok(masks.into_iter().map(|mask| inferred_type_from_mask(mask.unwrap_or(TYPE_STRING))).collect())
|
||||
Ok(masks
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, mask)| {
|
||||
let inferred = inferred_type_from_mask(mask.unwrap_or(TYPE_STRING));
|
||||
let name = headers.get(index).map(String::as_str).unwrap_or("");
|
||||
if mask.is_some()
|
||||
&& all_object_id.get(index).copied().unwrap_or(false)
|
||||
&& column_should_infer_object_id(name, config)
|
||||
{
|
||||
MongoImportInferredType::ObjectId
|
||||
} else {
|
||||
inferred
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn record_strings(record: &csv::StringRecord) -> Vec<String> {
|
||||
@@ -2618,13 +2690,174 @@ mod tests {
|
||||
fn objectid_hex_stays_string_unless_opted_in() {
|
||||
let csv = "id\n507f1f77bcf86cd799439011\n";
|
||||
let preview = preview_csv(csv, MongoImportTypeMode::Auto);
|
||||
assert_eq!(preview.columns[0].inferred_type, MongoImportInferredType::String);
|
||||
assert_eq!(preview.rows[0]["id"], "507f1f77bcf86cd799439011");
|
||||
let mut parse = options(MongoImportTypeMode::Auto);
|
||||
parse.recognize_object_id_hex = Some(true);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.columns[0].inferred_type, MongoImportInferredType::ObjectId);
|
||||
assert_eq!(preview.rows[0]["id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_mode_infers_id_hex_as_object_id() {
|
||||
let csv = "_id,name\n507f1f77bcf86cd799439011,Ada\n";
|
||||
let preview = preview_csv(csv, MongoImportTypeMode::Auto);
|
||||
assert_eq!(preview.columns[0].inferred_type, MongoImportInferredType::ObjectId);
|
||||
assert_eq!(preview.rows[0]["_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_id_column_converts_hex_cells_per_cell() {
|
||||
let csv = "_id\n507f1f77bcf86cd799439011\nnot-an-object-id\n";
|
||||
let preview = preview_csv(csv, MongoImportTypeMode::Auto);
|
||||
assert_eq!(preview.columns[0].inferred_type, MongoImportInferredType::String);
|
||||
assert_eq!(preview.rows[0]["_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
assert_eq!(preview.rows[1]["_id"], "not-an-object-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_hex_cell_in_uniform_object_id_column_degrades_to_string() {
|
||||
// The first TYPE_SAMPLE_ROWS rows make the column look uniformly hex;
|
||||
// a non-hex value after the sample must keep importing as a string
|
||||
// instead of failing the row with TYPE_CONVERSION.
|
||||
let hex = "507f1f77bcf86cd799439011";
|
||||
let mut csv = String::from("user_id\n");
|
||||
for _ in 0..TYPE_SAMPLE_ROWS {
|
||||
csv.push_str(hex);
|
||||
csv.push('\n');
|
||||
}
|
||||
csv.push_str("not-an-object-id\n");
|
||||
let mut parse = options(MongoImportTypeMode::Auto);
|
||||
parse.recognize_object_id_hex = Some(true);
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(executed.len(), TYPE_SAMPLE_ROWS + 1);
|
||||
assert_eq!(executed[0]["user_id"]["$oid"], hex);
|
||||
assert_eq!(executed[TYPE_SAMPLE_ROWS]["user_id"], "not-an-object-id");
|
||||
}
|
||||
|
||||
fn with_column_types(
|
||||
type_mode: MongoImportTypeMode,
|
||||
column_types: HashMap<String, MongoImportInferredType>,
|
||||
) -> MongoImportParseOptions {
|
||||
MongoImportParseOptions { column_types: Some(column_types), ..options(type_mode) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn column_type_override_keeps_inferred_type_and_converts_preview_and_execute() {
|
||||
let csv = "name,count\nAda,1\nBob,2\n";
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("count".to_string(), MongoImportInferredType::String)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.columns[1].inferred_type, MongoImportInferredType::Integer);
|
||||
assert_eq!(preview.rows[0]["count"], "1");
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn column_type_override_parses_object_id() {
|
||||
let csv = "user_id\n507f1f77bcf86cd799439011\n";
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("user_id".to_string(), MongoImportInferredType::ObjectId)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.columns[0].inferred_type, MongoImportInferredType::String);
|
||||
assert_eq!(preview.rows[0]["user_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn column_type_override_keeps_id_hex_as_string() {
|
||||
let csv = "_id,name\n507f1f77bcf86cd799439011,Ada\n";
|
||||
let auto = preview_csv(csv, MongoImportTypeMode::Auto);
|
||||
assert_eq!(auto.columns[0].inferred_type, MongoImportInferredType::ObjectId);
|
||||
assert_eq!(auto.rows[0]["_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("_id".to_string(), MongoImportInferredType::String)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.rows[0]["_id"], "507f1f77bcf86cd799439011");
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_column_type_converts_each_cell() {
|
||||
let csv = "value\n1\ntrue\n";
|
||||
let auto = preview_csv(csv, MongoImportTypeMode::Auto);
|
||||
assert_eq!(auto.columns[0].inferred_type, MongoImportInferredType::String);
|
||||
assert_eq!(auto.rows[0]["value"], "1");
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("value".to_string(), MongoImportInferredType::Mixed)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.rows[0]["value"]["$numberInt"], "1");
|
||||
assert_eq!(preview.rows[1]["value"], true);
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_column_type_writes_object_id_for_id_hex() {
|
||||
let csv = "_id,name\n507f1f77bcf86cd799439011,Ada\nnot-an-object-id,Bob\n";
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("_id".to_string(), MongoImportInferredType::Mixed)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.rows[0]["_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
assert_eq!(preview.rows[1]["_id"], "not-an-object-id");
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_column_type_writes_object_id_when_recognize_hex() {
|
||||
let csv = "user_id\n507f1f77bcf86cd799439011\nnot-an-object-id\n";
|
||||
let mut parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("user_id".to_string(), MongoImportInferredType::Mixed)]),
|
||||
);
|
||||
parse.recognize_object_id_hex = Some(true);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.rows[0]["user_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
assert_eq!(preview.rows[1]["user_id"], "not-an-object-id");
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_column_type_keeps_hex_string_without_object_id_recognition() {
|
||||
let csv = "user_id\n507f1f77bcf86cd799439011\n";
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("user_id".to_string(), MongoImportInferredType::Mixed)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.rows[0]["user_id"], "507f1f77bcf86cd799439011");
|
||||
let executed = execute_source(csv.as_bytes(), "csv", MongoImportFormat::Csv, &parse);
|
||||
assert_eq!(preview.rows, executed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn column_type_override_reports_conversion_error() {
|
||||
let csv = "count\nabc\n";
|
||||
let parse = with_column_types(
|
||||
MongoImportTypeMode::Auto,
|
||||
HashMap::from([("count".to_string(), MongoImportInferredType::Integer)]),
|
||||
);
|
||||
let preview = preview_mongodb_import_bytes(csv.as_bytes(), MongoImportFormat::Csv, &parse, 10).unwrap();
|
||||
assert_eq!(preview.errors[0].code, "TYPE_CONVERSION");
|
||||
assert_eq!(preview.errors[0].column.as_deref(), Some("count"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extended_json_mode_preserves_oid_date_long_and_decimal() {
|
||||
let csv = "id,when,n,d\n\"{\"\"$oid\"\":\"\"507f1f77bcf86cd799439011\"\"}\",\"{\"\"$date\"\":\"\"2024-01-02T03:04:05.000Z\"\"}\",\"{\"\"$numberLong\"\":\"\"9223372036854775807\"\"}\",\"{\"\"$numberDecimal\"\":\"\"1.25\"\"}\"\n";
|
||||
@@ -2789,6 +3022,7 @@ mod tests {
|
||||
empty_as_null: true,
|
||||
type_mode: MongoImportTypeMode::String,
|
||||
recognize_object_id_hex: false,
|
||||
column_types: HashMap::new(),
|
||||
};
|
||||
let mut count = 0u64;
|
||||
stream_csv_documents(
|
||||
@@ -2851,6 +3085,7 @@ mod tests {
|
||||
empty_as_null: true,
|
||||
type_mode: MongoImportTypeMode::ExtendedJson,
|
||||
recognize_object_id_hex: true,
|
||||
column_types: HashMap::new(),
|
||||
};
|
||||
|
||||
let mut reimported = Vec::new();
|
||||
|
||||
Reference in New Issue
Block a user