1 parent
d5dcdabd22
commit
5f8bcf74a5
12 files changed
+758
-110
No files matched your search
@@ -140,7 +140,7 @@ const titleRow = ref(1);
|
|||||||
const dataStartRow = ref(2);
|
const dataStartRow = ref(2);
|
||||||
const lastDataRow = ref(0);
|
const lastDataRow = ref(0);
|
||||||
const trimValues = ref(false);
|
const trimValues = ref(false);
|
||||||
const skipDuplicateRows = ref(false);
|
const conflictPolicy = ref<api.TableImportConflictPolicy>("error");
|
||||||
const emptyStringAsNull = ref(defaultTableImportEmptyStringAsNull(sourceFormat.value));
|
const emptyStringAsNull = ref(defaultTableImportEmptyStringAsNull(sourceFormat.value));
|
||||||
const selectedSheet = ref("");
|
const selectedSheet = ref("");
|
||||||
const jsonShape = ref<api.TableImportJsonShape>("auto");
|
const jsonShape = ref<api.TableImportJsonShape>("auto");
|
||||||
@@ -176,10 +176,10 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
|
|||||||
|
|
||||||
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
|
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
|
||||||
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
|
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
|
||||||
// Mirrors the dialect list in skip-generator dispatch (transfer.rs): only these
|
// Mirrors the conflict SQL dispatch in transfer.rs. Other dialects must keep
|
||||||
// engines turn the skip-duplicates option into conflict-handling INSERTs.
|
// ordinary INSERT/error behavior instead of approximating an upsert.
|
||||||
const SKIP_DUPLICATE_ROWS_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
|
const IMPORT_CONFLICT_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
|
||||||
const supportsSkipDuplicateRows = computed(() => structureDatabaseType.value !== undefined && SKIP_DUPLICATE_ROWS_DATABASE_TYPES.has(structureDatabaseType.value));
|
const supportsImportConflictPolicy = computed(() => structureDatabaseType.value !== undefined && IMPORT_CONFLICT_DATABASE_TYPES.has(structureDatabaseType.value));
|
||||||
const targetSchema = computed(() => metadataSchemaForConnection(selectedConnection.value, props.prefillDatabase || "", props.prefillSchema));
|
const targetSchema = computed(() => metadataSchemaForConnection(selectedConnection.value, props.prefillDatabase || "", props.prefillSchema));
|
||||||
const dataTypeOptions = computed(() => mergeDataTypeOptions(dynamicDataTypeOptions.value, getDataTypeOptions(structureDatabaseType.value), Object.values(columnDataTypes.value)));
|
const dataTypeOptions = computed(() => mergeDataTypeOptions(dynamicDataTypeOptions.value, getDataTypeOptions(structureDatabaseType.value), Object.values(columnDataTypes.value)));
|
||||||
const hasExistingTarget = computed(() => !!props.prefillTable || loadingExistingTables.value || existingTableNames.value.length > 0);
|
const hasExistingTarget = computed(() => !!props.prefillTable || loadingExistingTables.value || existingTableNames.value.length > 0);
|
||||||
@@ -202,6 +202,20 @@ const mappedColumns = computed<api.TableImportColumnMapping[]>(() => {
|
|||||||
});
|
});
|
||||||
const mappedCount = computed(() => mappedColumns.value.length);
|
const mappedCount = computed(() => mappedColumns.value.length);
|
||||||
const mappingValidation = computed(() => validateImportMappings(mappedColumns.value));
|
const mappingValidation = computed(() => validateImportMappings(mappedColumns.value));
|
||||||
|
const primaryKeyColumns = computed(() => targetColumns.value.filter((column) => column.is_primary_key));
|
||||||
|
const updateExistingUnavailableReason = computed(() => {
|
||||||
|
if (targetMode.value !== "existing") return t("tableImport.updateExistingRequiresExistingTable");
|
||||||
|
if (!supportsImportConflictPolicy.value) return t("tableImport.updateExistingUnsupportedDialect");
|
||||||
|
if (!existingTargetMetadataReady.value || primaryKeyColumns.value.length === 0) return t("tableImport.updateExistingRequiresPrimaryKey");
|
||||||
|
const mappedTargets = mappedColumns.value.map((mapping) => mapping.targetColumn.toLowerCase());
|
||||||
|
const missingKeys = primaryKeyColumns.value.filter((column) => !mappedTargets.includes(column.name.toLowerCase()));
|
||||||
|
if (missingKeys.length > 0) return t("tableImport.updateExistingRequiresMappedPrimaryKey", { columns: missingKeys.map((column) => column.name).join(", ") });
|
||||||
|
const primaryKeyNames = primaryKeyColumns.value.map((column) => column.name.toLowerCase());
|
||||||
|
if (!mappedTargets.some((column) => !primaryKeyNames.includes(column))) return t("tableImport.updateExistingRequiresMappedValue");
|
||||||
|
return "";
|
||||||
|
});
|
||||||
|
const canUpdateExistingRows = computed(() => updateExistingUnavailableReason.value === "");
|
||||||
|
const conflictPolicyError = computed(() => (conflictPolicy.value === "updateExisting" ? updateExistingUnavailableReason.value : ""));
|
||||||
const requiredUnmappedColumns = computed(() =>
|
const requiredUnmappedColumns = computed(() =>
|
||||||
requiredImportTargetColumns(
|
requiredImportTargetColumns(
|
||||||
targetColumns.value,
|
targetColumns.value,
|
||||||
@@ -215,7 +229,7 @@ const batchTargetNamesValid = computed(() => {
|
|||||||
return tableNames.length > 0 && tableNames.every(Boolean) && new Set(tableNames).size === tableNames.length;
|
return tableNames.length > 0 && tableNames.every(Boolean) && new Set(tableNames).size === tableNames.length;
|
||||||
});
|
});
|
||||||
const canImport = computed(() => {
|
const canImport = computed(() => {
|
||||||
if (running.value || !props.prefillConnectionId || !existingTargetMetadataReady.value) return false;
|
if (running.value || !props.prefillConnectionId || !existingTargetMetadataReady.value || conflictPolicyError.value) return false;
|
||||||
if (!isBatchImport.value) return !!preview.value && !!targetTableName.value && mappingValidation.value.valid;
|
if (!isBatchImport.value) return !!preview.value && !!targetTableName.value && mappingValidation.value.valid;
|
||||||
if (!batchTargetNamesValid.value) return false;
|
if (!batchTargetNamesValid.value) return false;
|
||||||
return selectedBatchTasks.value.every((task) => {
|
return selectedBatchTasks.value.every((task) => {
|
||||||
@@ -231,7 +245,7 @@ const canGoNext = computed(() => {
|
|||||||
if (wizardStep.value === "mapping") return canImport.value;
|
if (wizardStep.value === "mapping") return canImport.value;
|
||||||
}
|
}
|
||||||
if (wizardStep.value === "options") return !!preview.value && !!targetTableName.value && existingTargetMetadataReady.value;
|
if (wizardStep.value === "options") return !!preview.value && !!targetTableName.value && existingTargetMetadataReady.value;
|
||||||
if (wizardStep.value === "mapping") return existingTargetMetadataReady.value && mappingValidation.value.valid;
|
if (wizardStep.value === "mapping") return existingTargetMetadataReady.value && mappingValidation.value.valid && !conflictPolicyError.value;
|
||||||
return false;
|
return false;
|
||||||
});
|
});
|
||||||
const rawProgressPercent = computed(() => tableImportProgressPercent(progress.value));
|
const rawProgressPercent = computed(() => tableImportProgressPercent(progress.value));
|
||||||
@@ -384,7 +398,7 @@ function resetState() {
|
|||||||
dataStartRow.value = 2;
|
dataStartRow.value = 2;
|
||||||
lastDataRow.value = 0;
|
lastDataRow.value = 0;
|
||||||
trimValues.value = false;
|
trimValues.value = false;
|
||||||
skipDuplicateRows.value = false;
|
conflictPolicy.value = "error";
|
||||||
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
|
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
|
||||||
selectedSheet.value = "";
|
selectedSheet.value = "";
|
||||||
jsonShape.value = "auto";
|
jsonShape.value = "auto";
|
||||||
@@ -851,7 +865,7 @@ function canOpenStep(step: TableImportWizardStep) {
|
|||||||
if (step === "review") return canImport.value;
|
if (step === "review") return canImport.value;
|
||||||
}
|
}
|
||||||
if (step === "mapping") return !!preview.value && existingTargetMetadataReady.value;
|
if (step === "mapping") return !!preview.value && existingTargetMetadataReady.value;
|
||||||
if (step === "review") return !!preview.value && existingTargetMetadataReady.value && mappingValidation.value.valid;
|
if (step === "review") return !!preview.value && existingTargetMetadataReady.value && mappingValidation.value.valid && !conflictPolicyError.value;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -941,7 +955,8 @@ async function startImport() {
|
|||||||
mode: targetMode.value === "create" ? "append" : importMode.value,
|
mode: targetMode.value === "create" ? "append" : importMode.value,
|
||||||
createTable: targetMode.value === "create",
|
createTable: targetMode.value === "create",
|
||||||
batchSize: Math.max(1, Number(batchSize.value) || 500),
|
batchSize: Math.max(1, Number(batchSize.value) || 500),
|
||||||
skipDuplicateRows: supportsSkipDuplicateRows.value && skipDuplicateRows.value,
|
conflictPolicy: conflictPolicy.value,
|
||||||
|
skipDuplicateRows: conflictPolicy.value === "skip",
|
||||||
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
|
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
|
||||||
preparedSource: preparedImportSource(currentPreview),
|
preparedSource: preparedImportSource(currentPreview),
|
||||||
},
|
},
|
||||||
@@ -1031,7 +1046,8 @@ async function startBatchImport() {
|
|||||||
mode: "append",
|
mode: "append",
|
||||||
createTable: true,
|
createTable: true,
|
||||||
batchSize: Math.max(1, Number(batchSize.value) || 500),
|
batchSize: Math.max(1, Number(batchSize.value) || 500),
|
||||||
skipDuplicateRows: supportsSkipDuplicateRows.value && skipDuplicateRows.value,
|
conflictPolicy: conflictPolicy.value,
|
||||||
|
skipDuplicateRows: conflictPolicy.value === "skip",
|
||||||
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
|
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
|
||||||
preparedSource: preparedImportSource(task.preview),
|
preparedSource: preparedImportSource(task.preview),
|
||||||
retainSource: true,
|
retainSource: true,
|
||||||
@@ -1198,13 +1214,17 @@ watch(targetMode, (mode) => {
|
|||||||
loadedTargetTableName.value = "";
|
loadedTargetTableName.value = "";
|
||||||
loadingTarget.value = false;
|
loadingTarget.value = false;
|
||||||
importMode.value = "append";
|
importMode.value = "append";
|
||||||
|
conflictPolicy.value = "error";
|
||||||
applyAutoMapping();
|
applyAutoMapping();
|
||||||
applySuggestedColumnDataTypes();
|
applySuggestedColumnDataTypes();
|
||||||
void loadDataTypeOptions();
|
void loadDataTypeOptions();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
watch(selectedExistingTable, () => {
|
watch(selectedExistingTable, () => {
|
||||||
if (targetMode.value === "existing") void loadTargetColumns();
|
if (targetMode.value === "existing") {
|
||||||
|
conflictPolicy.value = "error";
|
||||||
|
void loadTargetColumns();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
watch(rawProgressPercent, (percent) => {
|
watch(rawProgressPercent, (percent) => {
|
||||||
@@ -1422,10 +1442,6 @@ watch(rawProgressPercent, (percent) => {
|
|||||||
<input v-model="emptyStringAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
<input v-model="emptyStringAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
||||||
{{ t("tableImport.emptyStringAsNull") }}
|
{{ t("tableImport.emptyStringAsNull") }}
|
||||||
</label>
|
</label>
|
||||||
<label v-if="supportsSkipDuplicateRows" class="flex items-center gap-2 text-xs">
|
|
||||||
<input v-model="skipDuplicateRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
|
||||||
{{ t("tableImport.skipDuplicateRows") }}
|
|
||||||
</label>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else-if="sourceFormat === 'sql'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
<div v-else-if="sourceFormat === 'sql'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
||||||
@@ -1490,6 +1506,20 @@ watch(rawProgressPercent, (percent) => {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div v-if="supportsImportConflictPolicy" class="space-y-1.5 rounded-md border p-3">
|
||||||
|
<Label for="table-import-conflict-policy" class="text-xs">{{ t("tableImport.conflictPolicy") }}</Label>
|
||||||
|
<select id="table-import-conflict-policy" v-model="conflictPolicy" data-testid="table-import-conflict-policy" class="flex h-8 w-full rounded-md border border-input bg-background px-2 text-xs">
|
||||||
|
<option value="error">{{ t("tableImport.conflictError") }}</option>
|
||||||
|
<option value="skip">{{ t("tableImport.skipDuplicateRows") }}</option>
|
||||||
|
<option v-if="targetMode === 'existing'" value="updateExisting" :disabled="!canUpdateExistingRows">
|
||||||
|
{{ t("tableImport.updateExistingRows") }}
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
<p v-if="targetMode === 'existing' && !canUpdateExistingRows" class="text-[11px] text-muted-foreground">
|
||||||
|
{{ updateExistingUnavailableReason }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="flex items-center gap-2">
|
<div class="flex items-center gap-2">
|
||||||
<Button size="sm" :disabled="!selectedSource || loadingPreview" @click="loadPreview()">
|
<Button size="sm" :disabled="!selectedSource || loadingPreview" @click="loadPreview()">
|
||||||
<Loader2 v-if="loadingPreview" class="mr-1.5 h-3.5 w-3.5 animate-spin" />
|
<Loader2 v-if="loadingPreview" class="mr-1.5 h-3.5 w-3.5 animate-spin" />
|
||||||
@@ -1597,6 +1627,9 @@ watch(rawProgressPercent, (percent) => {
|
|||||||
<div v-if="mappingValidation.errors.length" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">
|
<div v-if="mappingValidation.errors.length" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">
|
||||||
{{ mappingValidation.errors.join("; ") }}
|
{{ mappingValidation.errors.join("; ") }}
|
||||||
</div>
|
</div>
|
||||||
|
<div v-else-if="conflictPolicyError" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">
|
||||||
|
{{ conflictPolicyError }}
|
||||||
|
</div>
|
||||||
<div v-else-if="requiredUnmappedColumns.length" class="flex items-start gap-2 rounded-md border border-amber-300 bg-amber-50 px-3 py-2 text-xs text-amber-800 dark:bg-amber-950/20 dark:text-amber-300">
|
<div v-else-if="requiredUnmappedColumns.length" class="flex items-start gap-2 rounded-md border border-amber-300 bg-amber-50 px-3 py-2 text-xs text-amber-800 dark:bg-amber-950/20 dark:text-amber-300">
|
||||||
<AlertTriangle class="mt-0.5 h-3.5 w-3.5 shrink-0" />
|
<AlertTriangle class="mt-0.5 h-3.5 w-3.5 shrink-0" />
|
||||||
<span>{{ t("tableImport.requiredUnmapped", { columns: requiredUnmappedColumns.join(", ") }) }}</span>
|
<span>{{ t("tableImport.requiredUnmapped", { columns: requiredUnmappedColumns.join(", ") }) }}</span>
|
||||||
|
|||||||
+20
@@ -294,6 +294,7 @@ describe("TableImportDialog batch selection", () => {
|
|||||||
await flushAsyncUpdates();
|
await flushAsyncUpdates();
|
||||||
expect(document.body.textContent).toContain("Trim values");
|
expect(document.body.textContent).toContain("Trim values");
|
||||||
expect(document.body.textContent).not.toContain("Skip duplicate rows");
|
expect(document.body.textContent).not.toContain("Skip duplicate rows");
|
||||||
|
expect(document.body.textContent).not.toContain("Update the existing row");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("offers the skip-duplicates option for dialects with conflict handling", async () => {
|
it("offers the skip-duplicates option for dialects with conflict handling", async () => {
|
||||||
@@ -304,6 +305,25 @@ describe("TableImportDialog batch selection", () => {
|
|||||||
expect(document.body.textContent).toContain("Skip duplicate rows");
|
expect(document.body.textContent).toContain("Skip duplicate rows");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("serializes the explicit skip policy with the legacy compatibility flag", async () => {
|
||||||
|
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve(delimitedPreview()));
|
||||||
|
await mountDialog([new File(["id,name\n1,a"], "rows.csv")]);
|
||||||
|
const policySelect = document.body.querySelector<HTMLSelectElement>('[data-testid="table-import-conflict-policy"]');
|
||||||
|
expect(policySelect).toBeTruthy();
|
||||||
|
if (policySelect) {
|
||||||
|
policySelect.value = "skip";
|
||||||
|
policySelect.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
|
}
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
|
||||||
|
await startImport();
|
||||||
|
expect(mocks.importTableFile).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mocks.importTableFile.mock.calls[0]![0]).toMatchObject({
|
||||||
|
conflictPolicy: "skip",
|
||||||
|
skipDuplicateRows: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps every worksheet selected by default and imports them in workbook order", async () => {
|
it("keeps every worksheet selected by default and imports them in workbook order", async () => {
|
||||||
await mountDialog();
|
await mountDialog();
|
||||||
for (const name of ["rows_First", "rows_Second", "rows_Third"]) expect(taskCheckbox(name).checked).toBe(true);
|
for (const name of ["rows_First", "rows_Second", "rows_Third"]) expect(taskCheckbox(name).checked).toBe(true);
|
||||||
|
|||||||
+73
-6
@@ -3,6 +3,7 @@
|
|||||||
import { createApp, defineComponent, h, nextTick, type App } from "vue";
|
import { createApp, defineComponent, h, nextTick, type App } from "vue";
|
||||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import i18n from "@/i18n";
|
import i18n from "@/i18n";
|
||||||
|
import type { TableImportProgress, TableImportRequest, TableImportSummary } from "@/lib/backend/api";
|
||||||
|
|
||||||
const mocks = vi.hoisted(() => ({
|
const mocks = vi.hoisted(() => ({
|
||||||
ensureConnected: vi.fn().mockResolvedValue(undefined),
|
ensureConnected: vi.fn().mockResolvedValue(undefined),
|
||||||
@@ -11,8 +12,8 @@ const mocks = vi.hoisted(() => ({
|
|||||||
{ name: "archived_target", table_type: "TABLE" },
|
{ name: "archived_target", table_type: "TABLE" },
|
||||||
]),
|
]),
|
||||||
getColumns: vi.fn().mockResolvedValue([
|
getColumns: vi.fn().mockResolvedValue([
|
||||||
{ name: "id", data_type: "INTEGER", nullable: false },
|
{ name: "id", data_type: "INTEGER", nullable: false, is_primary_key: true },
|
||||||
{ name: "name", data_type: "TEXT", nullable: true },
|
{ name: "name", data_type: "TEXT", nullable: true, is_primary_key: false },
|
||||||
]),
|
]),
|
||||||
listDataTypes: vi.fn().mockResolvedValue(["INTEGER", "TEXT"]),
|
listDataTypes: vi.fn().mockResolvedValue(["INTEGER", "TEXT"]),
|
||||||
previewTableImportFile: vi.fn().mockResolvedValue({
|
previewTableImportFile: vi.fn().mockResolvedValue({
|
||||||
@@ -26,6 +27,7 @@ const mocks = vi.hoisted(() => ({
|
|||||||
sourceFingerprint: "rows-xlsx",
|
sourceFingerprint: "rows-xlsx",
|
||||||
sheets: ["Data"],
|
sheets: ["Data"],
|
||||||
}),
|
}),
|
||||||
|
importTableFile: vi.fn<(request: TableImportRequest, onProgress: (progress: TableImportProgress) => void) => Promise<TableImportSummary>>(),
|
||||||
releaseTableImportSource: vi.fn().mockResolvedValue(undefined),
|
releaseTableImportSource: vi.fn().mockResolvedValue(undefined),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -52,7 +54,7 @@ vi.mock("@/lib/backend/api", () => ({
|
|||||||
listDataTypes: mocks.listDataTypes,
|
listDataTypes: mocks.listDataTypes,
|
||||||
previewTableImportFile: mocks.previewTableImportFile,
|
previewTableImportFile: mocks.previewTableImportFile,
|
||||||
releaseTableImportSource: mocks.releaseTableImportSource,
|
releaseTableImportSource: mocks.releaseTableImportSource,
|
||||||
importTableFile: vi.fn(),
|
importTableFile: mocks.importTableFile,
|
||||||
cancelTableImport: vi.fn(),
|
cancelTableImport: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -208,7 +210,12 @@ async function selectWorkbook() {
|
|||||||
value: [new File(["test workbook"], "rows.xlsx", { type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" })],
|
value: [new File(["test workbook"], "rows.xlsx", { type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" })],
|
||||||
});
|
});
|
||||||
fileInput?.dispatchEvent(new Event("change", { bubbles: true }));
|
fileInput?.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
await vi.waitFor(() => expect(mocks.previewTableImportFile).toHaveBeenCalled());
|
await flushAsyncUpdates();
|
||||||
|
if (!mocks.previewTableImportFile.mock.calls.length) {
|
||||||
|
buttonContaining("Load Preview")?.click();
|
||||||
|
}
|
||||||
|
await vi.waitFor(() => expect(mocks.previewTableImportFile).toHaveBeenCalled(), { timeout: 3000 });
|
||||||
|
await flushAsyncUpdates();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function selectExistingTarget(tableName: string) {
|
async function selectExistingTarget(tableName: string) {
|
||||||
@@ -232,11 +239,15 @@ beforeEach(() => {
|
|||||||
{ name: "archived_target", table_type: "TABLE" },
|
{ name: "archived_target", table_type: "TABLE" },
|
||||||
]);
|
]);
|
||||||
mocks.getColumns.mockReset().mockResolvedValue([
|
mocks.getColumns.mockReset().mockResolvedValue([
|
||||||
{ name: "id", data_type: "INTEGER", nullable: false },
|
{ name: "id", data_type: "INTEGER", nullable: false, is_primary_key: true },
|
||||||
{ name: "name", data_type: "TEXT", nullable: true },
|
{ name: "name", data_type: "TEXT", nullable: true, is_primary_key: false },
|
||||||
]);
|
]);
|
||||||
mocks.listDataTypes.mockClear();
|
mocks.listDataTypes.mockClear();
|
||||||
mocks.previewTableImportFile.mockClear();
|
mocks.previewTableImportFile.mockClear();
|
||||||
|
mocks.importTableFile.mockReset().mockImplementation(async (request, onProgress) => {
|
||||||
|
onProgress({ importId: request.importId, status: "done", rowsImported: 1, totalRows: 1, elapsedMs: 1 });
|
||||||
|
return { importId: request.importId, rowsImported: 1, totalRows: 1, elapsedMs: 1 };
|
||||||
|
});
|
||||||
mocks.releaseTableImportSource.mockClear();
|
mocks.releaseTableImportSource.mockClear();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -346,4 +357,60 @@ describe("TableImportDialog existing targets", () => {
|
|||||||
expect(mocks.listTables).not.toHaveBeenCalled();
|
expect(mocks.listTables).not.toHaveBeenCalled();
|
||||||
expect(mocks.getColumns).toHaveBeenCalledWith("connection-1", "main", "main", "existing_target");
|
expect(mocks.getColumns).toHaveBeenCalledWith("connection-1", "main", "main", "existing_target");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("offers update only with primary-key metadata and serializes the explicit policy", async () => {
|
||||||
|
i18n.global.locale.value = "en";
|
||||||
|
await mountDialog({ schema: "main" });
|
||||||
|
await selectWorkbook();
|
||||||
|
buttonContaining("Existing table")?.click();
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
await selectExistingTarget("existing_target");
|
||||||
|
await vi.waitFor(() => expect(buttonContaining("Next")?.disabled).toBe(false));
|
||||||
|
|
||||||
|
const policySelect = document.body.querySelector<HTMLSelectElement>('[data-testid="table-import-conflict-policy"]');
|
||||||
|
expect(policySelect).toBeTruthy();
|
||||||
|
const updateOption = policySelect?.querySelector<HTMLOptionElement>('option[value="updateExisting"]');
|
||||||
|
await vi.waitFor(() => expect(updateOption?.disabled, document.body.textContent || "").toBe(false));
|
||||||
|
if (policySelect) {
|
||||||
|
policySelect.value = "updateExisting";
|
||||||
|
policySelect.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
|
}
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
|
||||||
|
buttonContaining("Next")?.click();
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
buttonContaining("Next")?.click();
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
buttonContaining("Start Import")?.click();
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
|
||||||
|
expect(mocks.importTableFile).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mocks.importTableFile.mock.calls[0]![0]).toMatchObject({
|
||||||
|
conflictPolicy: "updateExisting",
|
||||||
|
skipDuplicateRows: false,
|
||||||
|
mappings: [
|
||||||
|
{ sourceColumn: "id", targetColumn: "id" },
|
||||||
|
{ sourceColumn: "name", targetColumn: "name" },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("disables update when target primary-key metadata is absent", async () => {
|
||||||
|
mocks.getColumns.mockResolvedValueOnce([
|
||||||
|
{ name: "id", data_type: "INTEGER", nullable: false, is_primary_key: false },
|
||||||
|
{ name: "name", data_type: "TEXT", nullable: true, is_primary_key: false },
|
||||||
|
]);
|
||||||
|
i18n.global.locale.value = "en";
|
||||||
|
await mountDialog({ schema: "main" });
|
||||||
|
await selectWorkbook();
|
||||||
|
buttonContaining("Existing table")?.click();
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
await selectExistingTarget("existing_target");
|
||||||
|
await vi.waitFor(() => expect(mocks.getColumns).toHaveBeenCalled());
|
||||||
|
await flushAsyncUpdates();
|
||||||
|
|
||||||
|
const updateOption = document.body.querySelector<HTMLOptionElement>('option[value="updateExisting"]');
|
||||||
|
expect(updateOption?.disabled).toBe(true);
|
||||||
|
expect(document.body.textContent).toContain("requires primary-key metadata");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -6856,7 +6856,15 @@ export default {
|
|||||||
lastDataRow: "Last data row (0 for end)",
|
lastDataRow: "Last data row (0 for end)",
|
||||||
trimValues: "Trim values",
|
trimValues: "Trim values",
|
||||||
emptyStringAsNull: "Empty string as NULL",
|
emptyStringAsNull: "Empty string as NULL",
|
||||||
|
conflictPolicy: "On primary-key conflict",
|
||||||
|
conflictError: "Stop with an error",
|
||||||
skipDuplicateRows: "Skip duplicate rows",
|
skipDuplicateRows: "Skip duplicate rows",
|
||||||
|
updateExistingRows: "Update the existing row",
|
||||||
|
updateExistingRequiresExistingTable: "Updating conflicts requires an existing target table.",
|
||||||
|
updateExistingUnsupportedDialect: "Updating conflicts is not supported for this database.",
|
||||||
|
updateExistingRequiresPrimaryKey: "Updating conflicts requires primary-key metadata for the target table.",
|
||||||
|
updateExistingRequiresMappedPrimaryKey: "Map every primary-key column to update conflicts. Missing: {columns}",
|
||||||
|
updateExistingRequiresMappedValue: "Map at least one non-primary-key column to update conflicts.",
|
||||||
jsonShape: "JSON rows",
|
jsonShape: "JSON rows",
|
||||||
jsonShapeAuto: "Auto-detect",
|
jsonShapeAuto: "Auto-detect",
|
||||||
jsonShapeObjects: "Objects",
|
jsonShapeObjects: "Objects",
|
||||||
|
|||||||
@@ -6823,6 +6823,15 @@ export default withEnglishFallback({
|
|||||||
lastDataRow: "最后数据行(0 表示末尾)",
|
lastDataRow: "最后数据行(0 表示末尾)",
|
||||||
trimValues: "裁剪空白",
|
trimValues: "裁剪空白",
|
||||||
emptyStringAsNull: "空字符串作为 NULL",
|
emptyStringAsNull: "空字符串作为 NULL",
|
||||||
|
conflictPolicy: "主键冲突时",
|
||||||
|
conflictError: "停止并报错",
|
||||||
|
skipDuplicateRows: "跳过重复行",
|
||||||
|
updateExistingRows: "更新已有行",
|
||||||
|
updateExistingRequiresExistingTable: "冲突更新仅适用于已有目标表。",
|
||||||
|
updateExistingUnsupportedDialect: "当前数据库不支持冲突更新。",
|
||||||
|
updateExistingRequiresPrimaryKey: "冲突更新需要目标表的主键元数据。",
|
||||||
|
updateExistingRequiresMappedPrimaryKey: "冲突更新需要映射所有主键列,缺少:{columns}",
|
||||||
|
updateExistingRequiresMappedValue: "冲突更新需要至少映射一个非主键列。",
|
||||||
jsonShape: "JSON 行结构",
|
jsonShape: "JSON 行结构",
|
||||||
jsonShapeAuto: "自动检测",
|
jsonShapeAuto: "自动检测",
|
||||||
jsonShapeObjects: "对象",
|
jsonShapeObjects: "对象",
|
||||||
|
|||||||
@@ -5525,6 +5525,15 @@ export default withEnglishFallback({
|
|||||||
hasHeader: "首列為標頭",
|
hasHeader: "首列為標頭",
|
||||||
trimValues: "裁剪空白",
|
trimValues: "裁剪空白",
|
||||||
emptyStringAsNull: "空字串作為 NULL",
|
emptyStringAsNull: "空字串作為 NULL",
|
||||||
|
conflictPolicy: "主鍵衝突時",
|
||||||
|
conflictError: "停止並回報錯誤",
|
||||||
|
skipDuplicateRows: "略過重複列",
|
||||||
|
updateExistingRows: "更新既有列",
|
||||||
|
updateExistingRequiresExistingTable: "衝突更新僅適用於既有目標資料表。",
|
||||||
|
updateExistingUnsupportedDialect: "目前資料庫不支援衝突更新。",
|
||||||
|
updateExistingRequiresPrimaryKey: "衝突更新需要目標資料表的主鍵中繼資料。",
|
||||||
|
updateExistingRequiresMappedPrimaryKey: "衝突更新需要映射所有主鍵欄位,缺少:{columns}",
|
||||||
|
updateExistingRequiresMappedValue: "衝突更新需要至少映射一個非主鍵欄位。",
|
||||||
jsonShape: "JSON 列結構",
|
jsonShape: "JSON 列結構",
|
||||||
jsonShapeAuto: "自動偵測",
|
jsonShapeAuto: "自動偵測",
|
||||||
jsonShapeObjects: "物件",
|
jsonShapeObjects: "物件",
|
||||||
|
|||||||
@@ -1094,6 +1094,7 @@ export type {
|
|||||||
TransferOwnershipPolicy,
|
TransferOwnershipPolicy,
|
||||||
TransferOwnershipPreview,
|
TransferOwnershipPreview,
|
||||||
TableImportMode,
|
TableImportMode,
|
||||||
|
TableImportConflictPolicy,
|
||||||
TableImportStatus,
|
TableImportStatus,
|
||||||
TableImportSourceFormat,
|
TableImportSourceFormat,
|
||||||
TableImportJsonShape,
|
TableImportJsonShape,
|
||||||
|
|||||||
@@ -5375,6 +5375,7 @@ export async function sortTablesByFkDependency(options: SortTablesByFkOptions):
|
|||||||
|
|
||||||
// --- Table File Import ---
|
// --- Table File Import ---
|
||||||
export type TableImportMode = "append" | "truncate";
|
export type TableImportMode = "append" | "truncate";
|
||||||
|
export type TableImportConflictPolicy = "error" | "skip" | "updateExisting";
|
||||||
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
|
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
|
||||||
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
||||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql";
|
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql";
|
||||||
@@ -5451,6 +5452,7 @@ export interface TableImportRequest {
|
|||||||
dateTimeFormat?: string;
|
dateTimeFormat?: string;
|
||||||
preparedSource?: TableImportPreparedSource | null;
|
preparedSource?: TableImportPreparedSource | null;
|
||||||
retainSource?: boolean;
|
retainSource?: boolean;
|
||||||
|
conflictPolicy?: TableImportConflictPolicy;
|
||||||
skipDuplicateRows?: boolean;
|
skipDuplicateRows?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -406,6 +406,7 @@ fn import_request(
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
|
conflict_policy: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,8 @@ use crate::sql::SqlParsingOptions;
|
|||||||
use crate::sql_file_import::{SqlFileStreamDecoder, StreamingSqlFileSplitter};
|
use crate::sql_file_import::{SqlFileStreamDecoder, StreamingSqlFileSplitter};
|
||||||
use crate::transfer::{
|
use crate::transfer::{
|
||||||
escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows, get_columns_for_transfer,
|
escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows, get_columns_for_transfer,
|
||||||
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
|
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier,
|
||||||
|
supports_primary_key_upsert, SqlBatchLimits,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
|
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
|
||||||
@@ -245,6 +246,15 @@ pub struct TableImportPreviewRequest {
|
|||||||
pub preview_limit: Option<usize>,
|
pub preview_limit: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub enum TableImportConflictPolicy {
|
||||||
|
#[default]
|
||||||
|
Error,
|
||||||
|
Skip,
|
||||||
|
UpdateExisting,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct TableImportRequest {
|
pub struct TableImportRequest {
|
||||||
@@ -271,10 +281,83 @@ pub struct TableImportRequest {
|
|||||||
pub prepared_source: Option<TableImportPreparedSource>,
|
pub prepared_source: Option<TableImportPreparedSource>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub retain_source: bool,
|
pub retain_source: bool,
|
||||||
|
/// Explicit conflict behavior for new callers. When omitted, the legacy
|
||||||
|
/// `skip_duplicate_rows` flag remains authoritative for compatibility.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub conflict_policy: Option<TableImportConflictPolicy>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub skip_duplicate_rows: bool,
|
pub skip_duplicate_rows: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl TableImportRequest {
|
||||||
|
fn effective_conflict_policy(&self) -> TableImportConflictPolicy {
|
||||||
|
self.conflict_policy.unwrap_or(if self.skip_duplicate_rows {
|
||||||
|
TableImportConflictPolicy::Skip
|
||||||
|
} else {
|
||||||
|
TableImportConflictPolicy::Error
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_update_existing_target(
|
||||||
|
conflict_policy: TableImportConflictPolicy,
|
||||||
|
db_type: &DatabaseType,
|
||||||
|
create_table: bool,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
if conflict_policy != TableImportConflictPolicy::UpdateExisting {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if create_table {
|
||||||
|
return Err("Update-existing import requires an existing target table".to_string());
|
||||||
|
}
|
||||||
|
if !supports_primary_key_upsert(db_type) {
|
||||||
|
return Err(format!("Update-existing import conflict policy is not supported for {}", db_type.as_str()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_existing_primary_key_columns(
|
||||||
|
conflict_policy: TableImportConflictPolicy,
|
||||||
|
mappings: &[TableImportColumnMapping],
|
||||||
|
target_columns: &[crate::db::ColumnInfo],
|
||||||
|
) -> Result<Vec<String>, String> {
|
||||||
|
if conflict_policy != TableImportConflictPolicy::UpdateExisting {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let primary_key_columns = target_columns.iter().filter(|column| column.is_primary_key).collect::<Vec<_>>();
|
||||||
|
if primary_key_columns.is_empty() {
|
||||||
|
return Err("Update-existing import requires target primary-key metadata".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut mapped_primary_keys = Vec::with_capacity(primary_key_columns.len());
|
||||||
|
let mut missing_primary_keys = Vec::new();
|
||||||
|
for primary_key in primary_key_columns {
|
||||||
|
if let Some(mapping) =
|
||||||
|
mappings.iter().find(|mapping| mapping.target_column.eq_ignore_ascii_case(&primary_key.name))
|
||||||
|
{
|
||||||
|
mapped_primary_keys.push(mapping.target_column.clone());
|
||||||
|
} else {
|
||||||
|
missing_primary_keys.push(primary_key.name.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !missing_primary_keys.is_empty() {
|
||||||
|
return Err(format!(
|
||||||
|
"Update-existing import requires every primary-key column to be mapped; missing: {}",
|
||||||
|
missing_primary_keys.join(", ")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let has_mapped_non_primary_key = mappings.iter().any(|mapping| {
|
||||||
|
!mapped_primary_keys.iter().any(|primary_key| primary_key.eq_ignore_ascii_case(&mapping.target_column))
|
||||||
|
});
|
||||||
|
if !has_mapped_non_primary_key {
|
||||||
|
return Err("Update-existing import requires at least one mapped non-primary-key column".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(mapped_primary_keys)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct TableImportPreparedSource {
|
pub struct TableImportPreparedSource {
|
||||||
@@ -4347,7 +4430,8 @@ fn build_import_insert_batches_with_plan(
|
|||||||
schema: &str,
|
schema: &str,
|
||||||
db_type: &DatabaseType,
|
db_type: &DatabaseType,
|
||||||
kingbase_oracle_mode: bool,
|
kingbase_oracle_mode: bool,
|
||||||
skip_duplicate_rows: bool,
|
conflict_policy: TableImportConflictPolicy,
|
||||||
|
primary_key_columns: &[String],
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
hard_sql_bytes: Option<usize>,
|
hard_sql_bytes: Option<usize>,
|
||||||
) -> Result<Vec<ImportSqlBatch>, String> {
|
) -> Result<Vec<ImportSqlBatch>, String> {
|
||||||
@@ -4355,16 +4439,33 @@ fn build_import_insert_batches_with_plan(
|
|||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||||
let batches = crate::data::transfer::generate_insert_typed_sql_batches_from_value_rows_with_options(
|
let limits = SqlBatchLimits::for_database(db_type, rows.len()).with_hard_sql_bytes(hard_sql_bytes);
|
||||||
&plan.target_columns,
|
let batches = match conflict_policy {
|
||||||
&value_rows,
|
TableImportConflictPolicy::Error | TableImportConflictPolicy::Skip => {
|
||||||
table,
|
crate::data::transfer::generate_insert_typed_sql_batches_from_value_rows_with_options(
|
||||||
schema,
|
&plan.target_columns,
|
||||||
db_type,
|
&value_rows,
|
||||||
None,
|
table,
|
||||||
SqlBatchLimits::for_database(db_type, rows.len()).with_hard_sql_bytes(hard_sql_bytes),
|
schema,
|
||||||
skip_duplicate_rows,
|
db_type,
|
||||||
)?;
|
None,
|
||||||
|
limits,
|
||||||
|
conflict_policy == TableImportConflictPolicy::Skip,
|
||||||
|
)?
|
||||||
|
}
|
||||||
|
TableImportConflictPolicy::UpdateExisting => {
|
||||||
|
crate::data::transfer::generate_primary_key_upsert_sql_batches_from_value_rows(
|
||||||
|
&plan.target_columns,
|
||||||
|
&value_rows,
|
||||||
|
table,
|
||||||
|
schema,
|
||||||
|
db_type,
|
||||||
|
None,
|
||||||
|
limits,
|
||||||
|
primary_key_columns,
|
||||||
|
)?
|
||||||
|
}
|
||||||
|
};
|
||||||
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
|
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4429,7 +4530,8 @@ fn build_import_execution_batches(
|
|||||||
schema: &str,
|
schema: &str,
|
||||||
db_type: &DatabaseType,
|
db_type: &DatabaseType,
|
||||||
kingbase_oracle_mode: bool,
|
kingbase_oracle_mode: bool,
|
||||||
skip_duplicate_rows: bool,
|
conflict_policy: TableImportConflictPolicy,
|
||||||
|
primary_key_columns: &[String],
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
hard_sql_bytes: Option<usize>,
|
hard_sql_bytes: Option<usize>,
|
||||||
) -> Result<Vec<ImportSqlBatch>, String> {
|
) -> Result<Vec<ImportSqlBatch>, String> {
|
||||||
@@ -4441,12 +4543,13 @@ fn build_import_execution_batches(
|
|||||||
schema,
|
schema,
|
||||||
db_type,
|
db_type,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
skip_duplicate_rows,
|
conflict_policy,
|
||||||
|
primary_key_columns,
|
||||||
date_time_format,
|
date_time_format,
|
||||||
hard_sql_bytes,
|
hard_sql_bytes,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if *db_type == DatabaseType::CloudflareD1 {
|
if *db_type == DatabaseType::CloudflareD1 && conflict_policy != TableImportConflictPolicy::UpdateExisting {
|
||||||
return crate::db::cloudflare_d1::build_import_insert_batches(
|
return crate::db::cloudflare_d1::build_import_insert_batches(
|
||||||
rows,
|
rows,
|
||||||
columns,
|
columns,
|
||||||
@@ -4465,7 +4568,8 @@ fn build_import_execution_batches(
|
|||||||
schema,
|
schema,
|
||||||
db_type,
|
db_type,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
skip_duplicate_rows,
|
conflict_policy,
|
||||||
|
primary_key_columns,
|
||||||
date_time_format,
|
date_time_format,
|
||||||
hard_sql_bytes,
|
hard_sql_bytes,
|
||||||
)
|
)
|
||||||
@@ -4664,7 +4768,8 @@ fn build_import_insert_batches_with_format(
|
|||||||
schema,
|
schema,
|
||||||
db_type,
|
db_type,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
false,
|
TableImportConflictPolicy::Error,
|
||||||
|
&[],
|
||||||
date_time_format,
|
date_time_format,
|
||||||
None,
|
None,
|
||||||
)?);
|
)?);
|
||||||
@@ -5503,6 +5608,18 @@ fn import_batch_execution_policy(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn completed_import_rows(
|
||||||
|
conflict_policy: TableImportConflictPolicy,
|
||||||
|
submitted_rows: usize,
|
||||||
|
affected_rows: u64,
|
||||||
|
) -> usize {
|
||||||
|
if conflict_policy == TableImportConflictPolicy::Skip {
|
||||||
|
affected_rows as usize
|
||||||
|
} else {
|
||||||
|
submitted_rows
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn execute_import_transaction(
|
async fn execute_import_transaction(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
@@ -5639,14 +5756,15 @@ async fn execute_import_rows_batch(
|
|||||||
postgres_copy_accumulator: &mut Option<PostgresCopyAccumulator>,
|
postgres_copy_accumulator: &mut Option<PostgresCopyAccumulator>,
|
||||||
sqlite_append_transaction: &mut Option<SqliteAppendTransaction>,
|
sqlite_append_transaction: &mut Option<SqliteAppendTransaction>,
|
||||||
kingbase_oracle_mode: bool,
|
kingbase_oracle_mode: bool,
|
||||||
skip_duplicate_rows: bool,
|
conflict_policy: TableImportConflictPolicy,
|
||||||
|
primary_key_columns: &[String],
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
hard_sql_bytes: Option<usize>,
|
hard_sql_bytes: Option<usize>,
|
||||||
db_write_ms: &mut u128,
|
db_write_ms: &mut u128,
|
||||||
statement_count: &mut usize,
|
statement_count: &mut usize,
|
||||||
) -> Result<usize, ImportRowsBatchError> {
|
) -> Result<usize, ImportRowsBatchError> {
|
||||||
let execution_policy = import_batch_execution_policy(mode, pending_truncate, db_type);
|
let execution_policy = import_batch_execution_policy(mode, pending_truncate, db_type);
|
||||||
if !skip_duplicate_rows {
|
if conflict_policy == TableImportConflictPolicy::Error {
|
||||||
if let Some((import_plan, bulk_plan)) = sqlserver_bulk_plans_for_rows(db_type, plan, sqlserver_bulk_plan, rows)
|
if let Some((import_plan, bulk_plan)) = sqlserver_bulk_plans_for_rows(db_type, plan, sqlserver_bulk_plan, rows)
|
||||||
{
|
{
|
||||||
return execute_sqlserver_bulk_rows_batch(
|
return execute_sqlserver_bulk_rows_batch(
|
||||||
@@ -5667,7 +5785,7 @@ async fn execute_import_rows_batch(
|
|||||||
}
|
}
|
||||||
// COPY is used only for plain scalar PostgreSQL rows and ordinary tables. Any unsupported
|
// COPY is used only for plain scalar PostgreSQL rows and ordinary tables. Any unsupported
|
||||||
// value or table feature falls through to the portable INSERT generator below.
|
// value or table feature falls through to the portable INSERT generator below.
|
||||||
if !skip_duplicate_rows
|
if conflict_policy == TableImportConflictPolicy::Error
|
||||||
&& execution_policy.allow_postgres_copy
|
&& execution_policy.allow_postgres_copy
|
||||||
&& *db_type == DatabaseType::Postgres
|
&& *db_type == DatabaseType::Postgres
|
||||||
&& !rows
|
&& !rows
|
||||||
@@ -5711,7 +5829,8 @@ async fn execute_import_rows_batch(
|
|||||||
schema,
|
schema,
|
||||||
db_type,
|
db_type,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
skip_duplicate_rows,
|
conflict_policy,
|
||||||
|
primary_key_columns,
|
||||||
date_time_format,
|
date_time_format,
|
||||||
hard_sql_bytes,
|
hard_sql_bytes,
|
||||||
)
|
)
|
||||||
@@ -5774,22 +5893,19 @@ async fn execute_import_rows_batch(
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|message| ImportRowsBatchError::with_rows_imported(rows_imported, message))?;
|
.map_err(|message| ImportRowsBatchError::with_rows_imported(rows_imported, message))?;
|
||||||
return Ok(if skip_duplicate_rows {
|
return Ok(rows_imported.saturating_add(completed_import_rows(
|
||||||
rows_imported.saturating_add(result.affected_rows as usize)
|
conflict_policy,
|
||||||
} else {
|
rows.len(),
|
||||||
rows_imported.saturating_add(rows.len())
|
result.affected_rows,
|
||||||
});
|
)));
|
||||||
}
|
}
|
||||||
for batch in batches {
|
for batch in batches {
|
||||||
ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
|
ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
|
||||||
let result = execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count)
|
let result = execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| ImportRowsBatchError::with_rows_imported(rows_imported, error))?;
|
.map_err(|error| ImportRowsBatchError::with_rows_imported(rows_imported, error))?;
|
||||||
rows_imported = if skip_duplicate_rows {
|
rows_imported =
|
||||||
rows_imported.saturating_add(result.affected_rows as usize)
|
rows_imported.saturating_add(completed_import_rows(conflict_policy, batch.row_count, result.affected_rows));
|
||||||
} else {
|
|
||||||
rows_imported.saturating_add(batch.row_count)
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
Ok(rows_imported)
|
Ok(rows_imported)
|
||||||
}
|
}
|
||||||
@@ -6594,6 +6710,10 @@ where
|
|||||||
let mut statement_count = 0usize;
|
let mut statement_count = 0usize;
|
||||||
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
|
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
|
||||||
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
|
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
|
||||||
|
let conflict_policy = request.effective_conflict_policy();
|
||||||
|
if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) {
|
||||||
|
return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error));
|
||||||
|
}
|
||||||
let source_format = match effective_source_format(&request.file_path, request.source_format) {
|
let source_format = match effective_source_format(&request.file_path, request.source_format) {
|
||||||
Ok(format) => format,
|
Ok(format) => format,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
@@ -6750,6 +6870,43 @@ where
|
|||||||
create_table_sample = Some(parsed);
|
create_table_sample = Some(parsed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let target_columns = match get_columns_for_transfer(
|
||||||
|
state,
|
||||||
|
pool_key,
|
||||||
|
&request.connection_id,
|
||||||
|
&request.database,
|
||||||
|
&request.schema,
|
||||||
|
&request.table,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(columns) => columns,
|
||||||
|
Err(error) if conflict_policy == TableImportConflictPolicy::UpdateExisting => {
|
||||||
|
return Err(emit_import_error(
|
||||||
|
&mut progress_callback,
|
||||||
|
request,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
started_at,
|
||||||
|
format!("Update-existing import could not load target primary-key metadata: {error}"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(_) => Vec::new(),
|
||||||
|
};
|
||||||
|
let primary_key_columns =
|
||||||
|
match update_existing_primary_key_columns(conflict_policy, &request.mappings, &target_columns) {
|
||||||
|
Ok(columns) => columns,
|
||||||
|
Err(error) => {
|
||||||
|
return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut target_column_types =
|
||||||
|
target_columns.iter().map(|column| (column.name.clone(), column.data_type.clone())).collect::<Vec<_>>();
|
||||||
|
if target_column_types.is_empty() {
|
||||||
|
target_column_types = created_column_types.clone().unwrap_or_default();
|
||||||
|
}
|
||||||
|
|
||||||
if source_format.is_delimited() {
|
if source_format.is_delimited() {
|
||||||
let parsed = if let Some(parsed) = create_table_sample.clone().or_else(|| prepared_source.clone()) {
|
let parsed = if let Some(parsed) = create_table_sample.clone().or_else(|| prepared_source.clone()) {
|
||||||
parsed
|
parsed
|
||||||
@@ -6773,23 +6930,6 @@ where
|
|||||||
|
|
||||||
let total_bytes = tokio::fs::metadata(&request.file_path).await.map(|metadata| metadata.len()).unwrap_or(0);
|
let total_bytes = tokio::fs::metadata(&request.file_path).await.map(|metadata| metadata.len()).unwrap_or(0);
|
||||||
|
|
||||||
let mut target_column_types = get_columns_for_transfer(
|
|
||||||
state,
|
|
||||||
pool_key,
|
|
||||||
&request.connection_id,
|
|
||||||
&request.database,
|
|
||||||
&request.schema,
|
|
||||||
&request.table,
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.map(|column| (column.name, column.data_type))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
if target_column_types.is_empty() {
|
|
||||||
target_column_types = created_column_types.clone().unwrap_or_default();
|
|
||||||
}
|
|
||||||
let (resolved_encoding, _) =
|
let (resolved_encoding, _) =
|
||||||
validated_text_encoding.ok_or_else(|| "Delimited import encoding was not validated".to_string())?;
|
validated_text_encoding.ok_or_else(|| "Delimited import encoding was not validated".to_string())?;
|
||||||
let mut streaming_options = import_parse_options.clone();
|
let mut streaming_options = import_parse_options.clone();
|
||||||
@@ -6949,7 +7089,8 @@ where
|
|||||||
&mut postgres_copy_accumulator,
|
&mut postgres_copy_accumulator,
|
||||||
&mut sqlite_append_transaction,
|
&mut sqlite_append_transaction,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
request.skip_duplicate_rows,
|
conflict_policy,
|
||||||
|
&primary_key_columns,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
import_sql_hard_limit,
|
import_sql_hard_limit,
|
||||||
&mut db_write_ms,
|
&mut db_write_ms,
|
||||||
@@ -7141,23 +7282,6 @@ where
|
|||||||
let effective_batch_size = effective_import_batch_size(db_type, batch_size);
|
let effective_batch_size = effective_import_batch_size(db_type, batch_size);
|
||||||
let expected_columns =
|
let expected_columns =
|
||||||
create_table_sample.as_ref().or(prepared_source.as_ref()).map(|source| source.columns.clone());
|
create_table_sample.as_ref().or(prepared_source.as_ref()).map(|source| source.columns.clone());
|
||||||
let mut target_column_types = get_columns_for_transfer(
|
|
||||||
state,
|
|
||||||
pool_key,
|
|
||||||
&request.connection_id,
|
|
||||||
&request.database,
|
|
||||||
&request.schema,
|
|
||||||
&request.table,
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.map(|column| (column.name, column.data_type))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
if target_column_types.is_empty() {
|
|
||||||
target_column_types = created_column_types.clone().unwrap_or_default();
|
|
||||||
}
|
|
||||||
let text_source_columns = textual_source_columns_for_import(&request.mappings, &target_column_types);
|
let text_source_columns = textual_source_columns_for_import(&request.mappings, &target_column_types);
|
||||||
// No truncate, INSERT, or COPY may run until the selected worksheet parses to EOF.
|
// No truncate, INSERT, or COPY may run until the selected worksheet parses to EOF.
|
||||||
let mut last_xlsx_read_bytes = 0u64;
|
let mut last_xlsx_read_bytes = 0u64;
|
||||||
@@ -7452,7 +7576,8 @@ where
|
|||||||
&mut postgres_copy_accumulator,
|
&mut postgres_copy_accumulator,
|
||||||
&mut sqlite_append_transaction,
|
&mut sqlite_append_transaction,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
request.skip_duplicate_rows,
|
conflict_policy,
|
||||||
|
&primary_key_columns,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
import_sql_hard_limit,
|
import_sql_hard_limit,
|
||||||
&mut db_write_ms,
|
&mut db_write_ms,
|
||||||
@@ -7636,23 +7761,6 @@ where
|
|||||||
started_at,
|
started_at,
|
||||||
None,
|
None,
|
||||||
));
|
));
|
||||||
let mut target_column_types = get_columns_for_transfer(
|
|
||||||
state,
|
|
||||||
pool_key,
|
|
||||||
&request.connection_id,
|
|
||||||
&request.database,
|
|
||||||
&request.schema,
|
|
||||||
&request.table,
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default()
|
|
||||||
.into_iter()
|
|
||||||
.map(|column| (column.name, column.data_type))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
if target_column_types.is_empty() {
|
|
||||||
target_column_types = created_column_types.clone().unwrap_or_default();
|
|
||||||
}
|
|
||||||
let text_source_columns = textual_source_columns_for_import(&request.mappings, &target_column_types);
|
let text_source_columns = textual_source_columns_for_import(&request.mappings, &target_column_types);
|
||||||
let effective_batch_size = effective_import_batch_size(db_type, batch_size);
|
let effective_batch_size = effective_import_batch_size(db_type, batch_size);
|
||||||
let mut row_source = match ImportRowSource::open(
|
let mut row_source = match ImportRowSource::open(
|
||||||
@@ -7801,7 +7909,8 @@ where
|
|||||||
&mut postgres_copy_accumulator,
|
&mut postgres_copy_accumulator,
|
||||||
&mut sqlite_append_transaction,
|
&mut sqlite_append_transaction,
|
||||||
kingbase_oracle_mode,
|
kingbase_oracle_mode,
|
||||||
request.skip_duplicate_rows,
|
conflict_policy,
|
||||||
|
&primary_key_columns,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
import_sql_hard_limit,
|
import_sql_hard_limit,
|
||||||
&mut db_write_ms,
|
&mut db_write_ms,
|
||||||
@@ -7964,6 +8073,82 @@ mod tests {
|
|||||||
assert_eq!(progress_json["totalRowsExact"], serde_json::json!(true));
|
assert_eq!(progress_json["totalRowsExact"], serde_json::json!(true));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn table_import_conflict_policy_preserves_legacy_skip_requests() {
|
||||||
|
let request_json = serde_json::json!({
|
||||||
|
"importId": "import-1",
|
||||||
|
"connectionId": "connection-1",
|
||||||
|
"database": "db",
|
||||||
|
"schema": "public",
|
||||||
|
"table": "users",
|
||||||
|
"filePath": "users.csv",
|
||||||
|
"mappings": [],
|
||||||
|
"mode": "append",
|
||||||
|
"batchSize": 500
|
||||||
|
});
|
||||||
|
|
||||||
|
let default_request: TableImportRequest = serde_json::from_value(request_json.clone()).unwrap();
|
||||||
|
assert_eq!(default_request.effective_conflict_policy(), TableImportConflictPolicy::Error);
|
||||||
|
|
||||||
|
let mut legacy_skip_json = request_json.clone();
|
||||||
|
legacy_skip_json["skipDuplicateRows"] = serde_json::json!(true);
|
||||||
|
let legacy_skip: TableImportRequest = serde_json::from_value(legacy_skip_json).unwrap();
|
||||||
|
assert_eq!(legacy_skip.effective_conflict_policy(), TableImportConflictPolicy::Skip);
|
||||||
|
|
||||||
|
let mut explicit_json = request_json;
|
||||||
|
explicit_json["skipDuplicateRows"] = serde_json::json!(true);
|
||||||
|
explicit_json["conflictPolicy"] = serde_json::json!("updateExisting");
|
||||||
|
let explicit: TableImportRequest = serde_json::from_value(explicit_json).unwrap();
|
||||||
|
assert_eq!(explicit.effective_conflict_policy(), TableImportConflictPolicy::UpdateExisting);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn update_existing_progress_counts_source_rows_not_database_affected_rows() {
|
||||||
|
assert_eq!(completed_import_rows(TableImportConflictPolicy::Error, 3, 3), 3);
|
||||||
|
assert_eq!(completed_import_rows(TableImportConflictPolicy::Skip, 3, 1), 1);
|
||||||
|
// MySQL reports two affected rows for a changed row handled by ON DUPLICATE KEY UPDATE.
|
||||||
|
assert_eq!(completed_import_rows(TableImportConflictPolicy::UpdateExisting, 3, 6), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn update_existing_requires_supported_dialect_and_complete_primary_key_mapping() {
|
||||||
|
let mappings = vec![
|
||||||
|
TableImportColumnMapping {
|
||||||
|
source_column: "tenant".to_string(),
|
||||||
|
target_column: "tenant_id".to_string(),
|
||||||
|
target_data_type: None,
|
||||||
|
},
|
||||||
|
TableImportColumnMapping {
|
||||||
|
source_column: "name".to_string(),
|
||||||
|
target_column: "name".to_string(),
|
||||||
|
target_data_type: None,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let target_columns = vec![
|
||||||
|
crate::db::ColumnInfo { name: "tenant_id".to_string(), is_primary_key: true, ..Default::default() },
|
||||||
|
crate::db::ColumnInfo { name: "id".to_string(), is_primary_key: true, ..Default::default() },
|
||||||
|
crate::db::ColumnInfo { name: "name".to_string(), ..Default::default() },
|
||||||
|
];
|
||||||
|
|
||||||
|
let missing_key =
|
||||||
|
update_existing_primary_key_columns(TableImportConflictPolicy::UpdateExisting, &mappings, &target_columns)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(missing_key.contains("id"));
|
||||||
|
|
||||||
|
let missing_metadata = update_existing_primary_key_columns(
|
||||||
|
TableImportConflictPolicy::UpdateExisting,
|
||||||
|
&mappings,
|
||||||
|
&[crate::db::ColumnInfo { name: "name".to_string(), ..Default::default() }],
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(missing_metadata.contains("primary-key metadata"));
|
||||||
|
|
||||||
|
let unsupported =
|
||||||
|
validate_update_existing_target(TableImportConflictPolicy::UpdateExisting, &DatabaseType::Oracle, false)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(unsupported.contains("not supported for oracle"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn compiled_import_plan_reuses_source_indexes_and_target_types() {
|
fn compiled_import_plan_reuses_source_indexes_and_target_types() {
|
||||||
let columns = vec!["name".to_string(), "id".to_string()];
|
let columns = vec!["name".to_string(), "id".to_string()];
|
||||||
@@ -8047,6 +8232,7 @@ mod tests {
|
|||||||
effective_encoding: Some(TableImportTextEncoding::Utf8),
|
effective_encoding: Some(TableImportTextEncoding::Utf8),
|
||||||
}),
|
}),
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -9761,6 +9947,7 @@ mod tests {
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
};
|
};
|
||||||
let started_at = Instant::now();
|
let started_at = Instant::now();
|
||||||
@@ -10172,6 +10359,7 @@ mod tests {
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -10262,6 +10450,7 @@ mod tests {
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -10343,6 +10532,7 @@ mod tests {
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -11426,6 +11616,99 @@ mod tests {
|
|||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn import_conflict_policy_keeps_default_and_skip_sql_unchanged() {
|
||||||
|
let plan = CompiledImportPlan {
|
||||||
|
mapped_source_indexes: vec![0, 1],
|
||||||
|
target_columns: vec!["id".to_string(), "name".to_string()],
|
||||||
|
column_types: vec![Some("integer".to_string()), Some("text".to_string())],
|
||||||
|
};
|
||||||
|
let rows = vec![vec![serde_json::json!(1), serde_json::json!("Ada")]];
|
||||||
|
|
||||||
|
let default_batches = build_import_insert_batches_with_plan(
|
||||||
|
&rows,
|
||||||
|
&plan,
|
||||||
|
"users",
|
||||||
|
"public",
|
||||||
|
&DatabaseType::Postgres,
|
||||||
|
false,
|
||||||
|
TableImportConflictPolicy::Error,
|
||||||
|
&[],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(default_batches[0].sql, "INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES\n(1, 'Ada')");
|
||||||
|
|
||||||
|
let skip_batches = build_import_insert_batches_with_plan(
|
||||||
|
&rows,
|
||||||
|
&plan,
|
||||||
|
"users",
|
||||||
|
"public",
|
||||||
|
&DatabaseType::Postgres,
|
||||||
|
false,
|
||||||
|
TableImportConflictPolicy::Skip,
|
||||||
|
&[],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
skip_batches[0].sql,
|
||||||
|
"INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES\n(1, 'Ada')\nON CONFLICT DO NOTHING"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn update_existing_sql_uses_composite_primary_key_and_only_updates_non_keys() {
|
||||||
|
let plan = CompiledImportPlan {
|
||||||
|
mapped_source_indexes: vec![0, 1, 2, 3],
|
||||||
|
target_columns: vec!["tenant_id".to_string(), "id".to_string(), "name".to_string(), "status".to_string()],
|
||||||
|
column_types: vec![None, None, None, None],
|
||||||
|
};
|
||||||
|
let rows = vec![vec![
|
||||||
|
serde_json::json!(7),
|
||||||
|
serde_json::json!(42),
|
||||||
|
serde_json::json!("Ada"),
|
||||||
|
serde_json::json!("active"),
|
||||||
|
]];
|
||||||
|
|
||||||
|
let postgres = build_import_insert_batches_with_plan(
|
||||||
|
&rows,
|
||||||
|
&plan,
|
||||||
|
"users",
|
||||||
|
"public",
|
||||||
|
&DatabaseType::Postgres,
|
||||||
|
false,
|
||||||
|
TableImportConflictPolicy::UpdateExisting,
|
||||||
|
&["tenant_id".to_string(), "id".to_string()],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(postgres[0].sql.contains("ON CONFLICT (\"tenant_id\", \"id\") DO UPDATE SET"));
|
||||||
|
assert!(postgres[0].sql.contains("\"name\" = EXCLUDED.\"name\""));
|
||||||
|
assert!(postgres[0].sql.contains("\"status\" = EXCLUDED.\"status\""));
|
||||||
|
assert!(!postgres[0].sql.contains("\"id\" = EXCLUDED.\"id\""));
|
||||||
|
assert_eq!(postgres[0].row_count, 1);
|
||||||
|
|
||||||
|
let mysql = build_import_insert_batches_with_plan(
|
||||||
|
&rows,
|
||||||
|
&plan,
|
||||||
|
"users",
|
||||||
|
"",
|
||||||
|
&DatabaseType::Mysql,
|
||||||
|
false,
|
||||||
|
TableImportConflictPolicy::UpdateExisting,
|
||||||
|
&["tenant_id".to_string(), "id".to_string()],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(mysql[0].sql.contains("ON DUPLICATE KEY UPDATE `name` = VALUES(`name`), `status` = VALUES(`status`)"));
|
||||||
|
assert!(!mysql[0].sql.contains("`id` = VALUES(`id`)"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn iris_import_uses_single_row_values_statements() {
|
fn iris_import_uses_single_row_values_statements() {
|
||||||
let mappings = vec![TableImportColumnMapping {
|
let mappings = vec![TableImportColumnMapping {
|
||||||
@@ -11698,7 +11981,8 @@ mod tests {
|
|||||||
"dbo",
|
"dbo",
|
||||||
&DatabaseType::SqlServer,
|
&DatabaseType::SqlServer,
|
||||||
false,
|
false,
|
||||||
false,
|
TableImportConflictPolicy::Error,
|
||||||
|
&[],
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
@@ -12024,7 +12308,8 @@ mod tests {
|
|||||||
&mut sqlite_append_transaction,
|
&mut sqlite_append_transaction,
|
||||||
&mut postgres_copy_accumulator,
|
&mut postgres_copy_accumulator,
|
||||||
false,
|
false,
|
||||||
false,
|
TableImportConflictPolicy::Error,
|
||||||
|
&[],
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
&mut db_write_ms,
|
&mut db_write_ms,
|
||||||
@@ -12242,7 +12527,8 @@ mod tests {
|
|||||||
&mut self.postgres_copy_accumulator,
|
&mut self.postgres_copy_accumulator,
|
||||||
&mut self.transaction,
|
&mut self.transaction,
|
||||||
false,
|
false,
|
||||||
false,
|
TableImportConflictPolicy::Error,
|
||||||
|
&[],
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
&mut self.db_write_ms,
|
&mut self.db_write_ms,
|
||||||
@@ -12368,6 +12654,7 @@ mod tests {
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -12395,6 +12682,97 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn sqlite_update_existing_import_counts_updated_rows_once() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
|
||||||
|
let state = AppState::new(storage);
|
||||||
|
let connection_id = "sqlite-update-existing";
|
||||||
|
let pool_key = format!("{connection_id}:session:import");
|
||||||
|
let database_path = dir.path().join("target.db");
|
||||||
|
let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
|
||||||
|
crate::db::sqlite::execute_query(
|
||||||
|
&sqlite,
|
||||||
|
"CREATE TABLE items (tenant_id INTEGER, id INTEGER, name TEXT, PRIMARY KEY (tenant_id, id))",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
crate::db::sqlite::execute_query(&sqlite, "INSERT INTO items (tenant_id, id, name) VALUES (1, 1, 'Old')")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
state
|
||||||
|
.update_connection_pools(|connections| {
|
||||||
|
connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": connection_id,
|
||||||
|
"name": "SQLite update-existing test",
|
||||||
|
"db_type": "sqlite",
|
||||||
|
"host": "",
|
||||||
|
"port": 0,
|
||||||
|
"username": "",
|
||||||
|
"password": "",
|
||||||
|
"database": database_path.to_string_lossy()
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
state.configs.write().await.insert(connection_id.to_string(), config);
|
||||||
|
let data_path = dir.path().join("rows.csv");
|
||||||
|
std::fs::write(&data_path, b"tenant_id,id,name\n1,1,Updated\n1,2,Inserted\n").unwrap();
|
||||||
|
let request = TableImportRequest {
|
||||||
|
import_id: "sqlite-update-existing".to_string(),
|
||||||
|
connection_id: connection_id.to_string(),
|
||||||
|
database: String::new(),
|
||||||
|
schema: String::new(),
|
||||||
|
table: "items".to_string(),
|
||||||
|
file_path: data_path.to_string_lossy().to_string(),
|
||||||
|
source_ref: None,
|
||||||
|
source_format: Some(TableImportSourceFormat::Csv),
|
||||||
|
parse_options: TableImportParseOptions::default(),
|
||||||
|
mappings: ["tenant_id", "id", "name"]
|
||||||
|
.into_iter()
|
||||||
|
.map(|column| TableImportColumnMapping {
|
||||||
|
source_column: column.to_string(),
|
||||||
|
target_column: column.to_string(),
|
||||||
|
target_data_type: None,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
mode: TableImportMode::Append,
|
||||||
|
create_table: false,
|
||||||
|
batch_size: 2,
|
||||||
|
date_time_format: None,
|
||||||
|
prepared_source: None,
|
||||||
|
retain_source: false,
|
||||||
|
conflict_policy: Some(TableImportConflictPolicy::UpdateExisting),
|
||||||
|
skip_duplicate_rows: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let summary = import_table_file_core(
|
||||||
|
&state,
|
||||||
|
&request,
|
||||||
|
&DatabaseType::Sqlite,
|
||||||
|
&pool_key,
|
||||||
|
|_| Box::pin(async { false }),
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(summary.rows_imported, 2);
|
||||||
|
let rows =
|
||||||
|
crate::db::sqlite::execute_query(&sqlite, "SELECT tenant_id, id, name FROM items ORDER BY tenant_id, id")
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.rows;
|
||||||
|
assert_eq!(
|
||||||
|
rows,
|
||||||
|
vec![
|
||||||
|
vec![serde_json::json!(1), serde_json::json!(1), serde_json::json!("Updated")],
|
||||||
|
vec![serde_json::json!(1), serde_json::json!(2), serde_json::json!("Inserted")],
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn preview_missing_source_fails_before_parsing() {
|
async fn preview_missing_source_fails_before_parsing() {
|
||||||
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
|
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
|
||||||
|
|||||||
@@ -4036,6 +4036,76 @@ fn uses_mysql_style_upsert(db_type: &DatabaseType) -> bool {
|
|||||||
matches!(db_type, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::OpenGauss)
|
matches!(db_type, DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::OpenGauss)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn supports_primary_key_upsert(db_type: &DatabaseType) -> bool {
|
||||||
|
matches!(
|
||||||
|
db_type,
|
||||||
|
DatabaseType::Postgres
|
||||||
|
| DatabaseType::Kingbase
|
||||||
|
| DatabaseType::Sqlite
|
||||||
|
| DatabaseType::CloudflareD1
|
||||||
|
| DatabaseType::DuckDb
|
||||||
|
| DatabaseType::Mysql
|
||||||
|
| DatabaseType::Doris
|
||||||
|
| DatabaseType::StarRocks
|
||||||
|
| DatabaseType::OpenGauss
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn primary_key_upsert_clause(
|
||||||
|
columns: &[String],
|
||||||
|
pk_columns: &[String],
|
||||||
|
db_type: &DatabaseType,
|
||||||
|
quote_target_column_names: bool,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
if pk_columns.is_empty() {
|
||||||
|
return Err("Update-existing import requires target primary-key metadata".to_string());
|
||||||
|
}
|
||||||
|
if pk_columns.iter().any(|primary_key| !columns.iter().any(|column| column.eq_ignore_ascii_case(primary_key))) {
|
||||||
|
return Err("Update-existing import requires every target primary-key column to be mapped".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let non_pk_columns = columns
|
||||||
|
.iter()
|
||||||
|
.filter(|column| !pk_columns.iter().any(|primary_key| column.eq_ignore_ascii_case(primary_key)))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if non_pk_columns.is_empty() {
|
||||||
|
return Err("Update-existing import requires at least one mapped non-primary-key column".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (is_postgres_transfer_dialect(db_type) && !matches!(db_type, DatabaseType::OpenGauss))
|
||||||
|
|| matches!(db_type, DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb)
|
||||||
|
{
|
||||||
|
let primary_keys = pk_columns
|
||||||
|
.iter()
|
||||||
|
.map(|column| transfer_column_identifier(column, db_type, quote_target_column_names))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
let updates = non_pk_columns
|
||||||
|
.iter()
|
||||||
|
.map(|column| {
|
||||||
|
let column = transfer_column_identifier(column, db_type, quote_target_column_names);
|
||||||
|
format!("{column} = EXCLUDED.{column}")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
return Ok(format!("\nON CONFLICT ({primary_keys}) DO UPDATE SET {updates}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
if uses_mysql_style_upsert(db_type) {
|
||||||
|
let updates = non_pk_columns
|
||||||
|
.iter()
|
||||||
|
.map(|column| {
|
||||||
|
let column = transfer_column_identifier(column, db_type, quote_target_column_names);
|
||||||
|
format!("{column} = VALUES({column})")
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
return Ok(format!("\nON DUPLICATE KEY UPDATE {updates}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(format!("Update-existing import conflict policy is not supported for {}", db_type.as_str()))
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn generate_upsert_typed_for_transfer(
|
fn generate_upsert_typed_for_transfer(
|
||||||
columns: &[String],
|
columns: &[String],
|
||||||
@@ -4747,6 +4817,55 @@ pub(crate) fn generate_insert_typed_sql_batches_from_value_rows_with_options(
|
|||||||
Ok(statements)
|
Ok(statements)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(crate) fn generate_primary_key_upsert_sql_batches_from_value_rows(
|
||||||
|
columns: &[String],
|
||||||
|
value_rows: &[String],
|
||||||
|
table: &str,
|
||||||
|
schema: &str,
|
||||||
|
db_type: &DatabaseType,
|
||||||
|
catalog: Option<&str>,
|
||||||
|
limits: SqlBatchLimits,
|
||||||
|
pk_columns: &[String],
|
||||||
|
) -> Result<Vec<(String, usize)>, String> {
|
||||||
|
if !supports_primary_key_upsert(db_type) {
|
||||||
|
return Err(format!("Update-existing import conflict policy is not supported for {}", db_type.as_str()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let clause = primary_key_upsert_clause(columns, pk_columns, db_type, true)?;
|
||||||
|
let clause_bytes = sql_text_bytes(&clause, db_type);
|
||||||
|
let adjusted_limits = SqlBatchLimits {
|
||||||
|
max_rows: limits.max_rows,
|
||||||
|
target_sql_bytes: limits.target_sql_bytes.saturating_sub(clause_bytes).max(1),
|
||||||
|
hard_sql_bytes: limits.hard_sql_bytes.map(|limit| limit.saturating_sub(clause_bytes).max(1)),
|
||||||
|
};
|
||||||
|
let batches = generate_insert_sql_batches_from_value_rows(
|
||||||
|
columns,
|
||||||
|
value_rows,
|
||||||
|
table,
|
||||||
|
schema,
|
||||||
|
db_type,
|
||||||
|
catalog,
|
||||||
|
adjusted_limits,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
batches
|
||||||
|
.into_iter()
|
||||||
|
.map(|(mut sql, row_count)| {
|
||||||
|
sql.push_str(&clause);
|
||||||
|
if limits.hard_sql_bytes.is_some_and(|limit| sql_text_bytes(&sql, db_type) > limit) {
|
||||||
|
return Err(format!(
|
||||||
|
"SQL batch with update-existing conflict handling exceeds the {} byte hard limit",
|
||||||
|
limits.hard_sql_bytes.unwrap_or_default()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok((sql, row_count))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn generate_insert_typed_sql_batches_for_transfer(
|
fn generate_insert_typed_sql_batches_for_transfer(
|
||||||
columns: &[String],
|
columns: &[String],
|
||||||
|
|||||||
@@ -133,6 +133,7 @@ fn live_sqlserver_import_request(
|
|||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
prepared_source: None,
|
prepared_source: None,
|
||||||
retain_source: false,
|
retain_source: false,
|
||||||
|
conflict_policy: None,
|
||||||
skip_duplicate_rows: false,
|
skip_duplicate_rows: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user