feat(import): add option to skip duplicate rows
This commit is contained in:
1 parent
daebae9b50
commit
3c46f8d440
8 files changed
+328
-32
No files matched your search
@@ -32,7 +32,7 @@ import {
|
||||
import { importPreviewInput, importSourceDisplayName, uploadedImportSourceFromPreview } from "@/lib/import/importSource";
|
||||
import { getDataTypeOptions } from "@/lib/table/tableStructureEditorState";
|
||||
import { metadataSchemaForConnection, tableStructureDatabaseTypeForConnection } from "@/lib/database/jdbcDialect";
|
||||
import type { ColumnInfo } from "@/types/database";
|
||||
import type { ColumnInfo, DatabaseType } from "@/types/database";
|
||||
import * as api from "@/lib/backend/api";
|
||||
|
||||
const { t } = useI18n();
|
||||
@@ -140,6 +140,7 @@ const titleRow = ref(1);
|
||||
const dataStartRow = ref(2);
|
||||
const lastDataRow = ref(0);
|
||||
const trimValues = ref(false);
|
||||
const skipDuplicateRows = ref(false);
|
||||
const emptyStringAsNull = ref(defaultTableImportEmptyStringAsNull(sourceFormat.value));
|
||||
const selectedSheet = ref("");
|
||||
const jsonShape = ref<api.TableImportJsonShape>("auto");
|
||||
@@ -175,6 +176,10 @@ const wizardSteps: Array<{ value: TableImportWizardStep; labelKey: string }> = [
|
||||
|
||||
const selectedConnection = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId) : undefined));
|
||||
const structureDatabaseType = computed(() => tableStructureDatabaseTypeForConnection(selectedConnection.value));
|
||||
// Mirrors the dialect list in skip-generator dispatch (transfer.rs): only these
|
||||
// engines turn the skip-duplicates option into conflict-handling INSERTs.
|
||||
const SKIP_DUPLICATE_ROWS_DATABASE_TYPES = new Set<DatabaseType>(["postgres", "kingbase", "opengauss", "sqlite", "cloudflare-d1", "duckdb", "mysql", "doris", "starrocks"]);
|
||||
const supportsSkipDuplicateRows = computed(() => structureDatabaseType.value !== undefined && SKIP_DUPLICATE_ROWS_DATABASE_TYPES.has(structureDatabaseType.value));
|
||||
const targetSchema = computed(() => metadataSchemaForConnection(selectedConnection.value, props.prefillDatabase || "", props.prefillSchema));
|
||||
const dataTypeOptions = computed(() => mergeDataTypeOptions(dynamicDataTypeOptions.value, getDataTypeOptions(structureDatabaseType.value), Object.values(columnDataTypes.value)));
|
||||
const hasExistingTarget = computed(() => !!props.prefillTable || loadingExistingTables.value || existingTableNames.value.length > 0);
|
||||
@@ -379,6 +384,7 @@ function resetState() {
|
||||
dataStartRow.value = 2;
|
||||
lastDataRow.value = 0;
|
||||
trimValues.value = false;
|
||||
skipDuplicateRows.value = false;
|
||||
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
|
||||
selectedSheet.value = "";
|
||||
jsonShape.value = "auto";
|
||||
@@ -935,6 +941,7 @@ async function startImport() {
|
||||
mode: targetMode.value === "create" ? "append" : importMode.value,
|
||||
createTable: targetMode.value === "create",
|
||||
batchSize: Math.max(1, Number(batchSize.value) || 500),
|
||||
skipDuplicateRows: supportsSkipDuplicateRows.value && skipDuplicateRows.value,
|
||||
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
|
||||
preparedSource: preparedImportSource(currentPreview),
|
||||
},
|
||||
@@ -1024,6 +1031,7 @@ async function startBatchImport() {
|
||||
mode: "append",
|
||||
createTable: true,
|
||||
batchSize: Math.max(1, Number(batchSize.value) || 500),
|
||||
skipDuplicateRows: supportsSkipDuplicateRows.value && skipDuplicateRows.value,
|
||||
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
|
||||
preparedSource: preparedImportSource(task.preview),
|
||||
retainSource: true,
|
||||
@@ -1414,6 +1422,10 @@ watch(rawProgressPercent, (percent) => {
|
||||
<input v-model="emptyStringAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
||||
{{ t("tableImport.emptyStringAsNull") }}
|
||||
</label>
|
||||
<label v-if="supportsSkipDuplicateRows" class="flex items-center gap-2 text-xs">
|
||||
<input v-model="skipDuplicateRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
||||
{{ t("tableImport.skipDuplicateRows") }}
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div v-else-if="sourceFormat === 'sql'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
||||
|
||||
+34
-2
@@ -26,6 +26,7 @@ vi.mock("@/stores/connectionStore", () => ({
|
||||
getConfig: (id: string) => {
|
||||
if (id === "connection-1") return { id, name: "SQLite", db_type: "sqlite" };
|
||||
if (id === "postgres-1") return { id, name: "PostgreSQL", db_type: "postgres" };
|
||||
if (id === "sqlserver-1") return { id, name: "SQL Server", db_type: "sqlserver" };
|
||||
return undefined;
|
||||
},
|
||||
ensureConnected: mocks.ensureConnected,
|
||||
@@ -158,6 +159,21 @@ import TableImportDialog from "@/components/import/TableImportDialog.vue";
|
||||
const mountedApps: App[] = [];
|
||||
const sheets = ["First", "Second", "Third"];
|
||||
|
||||
function delimitedPreview(): TableImportPreview {
|
||||
return {
|
||||
fileName: "rows.csv",
|
||||
filePath: "/tmp/rows.csv",
|
||||
sourceRef: "delimited-source",
|
||||
fileType: "csv",
|
||||
sizeBytes: 10,
|
||||
columns: ["id", "name"],
|
||||
rows: [[1, "a"]],
|
||||
totalRows: 2,
|
||||
totalRowsExact: true,
|
||||
sourceFingerprint: "rows-csv",
|
||||
};
|
||||
}
|
||||
|
||||
function workbookPreview(sheetName: string): TableImportPreview {
|
||||
const index = sheets.indexOf(sheetName);
|
||||
return {
|
||||
@@ -182,10 +198,10 @@ async function flushAsyncUpdates() {
|
||||
}
|
||||
}
|
||||
|
||||
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")]) {
|
||||
async function mountDialog(files = [new File(["workbook"], "rows.xlsx")], connectionId = "connection-1") {
|
||||
const container = document.createElement("div");
|
||||
document.body.append(container);
|
||||
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: "connection-1", prefillDatabase: "main" }) }));
|
||||
const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: connectionId, prefillDatabase: "main" }) }));
|
||||
mountedApps.push(app);
|
||||
app.use(i18n);
|
||||
app.mount(container);
|
||||
@@ -272,6 +288,22 @@ describe("TableImportDialog batch selection", () => {
|
||||
expect(mocks.importTableFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("hides the skip-duplicates option for dialects without conflict handling", async () => {
|
||||
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve(delimitedPreview()));
|
||||
await mountDialog([new File(["id,name\n1,a"], "rows.csv")], "sqlserver-1");
|
||||
await flushAsyncUpdates();
|
||||
expect(document.body.textContent).toContain("Trim values");
|
||||
expect(document.body.textContent).not.toContain("Skip duplicate rows");
|
||||
});
|
||||
|
||||
it("offers the skip-duplicates option for dialects with conflict handling", async () => {
|
||||
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve(delimitedPreview()));
|
||||
await mountDialog([new File(["id,name\n1,a"], "rows.csv")]);
|
||||
await flushAsyncUpdates();
|
||||
expect(document.body.textContent).toContain("Trim values");
|
||||
expect(document.body.textContent).toContain("Skip duplicate rows");
|
||||
});
|
||||
|
||||
it("keeps every worksheet selected by default and imports them in workbook order", async () => {
|
||||
await mountDialog();
|
||||
for (const name of ["rows_First", "rows_Second", "rows_Third"]) expect(taskCheckbox(name).checked).toBe(true);
|
||||
|
||||
Reference in new issue
Block a user