diff --git a/apps/desktop/src/components/editor/EditorSettingsDialog.vue b/apps/desktop/src/components/editor/EditorSettingsDialog.vue index 6e7172cfc7..855fff632a 100644 --- a/apps/desktop/src/components/editor/EditorSettingsDialog.vue +++ b/apps/desktop/src/components/editor/EditorSettingsDialog.vue @@ -101,6 +101,7 @@ import { type TabSortMode, type UpdateDownloadSource, type CsvQuoteMode, + type CsvNullMode, type CustomThemeColors, type CustomTheme, type McpConnectionPolicy, @@ -854,6 +855,7 @@ const editSidebarIndent = ref(settingsStore.editorSettings.sidebarIndent); const editSidebarFontSize = ref(settingsStore.editorSettings.sidebarFontSize); const editExportBatchSize = ref(settingsStore.editorSettings.exportBatchSize); const editCsvQuoteMode = ref(settingsStore.editorSettings.csvQuoteMode); +const editCsvNullMode = ref(settingsStore.editorSettings.csvNullMode); const editGlobalDateTimeDisplayFormat = ref(settingsStore.editorSettings.globalDateTimeDisplayFormat); const editGlobalDateTimeExportFormat = ref(settingsStore.editorSettings.globalDateTimeExportFormat); const editGlobalDateTimeImportFormat = ref(settingsStore.editorSettings.globalDateTimeImportFormat); @@ -1117,6 +1119,7 @@ function currentEditorSettingsDraft(): EditorSettingsDraft { redisDatabaseDisplayLimit: editRedisDatabaseDisplayLimit.value, exportBatchSize: editExportBatchSize.value, csvQuoteMode: editCsvQuoteMode.value, + csvNullMode: editCsvNullMode.value, globalDateTimeDisplayFormat: editGlobalDateTimeDisplayFormat.value, globalDateTimeExportFormat: editGlobalDateTimeExportFormat.value, globalDateTimeImportFormat: editGlobalDateTimeImportFormat.value, @@ -1779,6 +1782,7 @@ function syncEditorSettingsDraftFromStore() { editSidebarFontSize.value = settingsStore.editorSettings.sidebarFontSize; editExportBatchSize.value = settingsStore.editorSettings.exportBatchSize; editCsvQuoteMode.value = settingsStore.editorSettings.csvQuoteMode; + editCsvNullMode.value = settingsStore.editorSettings.csvNullMode; editGlobalDateTimeDisplayFormat.value = settingsStore.editorSettings.globalDateTimeDisplayFormat; editGlobalDateTimeExportFormat.value = settingsStore.editorSettings.globalDateTimeExportFormat; editGlobalDateTimeImportFormat.value = settingsStore.editorSettings.globalDateTimeImportFormat; @@ -1915,6 +1919,7 @@ const editorSettingsDraftRefs: EditorSettingsDraftRefMap = { redisDatabaseDisplayLimit: editRedisDatabaseDisplayLimit, exportBatchSize: editExportBatchSize, csvQuoteMode: editCsvQuoteMode, + csvNullMode: editCsvNullMode, exportRowLimitEnabled: editExportRowLimitEnabled, exportRowLimit: editExportRowLimit, queryExportKeysetOptimizationEnabled: editQueryExportKeysetOptimizationEnabled, @@ -2428,6 +2433,7 @@ function resetDefaultsForTab(tab: SettingsCategory) { editRedisDatabaseDisplayLimit.value = DEFAULT_EDITOR_SETTINGS.redisDatabaseDisplayLimit; editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize; editCsvQuoteMode.value = DEFAULT_EDITOR_SETTINGS.csvQuoteMode; + editCsvNullMode.value = DEFAULT_EDITOR_SETTINGS.csvNullMode; editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat; editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat; editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat; @@ -2572,6 +2578,7 @@ function resetAllDefaults() { editRedisDatabaseDisplayLimit.value = DEFAULT_EDITOR_SETTINGS.redisDatabaseDisplayLimit; editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize; editCsvQuoteMode.value = DEFAULT_EDITOR_SETTINGS.csvQuoteMode; + editCsvNullMode.value = DEFAULT_EDITOR_SETTINGS.csvNullMode; editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat; editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat; editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat; @@ -8550,6 +8557,23 @@ onUnmounted(() => { +
+
+ +

+ {{ t("settings.csvNullModeDescription") }} +

+
+ +
diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue index 2c4c371c8b..e6f7571181 100644 --- a/apps/desktop/src/components/import/TableImportDialog.vue +++ b/apps/desktop/src/components/import/TableImportDialog.vue @@ -769,7 +769,7 @@ async function prepareBatchSources(sources: ImportSource[]) { loadingPreview.value = false; return; } - emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true; + emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : defaultTableImportEmptyStringAsNull(formats[0] ?? "csv"); try { for (const [index, source] of sources.entries()) { const format = formats[index]!; @@ -1471,10 +1471,13 @@ watch(rawProgressPercent, (percent) => { {{ t("tableImport.trimValues") }} - +
+ +

{{ t("tableImport.emptyStringAsNullHint") }}

+
diff --git a/apps/desktop/src/components/objects/ObjectBrowser.vue b/apps/desktop/src/components/objects/ObjectBrowser.vue index c1c00783d1..de94dab757 100644 --- a/apps/desktop/src/components/objects/ObjectBrowser.vue +++ b/apps/desktop/src/components/objects/ObjectBrowser.vue @@ -102,6 +102,7 @@ import { isTauriRuntime } from "@/lib/backend/tauriRuntime"; import { generateDatabaseExportId } from "@/lib/export/databaseExport"; import { buildXlsxHeaderOverrides, hasXlsxHeaderComments, type XlsxExportOptions, type XlsxHeaderMode } from "@/lib/export/xlsxHeader"; import { showSqlInsertModeDialog, type SqlInsertDialect, type SqlInsertMode } from "@/lib/export/sqlInsertMode"; +import { csvNullLiteralForMode } from "@/lib/export/csvNullMode"; import { copyToClipboard, eventTargetAllowsAppClipboardShortcut } from "@/lib/common/clipboard"; import { defaultPasteTableMode, @@ -2423,7 +2424,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | " executePage: (sql) => api.executeQuery(props.connection.id, props.database, sql), }); if (format === "csv") { - await api.exportQueryResultCsv(filePath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), settingsStore.editorSettings.csvQuoteMode); + await api.exportQueryResultCsv(filePath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), settingsStore.editorSettings.csvQuoteMode, csvNullLiteralForMode(settingsStore.editorSettings.csvNullMode)); } else { const comments = result.columns.map((name) => columnInfos?.find((column) => column.name.toLocaleLowerCase() === name.toLocaleLowerCase())?.comment); const headerOverrides = buildXlsxHeaderOverrides(result.columns, comments, headerMode); @@ -2463,6 +2464,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | " format, ...(format === "sql" ? { insertMode, insertDialect, splitMaxMb } : {}), csvQuoteMode: settingsStore.editorSettings.csvQuoteMode, + nullLiteral: csvNullLiteralForMode(settingsStore.editorSettings.csvNullMode), columns, columnComments: format === "xlsx" ? columnComments : undefined, autoFilter: format === "xlsx" ? autoFilter : undefined, diff --git a/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts b/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts index aa4fbc9bfa..c563a403a4 100644 --- a/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts +++ b/apps/desktop/src/composables/__tests__/useDataGridExport.spec.ts @@ -13,6 +13,7 @@ import { DEFAULT_DATA_GRID_EXTRACTOR_OPTIONS } from "@/lib/dataGrid/dataGridCopy import { clearDataGridClipboardCopy, parseDataGridClipboard } from "@/lib/dataGrid/dataGridClipboard"; import { MONGO_DOCUMENT_GRID_NULL, mongoDocumentGridExternalValue } from "@/lib/mongo/mongoDocumentValues"; import { saveTextFile } from "@/lib/export/saveTextFile"; +import { useSettingsStore } from "@/stores/settingsStore"; const toast = vi.fn(); @@ -780,8 +781,8 @@ describe("useDataGridExport prepared row statements", () => { expect(copyToClipboard).toHaveBeenCalledWith("2020-12-02 15:18:29"); }); - it("copies all rows with empty fields for NULL cells", async () => { - const text = "id\tname\n1\t\n2\tAda"; + it("copies all rows with the null literal for NULL cells", async () => { + const text = "id\tname\n1\t\\N\n2\tAda"; const state = createExportState(editableTable, ["id", "name"], undefined, undefined, undefined, [ [1, null], [2, "Ada"], @@ -790,6 +791,7 @@ describe("useDataGridExport prepared row statements", () => { await state.copyAll(); expect(copyToClipboard).toHaveBeenCalledWith(text); + // 内部剪贴板按文本全等匹配保留逻辑矩阵:粘贴回网格时 NULL 语义无损还原 expect(parseDataGridClipboard(text)).toEqual([ ["id", "name"], ["1", null], @@ -1672,7 +1674,7 @@ describe("useDataGridExport prepared row statements", () => { }); await state.copyAll(); - expect(copyToClipboard).toHaveBeenCalledWith("_id\tnullable\n1\t"); + expect(copyToClipboard).toHaveBeenCalledWith("_id\tnullable\n1\t\\N"); vi.mocked(extractDataGridSelection).mockResolvedValueOnce({ text: "", mimeType: "text/tab-separated-values", fileExtension: "tsv", rowCount: 1, columnCount: 1 }); await expect(state.copyWithExtractor("tsv")).resolves.toBe(true); @@ -1680,7 +1682,7 @@ describe("useDataGridExport prepared row statements", () => { setActivePinia(createPinia()); await state.exportCurrentPageCsv(); - expect(exportQueryResultCsv).toHaveBeenCalledWith(expect.any(String), ["_id", "nullable"], [["1", null]], expect.anything()); + expect(exportQueryResultCsv).toHaveBeenCalledWith(expect.any(String), ["_id", "nullable"], [["1", null]], expect.anything(), expect.anything()); const reservedString = MONGO_DOCUMENT_GRID_NULL; const fullExportState = createMongoExportState({ @@ -1699,7 +1701,7 @@ describe("useDataGridExport prepared row statements", () => { }); await fullExportState.exportCsv(); - expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["_id", "value"], [["1", reservedString]], expect.anything()); + expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["_id", "value"], [["1", reservedString]], expect.anything(), expect.anything()); }); it("exports only visible Mongo columns from the full result set", async () => { @@ -1722,7 +1724,7 @@ describe("useDataGridExport prepared row statements", () => { await state.exportCsv(); - expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything()); + expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything(), expect.anything()); }); it("applies visible Mongo columns when the complete result is already local", async () => { @@ -1751,7 +1753,23 @@ describe("useDataGridExport prepared row statements", () => { await state.exportCsv(); - expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything()); + expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything(), expect.anything()); + }); + + it("passes the CSV NULL setting to the export call", async () => { + setActivePinia(createPinia()); + const state = createMongoExportState({ + columns: ["name"], + item: { ...row(["Visible"]), sourceIndex: 0 }, + mongoDocuments: [{ name: "Visible" }], + }); + + await state.exportCsv(); + expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"]], expect.anything(), "\\N"); + + useSettingsStore().editorSettings.csvNullMode = "empty"; + await state.exportCsv(); + expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"]], expect.anything(), ""); }); it("exports missing Mongo fields as null while retaining explicit empty strings", async () => { diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index d71a54a60e..8bc4b8af6b 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -21,6 +21,7 @@ import { summarizeExportRows } from "@/lib/export/exportDiagnostics"; import { appendDebugLog, appendNativeProcessMemoryLog, getBrowserMemorySnapshot, isDebugLoggingEnabled } from "@/lib/backend/debugLog"; import { uuid } from "@/lib/common/utils"; import { useSettingsStore } from "@/stores/settingsStore"; +import { csvNullLiteralForMode } from "@/lib/export/csvNullMode"; import { expandNestedJsonStringsForCopy } from "@/lib/common/jsonCopyValue"; import { buildMongoCopyDocumentFromOriginal, buildMongoCopyInsertDocument, buildMongoCopyUpdateDocument, formatMongoShellLiteral, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues"; import { formatMongoShellText } from "@/lib/mongo/mongoFormatter"; @@ -912,7 +913,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { } outputPath = path as string; } - await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode); + await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode, csvNullLiteralForMode(useSettingsStore().editorSettings.csvNullMode)); if (needsFullExport && exportProgressState) { exportProgressState.value = { ...exportProgressState.value, @@ -952,7 +953,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { } const result = await resultToExport(undefined, undefined, false); const rows = forceCsvTextForTemporalColumns(result.rows, result.columnTypes); - await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode); + await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode, csvNullLiteralForMode(useSettingsStore().editorSettings.csvNullMode)); toast(t("grid.exported")); } catch (e: any) { toast(t("grid.exportFailed", { message: translateBackendError(t, e) }), 5000); @@ -1389,6 +1390,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { format, ...(format === "sql" && sqlExportOptions ? { insertMode: sqlExportOptions.insertMode, splitMaxMb: sqlExportOptions.splitMaxMb, selectedColumns: sqlExportOptions.selectedColumns } : {}), csvQuoteMode: editorSettings.csvQuoteMode, + nullLiteral: csvNullLiteralForMode(editorSettings.csvNullMode), columns: format === "sql" ? effectiveColumns(sourceColumns.value, columns.value).map((column, index) => column ?? columns.value[index]!) : columns.value, columnTypes: columnTypes.value, ...(format === "sql" ? { columnExtras: sqlExportColumnExtras(effectiveColumns(sourceColumns.value, columns.value).map((column, index) => column ?? columns.value[index]!)) } : {}), @@ -1470,6 +1472,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { ...baseRequest, ...(format === "sql" ? { selectedColumns: sqlExportOptions?.selectedColumns } : {}), csvQuoteMode: useSettingsStore().editorSettings.csvQuoteMode, + nullLiteral: csvNullLiteralForMode(useSettingsStore().editorSettings.csvNullMode), ...sqlExportPrimaryKeyOptions(), dateTimeFormat: useSettingsStore().editorSettings.globalDateTimeExportFormat || undefined, numericColumnRightAlign: useSettingsStore().editorSettings.numericColumnRightAlign ?? true, diff --git a/apps/desktop/src/composables/useSidebarTreeExportRuntime.ts b/apps/desktop/src/composables/useSidebarTreeExportRuntime.ts index 231506423c..02b2c1ad83 100644 --- a/apps/desktop/src/composables/useSidebarTreeExportRuntime.ts +++ b/apps/desktop/src/composables/useSidebarTreeExportRuntime.ts @@ -32,6 +32,7 @@ import { structurePreviewDdlStorageType, } from "@/components/sidebar/sidebarTreeDialogState"; import type { CsvQuoteMode } from "@/lib/export/csvQuoteMode"; +import { csvNullLiteralForMode } from "@/lib/export/csvNullMode"; import type { SqlInsertDialect } from "@/lib/export/sqlInsertMode"; import { uuid } from "@/lib/common/utils"; @@ -59,6 +60,8 @@ interface SidebarTableExportTarget { batchSize: number; rowLimit: number | null; csvQuoteMode: CsvQuoteMode; + /** 导出 CSV 时 NULL 的字面量;空串表示关闭(旧行为)。 */ + nullLiteral: string; fileNameBase?: string; } @@ -437,6 +440,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt batchSize: editorSettings.exportBatchSize, rowLimit: editorSettings.exportRowLimitEnabled ? editorSettings.exportRowLimit : null, csvQuoteMode: editorSettings.csvQuoteMode, + nullLiteral: csvNullLiteralForMode(editorSettings.csvNullMode), }; } @@ -472,7 +476,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt executePage: (sql) => api.executeQuery(connectionId, database, sql), }); if (format === "csv") { - await api.exportQueryResultCsv(outputPath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), target.csvQuoteMode); + await api.exportQueryResultCsv(outputPath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), target.csvQuoteMode, target.nullLiteral); } else { const comments = result.columns.map((name) => exportColumnInfos?.find((column) => column.name.toLocaleLowerCase() === name.toLocaleLowerCase())?.comment); const headerOverrides = buildXlsxHeaderOverrides(result.columns, comments, headerMode); @@ -503,6 +507,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt format, ...(format === "sql" ? { insertDialect } : {}), csvQuoteMode: target.csvQuoteMode, + nullLiteral: target.nullLiteral, columns: queryColumns, columnComments, autoFilter: format === "xlsx" ? autoFilter : undefined, diff --git a/apps/desktop/src/i18n/locales/az.ts b/apps/desktop/src/i18n/locales/az.ts index 905adf1e20..fe5f0648f7 100644 --- a/apps/desktop/src/i18n/locales/az.ts +++ b/apps/desktop/src/i18n/locales/az.ts @@ -6291,6 +6291,7 @@ export default withEnglishFallback({ lastDataRow: "Son məlumat sətri (sona qədər üçün 0)", trimValues: "Dəyərlərin əvvəlindəki və sonundakı boşluqları sil", emptyStringAsNull: "Boş sətri NULL kimi qəbul et", + emptyStringAsNullHint: "İşarələndikdə boş sahələr NULL kimi idxal olunur və \\N adi mətn olur; əks halda boş sahələr boş sətir qalır və \\N NULL kimi idxal olunur.", jsonShape: "JSON sətirləri", jsonShapeAuto: "Avtomatik müəyyən et", jsonShapeObjects: "Obyektlər", @@ -7822,6 +7823,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "CSV sahələrində qoşa dırnaqların necə istifadə olunacağını seçin.", csvQuoteModeAll: "Bütün sahələr", csvQuoteModeNecessary: "Yalnız lazım olduqda", + csvNullMode: "CSV NULL təsviri", + csvNullModeDescription: "CSV ixracında NULL-un yazılma üsulu. «\\N» NULL ilə boş sətri fərqləndirir, yenidən idxalda boş sətirlər NULL olmur; «Boş» köhnə davranışı saxlayır, lakin onları fərqləndirmir.", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Boş", performanceSection: "Məhsuldarlıq", shortcutExecuteSql: "SQL-i icra et", shortcutExecuteSqlInNewResultTab: "SQL-i yeni nəticə vərəqində icra et", diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index 026ce181f0..a8cb1026cf 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -6899,6 +6899,7 @@ export default { lastDataRow: "Last data row (0 for end)", trimValues: "Trim values", emptyStringAsNull: "Empty string as NULL", + emptyStringAsNullHint: "When checked, empty fields import as NULL and \\N becomes plain text; otherwise empty fields stay as empty strings and \\N imports as NULL.", conflictPolicy: "On primary-key conflict", conflictError: "Stop with an error", skipDuplicateRows: "Skip duplicate rows", @@ -8550,6 +8551,10 @@ export default { csvQuoteModeDescription: "Choose how CSV fields use double quotes.", csvQuoteModeAll: "All fields", csvQuoteModeNecessary: "Only when needed", + csvNullMode: "CSV NULL representation", + csvNullModeDescription: 'How NULL is written when exporting CSV. "\\N" keeps NULL distinguishable from an empty string, so re-importing does not turn empty strings into NULL; "Empty" matches older versions but leaves the two indistinguishable.', + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Empty", performanceSection: "Performance", shortcutExecuteSql: "Execute SQL", shortcutExecuteSqlInNewResultTab: "Execute SQL in new result tab", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index 45d0adc76d..a9b97ffd59 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -6326,6 +6326,7 @@ export default withEnglishFallback({ hasHeader: "First row is header", trimValues: "Trim values", emptyStringAsNull: "Empty string as NULL", + emptyStringAsNullHint: "Si se activa, los campos vacíos se importan como NULL y \\N pasa a ser texto; si no, los campos vacíos se mantienen como cadenas vacías y \\N se importa como NULL.", jsonShape: "JSON rows", jsonShapeAuto: "Auto-detect", jsonShapeObjects: "Objects", @@ -7853,6 +7854,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "Elige cómo usar comillas dobles en los campos CSV.", csvQuoteModeAll: "Todos los campos", csvQuoteModeNecessary: "Solo cuando sea necesario", + csvNullMode: "Representación de NULL en CSV", + csvNullModeDescription: "Cómo se escribe NULL al exportar CSV. «\\N» distingue NULL de la cadena vacía, de modo que al reimportar las cadenas vacías no se convierten en NULL; «Vacío» mantiene el comportamiento anterior, pero no los distingue.", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Vacío", performanceSection: "Rendimiento", shortcutExecuteSql: "Ejecutar SQL", shortcutExecuteSqlInNewResultTab: "Ejecutar SQL en una nueva pestaña de resultados", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index 066efb71aa..055d174e73 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -6248,6 +6248,7 @@ export default withEnglishFallback({ hasHeader: "First row is header", trimValues: "Trim values", emptyStringAsNull: "Empty string as NULL", + emptyStringAsNullHint: "Se attivo, i campi vuoti vengono importati come NULL e \\N diventa testo normale; altrimenti restano stringhe vuote e \\N viene importato come NULL.", jsonShape: "JSON rows", jsonShapeAuto: "Auto-detect", jsonShapeObjects: "Objects", @@ -7777,6 +7778,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "Scegli come usare le virgolette doppie nei campi CSV.", csvQuoteModeAll: "Tutti i campi", csvQuoteModeNecessary: "Solo quando necessario", + csvNullMode: "Rappresentazione di NULL nel CSV", + csvNullModeDescription: "Come viene scritto NULL durante l'esportazione CSV. «\\N» distingue NULL dalla stringa vuota, così al reimport le stringhe vuote non diventano NULL; «Vuoto» mantiene il comportamento precedente, ma non li distingue.", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Vuoto", performanceSection: "Prestazioni", shortcutExecuteSql: "Esegui SQL", shortcutExecuteSqlInNewResultTab: "Esegui SQL in una nuova scheda dei risultati", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index f449a8c70f..0c0f3fae1c 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -6267,6 +6267,7 @@ export default withEnglishFallback({ hasHeader: "First row is header", trimValues: "Trim values", emptyStringAsNull: "Empty string as NULL", + emptyStringAsNullHint: "チェックすると空フィールドを NULL として取り込み、\\N は通常のテキストになります。チェックしない場合は空フィールドは空文字列のままとなり、\\N を NULL として取り込みます。", jsonShape: "JSON rows", jsonShapeAuto: "Auto-detect", jsonShapeObjects: "Objects", @@ -7776,6 +7777,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "CSV フィールドを二重引用符で囲む方法を選択します。", csvQuoteModeAll: "すべてのフィールド", csvQuoteModeNecessary: "必要な場合のみ", + csvNullMode: "CSV の NULL 表記", + csvNullModeDescription: "CSV エクスポート時に NULL を書く方式です。「\\N」は NULL と空文字列を区別でき、再インポートしても空文字列が NULL になりません。「空」は以前の動作と同じですが区別できません。", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "空", performanceSection: "パフォーマンス", shortcutExecuteSql: "SQLを実行", shortcutExecuteSqlInNewResultTab: "新しい結果タブでSQLを実行", diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index b8cc10b0d2..4286a2c4dd 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -5981,6 +5981,7 @@ export default withEnglishFallback({ lastDataRow: "마지막 데이터 행 (끝까지면 0)", trimValues: "값 양끝 공백 제거", emptyStringAsNull: "빈 문자열을 NULL로", + emptyStringAsNullHint: "체크하면 빈 필드를 NULL로 가져오고 \\N은 일반 텍스트가 됩니다. 해제하면 빈 필드는 빈 문자열로 유지되고 \\N은 NULL로 가져옵니다.", jsonShape: "JSON 행", jsonShapeAuto: "자동 감지", jsonShapeObjects: "객체", @@ -8083,6 +8084,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "CSV 내보내기 시 필드에 큰따옴표를 사용하는 방식을 선택합니다.", csvQuoteModeAll: "모든 필드", csvQuoteModeNecessary: "필수 필드만", + csvNullMode: "CSV NULL 표기", + csvNullModeDescription: 'CSV 내보내기 시 NULL을 쓰는 방식입니다. "\\N"은 NULL과 빈 문자열을 구분하므로 다시 가져올 때 빈 문자열이 NULL이 되지 않습니다. "비움"은 이전 동작과 같지만 구분할 수 없습니다.', + csvNullModeMarker: "\\N", + csvNullModeEmpty: "비움", shortcutConvertNamingStyle: "네이밍 스타일 전환", dataGridKeepFilterEditorExpanded: "필터 편집기 기본 펼치기", dataGridKeepFilterEditorExpandedDescription: "조건 및 텍스트 보기에서 편집기를 처음에 펼칩니다. 수동으로 접을 수도 있습니다.", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index 0fa6017f57..bd597346f0 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -6250,6 +6250,7 @@ export default withEnglishFallback({ hasHeader: "First row is header", trimValues: "Trim values", emptyStringAsNull: "Empty string as NULL", + emptyStringAsNullHint: "Quando marcado, campos vazios são importados como NULL e \\N passa a ser texto; caso contrário, permanecem como strings vazias e \\N é importado como NULL.", jsonShape: "JSON rows", jsonShapeAuto: "Auto-detect", jsonShapeObjects: "Objects", @@ -7779,6 +7780,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "Escolha como usar aspas duplas nos campos CSV.", csvQuoteModeAll: "Todos os campos", csvQuoteModeNecessary: "Somente quando necessário", + csvNullMode: "Representação de NULL no CSV", + csvNullModeDescription: "Como NULL é escrito ao exportar CSV. «\\N» distingue NULL da string vazia, então ao reimportar as strings vazias não viram NULL; «Vazio» mantém o comportamento antigo, mas não os distingue.", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Vazio", performanceSection: "Desempenho", shortcutExecuteSql: "Executar SQL", shortcutExecuteSqlInNewResultTab: "Executar SQL em uma nova aba de resultados", diff --git a/apps/desktop/src/i18n/locales/ru.ts b/apps/desktop/src/i18n/locales/ru.ts index b107f3af09..aa87cfd36d 100644 --- a/apps/desktop/src/i18n/locales/ru.ts +++ b/apps/desktop/src/i18n/locales/ru.ts @@ -7418,6 +7418,7 @@ export default withEnglishFallback({ lastDataRow: "Последняя строка данных (0 — до конца)", trimValues: "Обрезать значения", emptyStringAsNull: "Пустая строка как NULL", + emptyStringAsNullHint: "Если включено, пустые поля импортируются как NULL, а \\N считается обычным текстом; иначе пустые поля остаются пустыми строками, а \\N импортируется как NULL.", jsonShape: "Строки JSON", jsonShapeAuto: "Автоопределение", jsonShapeObjects: "Объекты", @@ -9049,6 +9050,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "Выберите, как поля CSV используют двойные кавычки.", csvQuoteModeAll: "Все поля", csvQuoteModeNecessary: "Только когда нужно", + csvNullMode: "Представление NULL в CSV", + csvNullModeDescription: "Как записывается NULL при экспорте CSV. «\\N» отличает NULL от пустой строки, поэтому при повторном импорте пустые строки не станут NULL; «Пусто» сохраняет прежнее поведение, но не различает их.", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Пусто", performanceSection: "Производительность", shortcutExecuteSql: "Выполнить SQL", shortcutExecuteSqlInNewResultTab: "Выполнить SQL в новой вкладке результатов", diff --git a/apps/desktop/src/i18n/locales/tr.ts b/apps/desktop/src/i18n/locales/tr.ts index 99bfb6b789..a7db25b9fa 100644 --- a/apps/desktop/src/i18n/locales/tr.ts +++ b/apps/desktop/src/i18n/locales/tr.ts @@ -6173,6 +6173,7 @@ export default withEnglishFallback({ lastDataRow: "Son veri satırı (sona kadar için 0)", trimValues: "Değerleri kırp", emptyStringAsNull: "Boş dizeyi NULL say", + emptyStringAsNullHint: "İşaretlendiğinde boş alanlar NULL olarak içe aktarılır ve \\N düz metin olur; aksi halde boş alanlar boş dize kalır ve \\N NULL olarak içe aktarılır.", jsonShape: "JSON satırları", jsonShapeAuto: "Otomatik algıla", jsonShapeObjects: "Nesneler", @@ -7721,6 +7722,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "CSV alanlarının çift tırnağı nasıl kullanacağını seçin.", csvQuoteModeAll: "Tüm alanlar", csvQuoteModeNecessary: "Yalnızca gerektiğinde", + csvNullMode: "CSV NULL gösterimi", + csvNullModeDescription: "CSV dışa aktarılırken NULL nasıl yazılır. «\\N» NULL ile boş dizeyi ayırır, böylece yeniden içe aktarımda boş dizeler NULL olmaz; «Boş» eski davranışı korur ancak ikisini ayırmaz.", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "Boş", performanceSection: "Başarım", shortcutExecuteSql: "SQL'i çalıştır", shortcutExecuteSqlInNewResultTab: "SQL'i yeni sonuç sekmesinde çalıştır", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index 515e968b6f..677c31c0e8 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -6866,6 +6866,7 @@ export default withEnglishFallback({ lastDataRow: "最后数据行(0 表示末尾)", trimValues: "裁剪空白", emptyStringAsNull: "空字符串作为 NULL", + emptyStringAsNullHint: "勾选后空字段一律按 NULL 导入(此时文件里的 \\N 按普通文本导入);不勾选则空字段保留为空字符串,并把 \\N 按 NULL 导入。", conflictPolicy: "主键冲突时", conflictError: "停止并报错", skipDuplicateRows: "跳过重复行", @@ -8513,6 +8514,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "选择 CSV 导出时字段使用双引号的方式。", csvQuoteModeAll: "全部字段", csvQuoteModeNecessary: "仅必要字段", + csvNullMode: "CSV NULL 表示", + csvNullModeDescription: "导出 CSV 时 NULL 的写法。选「\\N」可区分 NULL 与空字符串,重新导入时不会把空字符串写成 NULL;选「留空」与旧版本一致,但两者在文件里无法区分。", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "留空", performanceSection: "性能", shortcutExecuteSql: "执行 SQL", shortcutExecuteSqlInNewResultTab: "在新结果标签页中执行 SQL", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index 1714981ada..b6124f8436 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -5557,6 +5557,7 @@ export default withEnglishFallback({ hasHeader: "首列為標頭", trimValues: "裁剪空白", emptyStringAsNull: "空字串作為 NULL", + emptyStringAsNullHint: "勾選後空欄位一律以 NULL 匯入(此時檔案裡的 \\N 以一般文字匯入);不勾選則空欄位保留為空字串,並把 \\N 以 NULL 匯入。", conflictPolicy: "主鍵衝突時", conflictError: "停止並回報錯誤", skipDuplicateRows: "略過重複列", @@ -7610,6 +7611,10 @@ export default withEnglishFallback({ csvQuoteModeDescription: "選擇 CSV 匯出時欄位使用雙引號的方式。", csvQuoteModeAll: "全部欄位", csvQuoteModeNecessary: "僅必要欄位", + csvNullMode: "CSV NULL 表示", + csvNullModeDescription: "匯出 CSV 時 NULL 的寫法。選「\\N」可區分 NULL 與空字串,重新匯入時不會把空字串寫成 NULL;選「留空」與舊版本一致,但兩者在檔案裡無法區分。", + csvNullModeMarker: "\\N", + csvNullModeEmpty: "留空", performanceSection: "效能", exportBatchSize: "匯出批次大小", dateTimeSection: "日期與時間", diff --git a/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts b/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts index a4ca241a08..bd152dc625 100644 --- a/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts +++ b/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts @@ -15,12 +15,34 @@ import { } from "@/lib/table/tableImport"; describe("tableImport", () => { - it("preserves explicit empty strings by default for Excel only", () => { + it("keeps empty fields as empty strings for every source format by default", () => { + // 导出端默认用 `\N` 表示 NULL,空字段代表空字符串;导入端默认必须与之一致, + // 否则 DBX 导出的 CSV 再导回时会把 '' 变成 NULL。 expect(defaultTableImportEmptyStringAsNull("excel")).toBe(false); - expect(defaultTableImportEmptyStringAsNull("csv")).toBe(true); - expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(true); - expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(true); - expect(defaultTableImportEmptyStringAsNull("json")).toBe(true); + expect(defaultTableImportEmptyStringAsNull("csv")).toBe(false); + expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(false); + expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(false); + expect(defaultTableImportEmptyStringAsNull("json")).toBe(false); + }); + + it("only disables the shared NULL literal when empty fields are treated as NULL", () => { + const baseSettings = { + delimiter: ",", + textEncoding: "auto" as const, + titleRow: 1, + dataStartRow: 2, + lastDataRow: 0, + trimValues: false, + jsonShape: "auto" as const, + }; + + const sharedLiteral = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: false }); + expect(sharedLiteral.nullLiteral).toBeUndefined(); + expect(sharedLiteral.emptyStringAsNull).toBe(false); + + const legacyEmptyField = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: true }); + expect(legacyEmptyField.nullLiteral).toBe(""); + expect(legacyEmptyField.emptyStringAsNull).toBe(true); }); it("formats import elapsed time for progress and terminal summaries", () => { diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts index ef5a3e9013..fecfa5d99b 100644 --- a/apps/desktop/src/lib/backend/http.ts +++ b/apps/desktop/src/lib/backend/http.ts @@ -3388,9 +3388,9 @@ export async function cancelQueryResultExport(exportId: string, executionId?: st }); } -export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise { +export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise { const { formatCsv } = await import("@/lib/export/exportFormats"); - const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode); + const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode, nullLiteral); const fileName = filePath.split(/[\\/]/).pop() || "export.csv"; const blob = new Blob(["\uFEFF", content], { type: "text/csv;charset=utf-8", diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 6184f1fabb..1024176e8f 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -5466,6 +5466,8 @@ export interface TableImportParseOptions { lastDataRow?: number | null; trimValues?: boolean | null; emptyStringAsNull?: boolean | null; + /** 分隔文本里代表 NULL 的字面量。缺省表示用后端默认值 `\N`;空串表示关闭字面量,退回「空字段即 NULL」。 */ + nullLiteral?: string | null; sheetName?: string | null; sheetIndex?: number | null; jsonShape?: TableImportJsonShape | null; @@ -5861,6 +5863,8 @@ export interface TableExportRequest { insertMode?: SqlInsertMode; insertDialect?: SqlInsertDialect; csvQuoteMode?: CsvQuoteMode; + /** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */ + nullLiteral?: string; columns?: string[]; selectedColumns?: SqlExportColumnSelection[]; columnTypes?: Array; @@ -5891,6 +5895,8 @@ export interface TableCsvExportOptions { pageSize?: number; timeoutSecs?: number; csvQuoteMode?: CsvQuoteMode; + /** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */ + nullLiteral?: string; } export interface TableExportProgress { @@ -5917,6 +5923,8 @@ export interface QueryResultExportRequest { format: "csv" | "xlsx" | "json" | "txt" | "sql"; insertMode?: SqlInsertMode; csvQuoteMode?: CsvQuoteMode; + /** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */ + nullLiteral?: string; includeSqlSheet?: boolean; pageSize: number; rowLimit?: number | null; @@ -6074,13 +6082,14 @@ export async function recordDatabaseExportDestination(directory: string): Promis await invoke("record_database_export_destination", { directory }); } -export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise { +export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise { return invoke("export_query_result_csv", { request: { filePath, columns, rows, csvQuoteMode, + ...(nullLiteral === undefined ? {} : { nullLiteral }), }, }); } diff --git a/apps/desktop/src/lib/export/__tests__/csvNullMode.spec.ts b/apps/desktop/src/lib/export/__tests__/csvNullMode.spec.ts new file mode 100644 index 0000000000..bb4a2d2670 --- /dev/null +++ b/apps/desktop/src/lib/export/__tests__/csvNullMode.spec.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { csvNullLiteralForMode, DEFAULT_CSV_NULL_LITERAL, DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode } from "@/lib/export/csvNullMode"; + +describe("normalizeCsvNullMode", () => { + it("defaults to writing the NULL marker", () => { + expect(DEFAULT_CSV_NULL_MODE).toBe("marker"); + expect(normalizeCsvNullMode(undefined)).toBe("marker"); + expect(normalizeCsvNullMode(null)).toBe("marker"); + expect(normalizeCsvNullMode("marker")).toBe("marker"); + }); + + it("keeps an explicit request for the legacy empty field", () => { + expect(normalizeCsvNullMode("empty")).toBe("empty"); + }); + + it("falls back to the default for unknown values", () => { + expect(normalizeCsvNullMode("")).toBe("marker"); + expect(normalizeCsvNullMode("\\N")).toBe("marker"); + expect(normalizeCsvNullMode(0)).toBe("marker"); + }); +}); + +describe("csvNullLiteralForMode", () => { + it("writes the MySQL-style NULL marker by default", () => { + expect(csvNullLiteralForMode("marker")).toBe(DEFAULT_CSV_NULL_LITERAL); + expect(csvNullLiteralForMode("marker")).toBe("\\N"); + }); + + it("disables the literal with an empty string to restore the old export", () => { + expect(csvNullLiteralForMode("empty")).toBe(""); + }); +}); diff --git a/apps/desktop/src/lib/export/__tests__/exportFormats.spec.ts b/apps/desktop/src/lib/export/__tests__/exportFormats.spec.ts index d6f48dc24c..fb22d357fb 100644 --- a/apps/desktop/src/lib/export/__tests__/exportFormats.spec.ts +++ b/apps/desktop/src/lib/export/__tests__/exportFormats.spec.ts @@ -1,5 +1,58 @@ import { describe, expect, it } from "vitest"; -import { formatTsv } from "@/lib/export/exportFormats"; +import { formatCsv, formatTsv } from "@/lib/export/exportFormats"; + +describe("formatCsv NULL handling", () => { + it("writes NULL as an unquoted marker and an empty string as an empty quoted field", () => { + // 即使 quote mode = all,NULL 字面量也必须裸写:PostgreSQL/ClickHouse 的 CSV 读取 + // 不把带引号的 `"\N"` 当作 NULL,裸 `\N` 才是它们和 MySQL 共同识别的写法。 + expect( + formatCsv( + ["id", "name"], + [ + [1, ""], + [2, null], + ], + "all", + "\\N", + ), + ).toBe('"id","name"\n"1",""\n"2",\\N'); + }); + + it("keeps the marker unquoted when only necessary fields are quoted", () => { + expect( + formatCsv( + ["id", "name"], + [ + [1, ""], + [2, null], + ], + "necessary", + "\\N", + ), + ).toBe("id,name\n1,\n2,\\N"); + }); + + it("quotes a marker that itself needs quoting", () => { + // 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段 + expect(formatCsv(["id", "name"], [[1, null]], "all", "a,b")).toBe('"id","name"\n"1","a,b"'); + }); + + it("restores the legacy empty field when the marker is disabled", () => { + // 关闭字面量时 NULL 是完全空字段、空字符串是 `""`:CSV 引号语义上仍有区别, + // 但 DBX 的导入端比较的是去引号后的值,因此旧格式在往返时依赖不了这个区别。 + expect( + formatCsv( + ["id", "name"], + [ + [1, ""], + [2, null], + ], + "all", + "", + ), + ).toBe('"id","name"\n"1",""\n"2",'); + }); +}); describe("formatTsv", () => { it("copies a value that only contains double quotes verbatim (#10087)", () => { @@ -14,8 +67,20 @@ describe("formatTsv", () => { expect(output).toBe('v\n"a\tb"\n"c\nd"'); }); - it("renders NULL as an empty field without adding quotes", () => { - const output = formatTsv(["id", "name"], [[1, null]]); + it("renders NULL as the null literal and an empty string as an empty field", () => { + const output = formatTsv( + ["id", "name"], + [ + [1, null], + [2, ""], + ], + ); + + expect(output).toBe("id\tname\n1\t\\N\n2\t"); + }); + + it("falls back to an empty NULL field when the literal is disabled", () => { + const output = formatTsv(["id", "name"], [[1, null]], ""); expect(output).toBe("id\tname\n1\t"); }); diff --git a/apps/desktop/src/lib/export/csvNullMode.ts b/apps/desktop/src/lib/export/csvNullMode.ts new file mode 100644 index 0000000000..530cc4a867 --- /dev/null +++ b/apps/desktop/src/lib/export/csvNullMode.ts @@ -0,0 +1,23 @@ +/** + * 导出 CSV 时 NULL 的写法。与 Rust `dbx_formats::csv_export` 保持一致: + * `marker` 把 NULL 写成 `\N`(MySQL `SELECT ... INTO OUTFILE`、ClickHouse + * `format_csv_null_representation` 的既定写法),空字符串才写成空字段,两者在文件里可区分, + * 导回 NOT NULL DEFAULT '' 的列时不会被还原成 NULL;`empty` 是旧行为:NULL 与空字符串都写成空字段。 + */ +export const DEFAULT_CSV_NULL_LITERAL = "\\N"; + +export const CSV_NULL_MODE_MARKER = "marker"; +export const CSV_NULL_MODE_EMPTY = "empty"; + +export type CsvNullMode = typeof CSV_NULL_MODE_MARKER | typeof CSV_NULL_MODE_EMPTY; + +export const DEFAULT_CSV_NULL_MODE: CsvNullMode = CSV_NULL_MODE_MARKER; + +export function normalizeCsvNullMode(value: unknown): CsvNullMode { + return value === CSV_NULL_MODE_EMPTY ? CSV_NULL_MODE_EMPTY : DEFAULT_CSV_NULL_MODE; +} + +/** 导出请求里的 NULL 字面量;空串表示关闭该字面量(旧行为)。 */ +export function csvNullLiteralForMode(mode: CsvNullMode): string { + return mode === CSV_NULL_MODE_EMPTY ? "" : DEFAULT_CSV_NULL_LITERAL; +} diff --git a/apps/desktop/src/lib/export/exportFormats.ts b/apps/desktop/src/lib/export/exportFormats.ts index c89c62fa20..eb00b22747 100644 --- a/apps/desktop/src/lib/export/exportFormats.ts +++ b/apps/desktop/src/lib/export/exportFormats.ts @@ -1,26 +1,35 @@ import type { DatabaseType, QueryResult } from "@/types/database"; import * as api from "@/lib/backend/api"; import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode"; +import { DEFAULT_CSV_NULL_LITERAL } from "@/lib/export/csvNullMode"; import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode"; export type ExportCellValue = string | number | boolean | null; -export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all"): string { +export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all", nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string { const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(","); - const body = rows.map((row) => row.map((cell) => (cell === null ? "" : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n"); + // NULL 写成独立字面量、空字符串写成空字段,否则导入端无法把两者区分开 + // (写回 NOT NULL DEFAULT '' 的列时会把空字符串还原成 NULL)。 + // + // 字面量裸写:PostgreSQL 的 COPY ... FORMAT csv 不认带引号的 NULL(`"\N"` 是普通字符串, + // 整数列直接报错),ClickHouse 的 CSV 读取在引号内遇到 `\N` 会解析失败;只有裸 `\N` 是 + // 它们与 MySQL 共同识别的 NULL。字面量自身含分隔符/引号/换行时才加引号。 + const nullField = nullLiteral ? escapeCsvField(nullLiteral, "necessary") : ""; + const body = rows.map((row) => row.map((cell) => (cell === null ? nullField : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n"); return `${header}\n${body}`; } // Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)" -// export: fields are joined by a tab and NULL becomes empty. A field is wrapped -// in double quotes only when it contains a tab or a line break - the characters -// that would otherwise corrupt the TSV row/column shape. A field that merely -// contains a double quote is emitted verbatim: TSV is parsed by splitting on the -// tab alone (there is no quote state machine on paste-back), so quoting such a -// value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`). -export function formatTsv(columns: string[], rows: ExportCellValue[][]): string { +// export: fields are joined by a tab and NULL becomes the null literal. A field +// is wrapped in double quotes only when it contains a tab or a line break - the +// characters that would otherwise corrupt the TSV row/column shape. A field that +// merely contains a double quote is emitted verbatim: TSV is parsed by splitting +// on the tab alone (there is no quote state machine on paste-back), so quoting +// such a value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`). +export function formatTsv(columns: string[], rows: ExportCellValue[][], nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string { const esc = (value: ExportCellValue) => { - const text = value === null ? "" : String(value); + // 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返 + const text = value === null ? nullLiteral || "" : String(value); if (text.includes("\t") || text.includes("\n") || text.includes("\r")) { return `"${text.replace(/"/g, '""')}"`; } diff --git a/apps/desktop/src/lib/settings/__tests__/editorSettingsDraft.spec.ts b/apps/desktop/src/lib/settings/__tests__/editorSettingsDraft.spec.ts index 247c3dd45a..31c76d3df9 100644 --- a/apps/desktop/src/lib/settings/__tests__/editorSettingsDraft.spec.ts +++ b/apps/desktop/src/lib/settings/__tests__/editorSettingsDraft.spec.ts @@ -236,6 +236,11 @@ describe("editorSettingsDraftFromSettings", () => { expect(draft.completionTriggerMode).toBe("positional"); }); + it("maps csvNullMode and normalizes an unknown value back to the marker", () => { + expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "empty" })).csvNullMode).toBe("empty"); + expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "bogus" as unknown } as Partial)).csvNullMode).toBe("marker"); + }); + it("maps and normalizes tableCompletionSchemaQualification", () => { expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "always" })).tableCompletionSchemaQualification).toBe("always"); expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "invalid" } as any)).tableCompletionSchemaQualification).toBe("collision"); diff --git a/apps/desktop/src/lib/settings/__tests__/settingsTransfer.spec.ts b/apps/desktop/src/lib/settings/__tests__/settingsTransfer.spec.ts index a150908fff..c91f9a9b29 100644 --- a/apps/desktop/src/lib/settings/__tests__/settingsTransfer.spec.ts +++ b/apps/desktop/src/lib/settings/__tests__/settingsTransfer.spec.ts @@ -413,4 +413,9 @@ describe("settingsTransfer", () => { expect(transferCategoryForKey("csvQuoteMode")).toBe("data"); expect(collectTransferCategories(["csvQuoteMode", "pageSize"])).toEqual(["data"]); }); + + it("maps csvNullMode into the data category", () => { + expect(transferCategoryForKey("csvNullMode")).toBe("data"); + expect(collectTransferCategories(["csvNullMode", "csvQuoteMode"])).toEqual(["data"]); + }); }); diff --git a/apps/desktop/src/lib/settings/editorSettingsDraft.ts b/apps/desktop/src/lib/settings/editorSettingsDraft.ts index 59147ddcfb..2cc9b5e89c 100644 --- a/apps/desktop/src/lib/settings/editorSettingsDraft.ts +++ b/apps/desktop/src/lib/settings/editorSettingsDraft.ts @@ -7,6 +7,7 @@ import { normalizeCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPo import { normalizeSqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification"; import { normalizeTableHoverLookupMode } from "@/lib/editor/hoverTableLookup"; import { normalizeRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates"; +import { normalizeCsvNullMode } from "@/lib/export/csvNullMode"; export const EDITOR_SETTINGS_DRAFT_KEYS = [ "fontFamily", @@ -120,6 +121,7 @@ export const EDITOR_SETTINGS_DRAFT_KEYS = [ "redisDatabaseDisplayLimit", "exportBatchSize", "csvQuoteMode", + "csvNullMode", "exportRowLimitEnabled", "exportRowLimit", "queryExportKeysetOptimizationEnabled", @@ -164,6 +166,7 @@ function normalizedDraftValue(key: EditorSettingsDraftKey, value: unknown): unkn if (key === "tableCompletionSchemaQualification") return normalizeSqlTableCompletionSchemaQualification(value); if (key === "tableHoverLookupMode") return normalizeTableHoverLookupMode(value); if (key === "redisKeyTemplates") return normalizeRedisKeyTemplates(value); + if (key === "csvNullMode") return normalizeCsvNullMode(value); if (key === "backgroundImage") return normalizeBackgroundImageSettings(value); return value; } diff --git a/apps/desktop/src/lib/settings/settingsTransfer.ts b/apps/desktop/src/lib/settings/settingsTransfer.ts index c2eb694467..14aa23339a 100644 --- a/apps/desktop/src/lib/settings/settingsTransfer.ts +++ b/apps/desktop/src/lib/settings/settingsTransfer.ts @@ -178,6 +178,7 @@ const SETTINGS_TRANSFER_CATEGORY_KEYS: Record { keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled, csvQuoteMode: settings.csvQuoteMode, clientSessionId, + nullLiteral: csvNullLiteralForMode(settings.csvNullMode), executionId: uuid(), exportTableName: options.exportTableName, exportColumnTypes: options.exportColumnTypes, @@ -9277,6 +9279,7 @@ export const useQueryStore = defineStore("query", () => { keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled, csvQuoteMode: settings.csvQuoteMode, clientSessionId: `${tabClientSessionId(tab, "export")}:${exportId}`, + nullLiteral: csvNullLiteralForMode(settings.csvNullMode), executionId: uuid(), numericColumnRightAlign: settings.numericColumnRightAlign, columnComments, diff --git a/apps/desktop/src/stores/settingsStore.ts b/apps/desktop/src/stores/settingsStore.ts index 88d4e08d75..e8cf71ffef 100644 --- a/apps/desktop/src/stores/settingsStore.ts +++ b/apps/desktop/src/stores/settingsStore.ts @@ -36,6 +36,7 @@ import { normalizeTableHoverLookupMode, type TableHoverLookupMode } from "@/lib/ import { normalizeCompletionTriggerMode, type SqlCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPolicy"; import { DEFAULT_SQL_TABLE_COMPLETION_SCHEMA_QUALIFICATION, normalizeSqlTableCompletionSchemaQualification, type SqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification"; import { DEFAULT_CSV_QUOTE_MODE, normalizeCsvQuoteMode, type CsvQuoteMode } from "@/lib/export/csvQuoteMode"; +import { DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode, type CsvNullMode } from "@/lib/export/csvNullMode"; import { configureMetadataRuntimeCache, METADATA_CACHE_DEFAULT_MEMORY_MB, normalizeMetadataCacheMemoryMb } from "@/lib/metadata/metadataRuntimeCache"; import type { AiApiStyle, AiAssistantMode, AiAuthMethod, AiChatSelectionState, AiConfig, AiConfigItem, AiConfiguredModel, AiEffortLevel, AiEffortSelection, AiModelEffortPreference, AiProvider, AiReasoningLevel, AiTestConnectionResult } from "@/types/ai"; import type { SqlShortcutAction, SqlSnippet, TableInfoTab } from "@/types/database"; @@ -53,6 +54,7 @@ export type { AiReasoningLevel, AiTestConnectionResult, CsvQuoteMode, + CsvNullMode, DataTabReuseMode, SavedSqlOpenTargetMode, SqlCompletionTriggerMode, @@ -998,6 +1000,7 @@ export interface EditorSettings { tableColumnTemplateFields: string[]; exportBatchSize: number; csvQuoteMode: CsvQuoteMode; + csvNullMode: CsvNullMode; /** Global Redis key-search templates; overridden by non-empty connection templates. */ redisKeyTemplates: string[]; /** Sidebar database-list cap for Redis connections; the rest are revealed via "load more". */ @@ -1279,6 +1282,7 @@ export const DEFAULT_EDITOR_SETTINGS: EditorSettings = { tableColumnTemplateFields: [...DEFAULT_TABLE_COLUMN_TEMPLATE_FIELDS], exportBatchSize: 2000, csvQuoteMode: DEFAULT_CSV_QUOTE_MODE, + csvNullMode: DEFAULT_CSV_NULL_MODE, redisKeyTemplates: [], redisDatabaseDisplayLimit: REDIS_DATABASE_DISPLAY_LIMIT_DEFAULT, exportRowLimitEnabled: false, @@ -1896,6 +1900,7 @@ export function normalizeEditorSettings(settings: Partial, exist tableColumnTemplateFields: normalizeTableColumnTemplateFields(settings.tableColumnTemplateFields), exportBatchSize: typeof settings.exportBatchSize === "number" && settings.exportBatchSize >= 100 && settings.exportBatchSize <= 100000 ? Math.round(settings.exportBatchSize) : DEFAULT_EDITOR_SETTINGS.exportBatchSize, csvQuoteMode: normalizeCsvQuoteMode(settings.csvQuoteMode), + csvNullMode: normalizeCsvNullMode(settings.csvNullMode), redisKeyTemplates: normalizeRedisKeyTemplates(settings.redisKeyTemplates), redisDatabaseDisplayLimit: typeof settings.redisDatabaseDisplayLimit === "number" && settings.redisDatabaseDisplayLimit >= REDIS_DATABASE_DISPLAY_LIMIT_MIN && settings.redisDatabaseDisplayLimit <= REDIS_DATABASE_DISPLAY_LIMIT_MAX @@ -2703,6 +2708,7 @@ export const useSettingsStore = defineStore("settings", () => { if (partial.tableColumnTemplateFields !== undefined) editorSettings.value.tableColumnTemplateFields = normalizeTableColumnTemplateFields(partial.tableColumnTemplateFields); if (partial.exportBatchSize !== undefined) editorSettings.value.exportBatchSize = Math.min(100000, Math.max(100, Math.round(partial.exportBatchSize))); if (partial.csvQuoteMode !== undefined) editorSettings.value.csvQuoteMode = normalizeCsvQuoteMode(partial.csvQuoteMode); + if (partial.csvNullMode !== undefined) editorSettings.value.csvNullMode = normalizeCsvNullMode(partial.csvNullMode); if (partial.redisKeyTemplates !== undefined) editorSettings.value.redisKeyTemplates = normalizeRedisKeyTemplates(partial.redisKeyTemplates); if (partial.redisDatabaseDisplayLimit !== undefined) editorSettings.value.redisDatabaseDisplayLimit = Math.min(REDIS_DATABASE_DISPLAY_LIMIT_MAX, Math.max(REDIS_DATABASE_DISPLAY_LIMIT_MIN, Math.round(partial.redisDatabaseDisplayLimit))); if (partial.exportRowLimitEnabled !== undefined) editorSettings.value.exportRowLimitEnabled = partial.exportRowLimitEnabled; diff --git a/crates/dbx-core/src/data/csv_export.rs b/crates/dbx-core/src/data/csv_export.rs index c410a429d9..f8c26da754 100644 --- a/crates/dbx-core/src/data/csv_export.rs +++ b/crates/dbx-core/src/data/csv_export.rs @@ -30,6 +30,10 @@ pub struct TableCsvExportOptions { pub timeout_secs: Option, #[serde(default)] pub csv_quote_mode: CsvQuoteMode, + /// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量, + /// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串无法区分)。 + #[serde(default = "default_csv_null_literal")] + pub null_literal: String, } async fn connection_database_type(state: &AppState, connection_id: &str) -> Result { @@ -156,7 +160,12 @@ where for row in result.rows { writer.write_all(b"\n").map_err(|err| err.to_string())?; - write_csv_value_row(writer, row, options.csv_quote_mode)?; + write_csv_value_row_with_options( + writer, + row, + options.csv_quote_mode, + csv_null_literal(&options.null_literal), + )?; } rows_exported += fetched as u64; @@ -191,6 +200,7 @@ mod tests { page_size: Some(page_size), timeout_secs: Some(30), csv_quote_mode: CsvQuoteMode::default(), + null_literal: String::new(), } } @@ -354,6 +364,194 @@ mod tests { config } + /// 表数据导出必须把 NULL 写成字面量、空字符串写成 `""`,否则导入端拿不到引号信息, + /// 两者都会变成 NULL(正是「NOT NULL DEFAULT '' 的列导入报错」的成因)。 + #[tokio::test] + async fn streamed_table_export_writes_null_literal_and_keeps_empty_strings() { + fn result_with_a_null_and_an_empty_string() -> QueryResult { + serde_json::from_value(serde_json::json!({ + "columns": ["name", "note"], + "rows": [["a", null], ["b", ""]], + "affected_rows": 0, + "execution_time_ms": 1, + "session_id": null, + "has_more": false + })) + .unwrap() + } + + let mut options = export_options(10); + options.null_literal = dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string(); + let mut pages = vec![result_with_a_null_and_an_empty_string()].into_iter(); + let mut output = Vec::new(); + write_table_csv_pages(&mut output, DatabaseType::Postgres, &options, None, |_, _| { + std::future::ready(Ok(pages.next().expect("single page"))) + }) + .await + .unwrap(); + assert_eq!(String::from_utf8(output).unwrap(), "\"name\",\"note\"\n\"a\",\\N\n\"b\",\"\""); + + // 关闭字面量(旧行为):NULL 与空串都写空字段,二者在文件里已无法区分 + options.null_literal = String::new(); + let mut pages = vec![result_with_a_null_and_an_empty_string()].into_iter(); + let mut output = Vec::new(); + write_table_csv_pages(&mut output, DatabaseType::Postgres, &options, None, |_, _| { + std::future::ready(Ok(pages.next().expect("single page"))) + }) + .await + .unwrap(); + assert_eq!(String::from_utf8(output).unwrap(), "\"name\",\"note\"\n\"a\",\n\"b\",\"\""); + } + + /// 用户反馈:`NOT NULL DEFAULT ''` 的列导出成 CSV、再导入到另一个环境的同名表时报 + /// 「不允许为 null」。用真实 SQLite 库走一遍「导出 → 导入」,确认空字符串仍是空字符串、 + /// NULL 仍是 NULL(而不是双双塌成 NULL)。 + #[tokio::test] + async fn exported_csv_round_trips_empty_strings_and_nulls_through_a_real_import() { + use crate::data::table_import::{ + import_table_file_core, TableImportColumnMapping, TableImportMode, TableImportParseOptions, + TableImportRequest, TableImportSourceFormat, + }; + use std::io::Write; + + const DDL: &str = "CREATE TABLE contacts (id INTEGER, name TEXT NOT NULL DEFAULT '', note TEXT)"; + + let directory = tempfile::tempdir().unwrap(); + let source_path = directory.path().join("source.db"); + let sqlite = crate::db::sqlite::connect_path_create_if_missing(source_path.to_str().unwrap()).await.unwrap(); + crate::db::sqlite::execute_query(&sqlite, DDL).await.unwrap(); + crate::db::sqlite::execute_query( + &sqlite, + "INSERT INTO contacts (id, name, note) VALUES (1, '', NULL), (2, 'Ada', 'x')", + ) + .await + .unwrap(); + + let mut options = export_options(100); + options.table_name = "contacts".to_string(); + options.columns = vec!["id".to_string(), "name".to_string(), "note".to_string()]; + options.null_literal = dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string(); + + let mut exported_bytes = "\u{FEFF}".as_bytes().to_vec(); + let exported = write_table_csv_pages(&mut exported_bytes, DatabaseType::Sqlite, &options, None, |sql, _| { + let sqlite = sqlite.clone(); + async move { crate::db::sqlite::execute_query(&sqlite, &sql).await.map_err(|error| error.to_string()) } + }) + .await + .unwrap(); + assert_eq!(exported, 2); + // 导出的 NULL 与空串必须已经可区分:空串 `""`、NULL 裸写 `\N` + assert_eq!( + String::from_utf8(exported_bytes.clone()).unwrap(), + "\u{FEFF}\"id\",\"name\",\"note\"\n\"1\",\"\",\\N\n\"2\",\"Ada\",\"x\"" + ); + + let csv_path = directory.path().join("contacts.csv"); + let mut csv_file = std::fs::File::create(&csv_path).unwrap(); + csv_file.write_all(&exported_bytes).unwrap(); + csv_file.flush().unwrap(); + drop(csv_file); + + // 目标环境:同名 DDL,`name` 同样是 NOT NULL DEFAULT '' + let target_path = directory.path().join("target.db"); + let target = crate::db::sqlite::connect_path_create_if_missing(target_path.to_str().unwrap()).await.unwrap(); + crate::db::sqlite::execute_query(&target, DDL).await.unwrap(); + + let storage = crate::persistence::test_storage::open(&directory.path().join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + let connection_id = "csv-round-trip"; + let pool_key = format!("{connection_id}:session:import"); + state + .update_connection_pools(|connections| { + connections.insert(pool_key.clone(), crate::connection::PoolKind::Sqlite(target.clone())); + }) + .await; + let config: ConnectionConfig = serde_json::from_value(serde_json::json!({ + "id": connection_id, + "name": "CSV round-trip", + "db_type": "sqlite", + "host": "", + "port": 0, + "username": "", + "password": "", + "database": target_path.to_string_lossy() + })) + .unwrap(); + state.configs.write().await.insert(connection_id.to_string(), config); + + let request = TableImportRequest { + import_id: "csv-round-trip".to_string(), + connection_id: connection_id.to_string(), + database: String::new(), + schema: String::new(), + table: "contacts".to_string(), + file_path: csv_path.to_string_lossy().into_owned(), + source_ref: None, + source_format: Some(TableImportSourceFormat::Csv), + parse_options: TableImportParseOptions::default(), + mappings: ["id", "name", "note"] + .into_iter() + .map(|column| TableImportColumnMapping { + source_column: column.to_string(), + target_column: column.to_string(), + target_data_type: None, + }) + .collect(), + mode: TableImportMode::Append, + create_table: false, + batch_size: 2, + date_time_format: None, + prepared_source: None, + retain_source: false, + conflict_policy: None, + skip_duplicate_rows: false, + }; + let summary = import_table_file_core( + &state, + &request, + &DatabaseType::Sqlite, + &pool_key, + |_| Box::pin(async { false }), + |_| {}, + ) + .await + .expect("NOT NULL DEFAULT '' 的列必须能原样导入空字符串"); + + assert_eq!(summary.rows_imported, 2); + assert_eq!( + crate::db::sqlite::execute_query(&target, "SELECT id, name, note FROM contacts ORDER BY id") + .await + .unwrap() + .rows, + vec![ + vec![serde_json::json!(1), serde_json::json!(""), serde_json::Value::Null], + vec![serde_json::json!(2), serde_json::json!("Ada"), serde_json::json!("x")], + ] + ); + + // 反向验证:把 NULL 字面量关掉、恢复「空字段即 NULL」的旧配置再导一次同一个文件, + // 必须重现用户报的「不允许为 null」,否则这条用例并没有真正覆盖该缺陷。 + let legacy_request = TableImportRequest { + import_id: "csv-round-trip-legacy".to_string(), + parse_options: TableImportParseOptions { + null_literal: Some(String::new()), + ..TableImportParseOptions::default() + }, + ..request + }; + let error = import_table_file_core( + &state, + &legacy_request, + &DatabaseType::Sqlite, + &pool_key, + |_| Box::pin(async { false }), + |_| {}, + ) + .await + .expect_err("旧配置会把空字符串写成 NULL,必须触发 NOT NULL 约束错误"); + assert!(error.to_lowercase().contains("not null"), "unexpected error: {error}"); + } + /// The refusal has to land before `File::create`, otherwise the user is left /// holding a header-only CSV that looks like an object with no records. #[tokio::test] @@ -377,6 +575,7 @@ mod tests { page_size: None, timeout_secs: None, csv_quote_mode: CsvQuoteMode::default(), + null_literal: String::new(), }, ) .await diff --git a/crates/dbx-core/src/data/query_result_export.rs b/crates/dbx-core/src/data/query_result_export.rs index 7928a3c387..d462e3ed94 100644 --- a/crates/dbx-core/src/data/query_result_export.rs +++ b/crates/dbx-core/src/data/query_result_export.rs @@ -11,8 +11,8 @@ use std::time::{Duration, Instant}; use crate::connection::{AppState, PoolKind}; use crate::csv_export::{ - format_query_result_csv_with_quote_mode, format_tsv, push_query_result_csv_row_with_quote_mode, push_tsv_row, - CsvQuoteMode, + csv_null_literal, format_query_result_csv_with_options, format_tsv, push_query_result_csv_row_with_options, + push_tsv_row, CsvQuoteMode, }; pub use crate::database_export::ExportStatus; use crate::database_export::{ @@ -88,6 +88,10 @@ pub struct QueryResultExportRequest { pub insert_mode: SqlInsertMode, #[serde(default)] pub csv_quote_mode: CsvQuoteMode, + /// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量, + /// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。 + #[serde(default = "dbx_formats::csv_export::default_csv_null_literal")] + pub null_literal: String, #[serde(default)] pub include_sql_sheet: bool, pub page_size: usize, @@ -543,9 +547,14 @@ fn effective_row_limit(request: &QueryResultExportRequest) -> Option { request.row_limit } -fn format_text_export_header(format: &str, columns: &[String], csv_quote_mode: CsvQuoteMode) -> String { +fn format_text_export_header( + format: &str, + columns: &[String], + csv_quote_mode: CsvQuoteMode, + null_literal: Option<&str>, +) -> String { let content = if format == "csv" { - format_query_result_csv_with_quote_mode(columns, &[], csv_quote_mode) + format_query_result_csv_with_options(columns, &[], csv_quote_mode, null_literal) } else { format_tsv(columns, &[]) }; @@ -558,13 +567,14 @@ fn write_text_export_row( row: &[Value], buffer: &mut String, csv_quote_mode: CsvQuoteMode, + null_literal: Option<&str>, ) -> Result<(), String> { buffer.clear(); buffer.push('\n'); if format == "csv" { - push_query_result_csv_row_with_quote_mode(buffer, row, csv_quote_mode); + push_query_result_csv_row_with_options(buffer, row, csv_quote_mode, null_literal); } else { - push_tsv_row(buffer, row); + push_tsv_row(buffer, row, null_literal); } file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) } @@ -575,14 +585,15 @@ fn write_text_export_rows( rows: &[Vec], buffer: &mut String, csv_quote_mode: CsvQuoteMode, + null_literal: Option<&str>, ) -> Result<(), String> { buffer.clear(); for row in rows { buffer.push('\n'); if format == "csv" { - push_query_result_csv_row_with_quote_mode(buffer, row, csv_quote_mode); + push_query_result_csv_row_with_options(buffer, row, csv_quote_mode, null_literal); } else { - push_tsv_row(buffer, row); + push_tsv_row(buffer, row, null_literal); } } file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) @@ -1093,7 +1104,12 @@ async fn export_query_result_core_inner( if format == "csv" || format == "txt" { if let Some(file) = text_file.as_mut() { if !wrote_text_header { - let header = format_text_export_header(&format, &columns, request.csv_quote_mode); + let header = format_text_export_header( + &format, + &columns, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; if row_count > 0 { write_text_export_rows( @@ -1102,6 +1118,7 @@ async fn export_query_result_core_inner( formatted_rows.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } wrote_text_header = true; @@ -1112,6 +1129,7 @@ async fn export_query_result_core_inner( formatted_rows.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } } @@ -1184,7 +1202,12 @@ async fn export_query_result_core_inner( if format == "csv" || format == "txt" { if !wrote_text_header { - let header = format_text_export_header(&format, &columns, request.csv_quote_mode); + let header = format_text_export_header( + &format, + &columns, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); if let Some(file) = text_file.as_mut() { file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; } @@ -1301,7 +1324,12 @@ async fn try_export_postgres_query_result_stream( if let Some(writer) = sql_writer.as_mut() { writer.set_columns(columns.clone(), &column_types, &[], request)?; } else if let Some(file) = text_file.as_mut() { - let header = format_text_export_header(format, &columns, request.csv_quote_mode); + let header = format_text_export_header( + format, + &columns, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; } else if json_writer.is_none() { let xlsx_file = @@ -1329,6 +1357,7 @@ async fn try_export_postgres_query_result_stream( formatted.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } else if let Some(writer) = json_writer.as_mut() { writer.write_row(&columns, formatted.as_ref())?; @@ -1547,7 +1576,12 @@ async fn try_export_mysql_query_result_stream( if let Some(writer) = sql_writer.as_mut() { writer.set_columns(columns.clone(), &column_types, &[], request)?; } else if let Some(file) = text_file.as_mut() { - let header = format_text_export_header(format, &columns, request.csv_quote_mode); + let header = format_text_export_header( + format, + &columns, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; } else if json_writer.is_none() { let xlsx_file = @@ -1575,6 +1609,7 @@ async fn try_export_mysql_query_result_stream( formatted.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } else if let Some(writer) = json_writer.as_mut() { writer.write_row(&columns, formatted.as_ref())?; @@ -1772,7 +1807,12 @@ async fn try_export_clickhouse_query_result_stream( if let Some(writer) = sql_writer.as_mut() { writer.set_columns(columns.clone(), &column_types, &[], request)?; } else if let Some(file) = text_file.as_mut() { - let header = format_text_export_header(format, &columns, request.csv_quote_mode); + let header = format_text_export_header( + format, + &columns, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; } else if json_writer.is_none() { let xlsx_file = @@ -1800,6 +1840,7 @@ async fn try_export_clickhouse_query_result_stream( formatted.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } else if let Some(writer) = json_writer.as_mut() { writer.write_row(&columns, formatted.as_ref())?; @@ -1966,7 +2007,12 @@ async fn try_export_sqlserver_query_result_stream( if let Some(writer) = sql_writer.as_mut() { writer.set_columns(columns.clone(), &temporal_column_types, &[], request)?; } else if let Some(file) = text_file.as_mut() { - let header = format_text_export_header(format, &columns, request.csv_quote_mode); + let header = format_text_export_header( + format, + &columns, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; } else if json_writer.is_none() { let xlsx_file = @@ -1990,6 +2036,7 @@ async fn try_export_sqlserver_query_result_stream( formatted.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } else if let Some(writer) = json_writer.as_mut() { writer.write_row(&columns, formatted.as_ref())?; @@ -2191,6 +2238,7 @@ mod tests { identifier_quote: None, insert_mode: SqlInsertMode::Batch, csv_quote_mode: Default::default(), + null_literal: String::new(), exclude_primary_keys: false, primary_keys: Vec::new(), } @@ -2281,6 +2329,7 @@ mod tests { identifier_quote: None, insert_mode: Default::default(), csv_quote_mode: Default::default(), + null_literal: String::new(), exclude_primary_keys: true, primary_keys: vec!["id".to_string()], }; @@ -2357,6 +2406,7 @@ mod tests { execution_id: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -2560,7 +2610,7 @@ mod tests { #[test] fn txt_export_header_keeps_columns_for_empty_results() { assert_eq!( - format_text_export_header("txt", &["id".to_string(), "note".to_string()], CsvQuoteMode::All), + format_text_export_header("txt", &["id".to_string(), "note".to_string()], CsvQuoteMode::All, None), "id\tnote" ); } @@ -2602,7 +2652,7 @@ mod tests { let mut output = Vec::new(); let mut buffer = String::new(); - write_text_export_row(&mut output, "csv", &row, &mut buffer, CsvQuoteMode::All).expect("write csv row"); + write_text_export_row(&mut output, "csv", &row, &mut buffer, CsvQuoteMode::All, None).expect("write csv row"); assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\""); } diff --git a/crates/dbx-core/src/data/table_export.rs b/crates/dbx-core/src/data/table_export.rs index 75b9c12276..67f1b85040 100644 --- a/crates/dbx-core/src/data/table_export.rs +++ b/crates/dbx-core/src/data/table_export.rs @@ -15,8 +15,8 @@ use crate::csv_export::format_csv; #[cfg(test)] use crate::csv_export::format_tsv_rows; use crate::csv_export::{ - format_csv_with_quote_mode, format_tsv, push_table_csv_row, push_table_csv_row_with_quote_mode, push_tsv_row, - CsvQuoteMode, + csv_null_literal, format_csv_with_options, format_tsv, push_table_csv_row, push_table_csv_row_with_options, + push_tsv_row, CsvQuoteMode, }; use crate::data_grid_sql::extra_is_auto_generated; pub use crate::database_export::ExportStatus; @@ -87,6 +87,10 @@ pub struct TableExportRequest { pub insert_dialect: SqlInsertDialect, #[serde(default)] pub csv_quote_mode: CsvQuoteMode, + /// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量, + /// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。 + #[serde(default = "dbx_formats::csv_export::default_csv_null_literal")] + pub null_literal: String, #[serde(default)] pub columns: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -164,13 +168,14 @@ fn write_table_text_row( row: &[Value], buffer: &mut String, csv_quote_mode: CsvQuoteMode, + null_literal: Option<&str>, ) -> Result<(), String> { buffer.clear(); buffer.push('\n'); if csv { - push_table_csv_row_with_quote_mode(buffer, row, csv_quote_mode); + push_table_csv_row_with_options(buffer, row, csv_quote_mode, null_literal); } else { - push_tsv_row(buffer, row); + push_tsv_row(buffer, row, null_literal); } file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) } @@ -181,14 +186,15 @@ fn write_table_text_rows( rows: &[Vec], buffer: &mut String, csv_quote_mode: CsvQuoteMode, + null_literal: Option<&str>, ) -> Result<(), String> { buffer.clear(); for row in rows { buffer.push('\n'); if csv { - push_table_csv_row_with_quote_mode(buffer, row, csv_quote_mode); + push_table_csv_row_with_options(buffer, row, csv_quote_mode, null_literal); } else { - push_tsv_row(buffer, row); + push_tsv_row(buffer, row, null_literal); } } file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) @@ -1128,7 +1134,12 @@ async fn try_export_native_table_stream( std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, ); file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?; - let header = format_csv_with_quote_mode(col_names, &[], request.csv_quote_mode); + let header = format_csv_with_options( + col_names, + &[], + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); let header = header.strip_suffix('\n').unwrap_or(&header); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; let mut row_buffer = String::new(); @@ -1147,7 +1158,14 @@ async fn try_export_native_table_stream( column_types, request.date_time_format.as_deref(), ); - write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer, request.csv_quote_mode)?; + write_table_text_row( + &mut file, + true, + formatted.as_ref(), + &mut row_buffer, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + )?; rows_exported += 1; if rows_exported.is_multiple_of(progress_interval) { on_progress(TableExportProgress { @@ -1197,6 +1215,7 @@ async fn try_export_native_table_stream( formatted.as_ref(), &mut row_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; rows_exported += 1; if rows_exported.is_multiple_of(progress_interval) { @@ -1780,8 +1799,12 @@ async fn export_table_data_core_inner( if is_first_batch { // First batch: write header + rows via format_csv - let csv_content = - format_csv_with_quote_mode(&col_names, formatted_rows.as_ref(), request.csv_quote_mode); + let csv_content = format_csv_with_options( + &col_names, + formatted_rows.as_ref(), + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; is_first_batch = false; } else { @@ -1792,6 +1815,7 @@ async fn export_table_data_core_inner( formatted_rows.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } @@ -1880,6 +1904,7 @@ async fn export_table_data_core_inner( formatted_rows.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; is_first_batch = false; } else { @@ -1889,6 +1914,7 @@ async fn export_table_data_core_inner( formatted_rows.as_ref(), &mut text_buffer, request.csv_quote_mode, + csv_null_literal(&request.null_literal), )?; } @@ -2476,6 +2502,7 @@ mod tests { row_limit, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -2631,6 +2658,7 @@ mod tests { insert_mode: Default::default(), insert_dialect: Default::default(), csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), exclude_primary_keys: false, columns: None, selected_columns: None, @@ -2730,7 +2758,7 @@ mod tests { let mut output = Vec::new(); let mut buffer = String::new(); - write_table_text_row(&mut output, true, &row, &mut buffer, CsvQuoteMode::All).expect("write csv row"); + write_table_text_row(&mut output, true, &row, &mut buffer, CsvQuoteMode::All, None).expect("write csv row"); assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\""); } @@ -2813,6 +2841,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -2876,6 +2905,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -2917,6 +2947,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -2953,6 +2984,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -2997,6 +3029,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -3053,6 +3086,7 @@ mod tests { row_limit: Some(1000), date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -3103,6 +3137,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -3181,6 +3216,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -3241,6 +3277,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, @@ -3303,6 +3340,7 @@ mod tests { row_limit: None, date_time_format: None, csv_quote_mode: CsvQuoteMode::All, + null_literal: String::new(), numeric_column_right_align: false, column_comments: None, auto_filter: None, diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs index 3a6b8c6b79..d19a7883be 100644 --- a/crates/dbx-core/src/data/table_import.rs +++ b/crates/dbx-core/src/data/table_import.rs @@ -208,6 +208,10 @@ pub struct TableImportParseOptions { pub last_data_row: Option, pub trim_values: Option, pub empty_string_as_null: Option, + /// 分隔文本里代表 NULL 的字面量。缺省表示使用与导出端一致的默认字面量(`\N`); + /// 显式配置空串表示关闭字面量,退回「空字段即 NULL」的旧行为。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub null_literal: Option, pub sheet_name: Option, pub sheet_index: Option, pub json_shape: Option, @@ -227,6 +231,7 @@ impl Default for TableImportParseOptions { last_data_row: None, trim_values: Some(false), empty_string_as_null: Some(true), + null_literal: None, sheet_name: None, sheet_index: None, json_shape: Some(TableImportJsonShape::Auto), @@ -651,14 +656,26 @@ fn unique_import_headers(headers: impl IntoIterator) -> Vec, pub row_range: ImportRowRange, } +/// 分隔文本里 NULL 字面量的默认值:与导出端共用 [`dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL`], +/// 因此 DBX 自己导出的 CSV 默认即可无损导入。 +fn effective_delimited_null_literal(raw: Option<&str>) -> Option { + match raw { + None => Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string()), + // 显式空串 = 关闭字面量 + Some(raw) => dbx_formats::csv_export::csv_null_literal(raw).map(str::to_string), + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ImportRowRange { pub title_row: Option, @@ -707,10 +724,15 @@ pub fn effective_delimited_config( } }; + let null_literal = effective_delimited_null_literal(options.null_literal.as_deref()); + Ok(DelimitedParseConfig { delimiter, trim_values: options.trim_values.unwrap_or(false), - empty_string_as_null: options.empty_string_as_null.unwrap_or(true), + // 配了 NULL 字面量时空字段一律是空串:否则字面量刚把 NULL 和空串分开, + // 这里又会把空串重新当成 NULL。 + empty_string_as_null: null_literal.is_none() && options.empty_string_as_null.unwrap_or(true), + null_literal, row_range: effective_import_row_range(options)?, }) } @@ -734,9 +756,13 @@ fn unwrap_csv_force_text(value: &str) -> &str { inner } -pub fn csv_value_with_config(value: &str, config: DelimitedParseConfig) -> serde_json::Value { +pub fn csv_value_with_config(value: &str, config: &DelimitedParseConfig) -> serde_json::Value { let value = if config.trim_values { value.trim() } else { value }; let value = unwrap_csv_force_text(value); + // 字面量优先:命中即为 NULL,空字段则原样保留为空字符串。 + if config.null_literal.as_deref() == Some(value) { + return serde_json::Value::Null; + } if config.empty_string_as_null && value.is_empty() { serde_json::Value::Null } else { @@ -747,10 +773,11 @@ pub fn csv_value_with_config(value: &str, config: DelimitedParseConfig) -> serde pub fn csv_value(value: &str) -> serde_json::Value { csv_value_with_config( value, - DelimitedParseConfig { + &DelimitedParseConfig { delimiter: b',', trim_values: false, empty_string_as_null: true, + null_literal: None, row_range: ImportRowRange { title_row: Some(1), data_start_row: 2, last_data_row: None }, }, ) @@ -1089,7 +1116,7 @@ fn open_delimited_csv_reader_with_progress( pub fn parse_delimited_reader( reader: R, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, preview_limit: usize, ) -> Result { parse_decoded_delimited_reader(reader, config, preview_limit, TableImportTextEncoding::Utf8) @@ -1097,7 +1124,7 @@ pub fn parse_delimited_reader( fn parse_decoded_delimited_reader( reader: R, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, preview_limit: usize, effective_encoding: TableImportTextEncoding, ) -> Result { @@ -1114,7 +1141,12 @@ pub fn parse_delimited_bytes_with_options( ) -> Result { let (encoding, bom_len) = resolve_text_encoding_from_bytes(bytes, options.encoding)?; let reader = StrictTranscodingReader::new(std::io::Cursor::new(&bytes[bom_len..]), encoding)?; - parse_decoded_delimited_reader(reader, effective_delimited_config(source_format, options)?, preview_limit, encoding) + parse_decoded_delimited_reader( + reader, + &effective_delimited_config(source_format, options)?, + preview_limit, + encoding, + ) } pub fn parse_delimited_file_with_options( @@ -1132,7 +1164,7 @@ pub fn parse_delimited_file_with_options( explicit_options.encoding = Some(encoding); let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?; - return parse_csv_reader(reader, config, preview_limit, encoding); + return parse_csv_reader(reader, &config, preview_limit, encoding); } for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] { @@ -1140,7 +1172,7 @@ pub fn parse_delimited_file_with_options( explicit_options.encoding = Some(encoding); let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?; - match parse_csv_reader(reader, config, preview_limit, encoding) { + match parse_csv_reader(reader, &config, preview_limit, encoding) { Ok(parsed) => return Ok(parsed), Err(error) if error.starts_with("Invalid byte sequence for ") => continue, Err(error) => return Err(error), @@ -1150,12 +1182,12 @@ pub fn parse_delimited_file_with_options( } let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?; - parse_csv_reader(reader, config, preview_limit, encoding) + parse_csv_reader(reader, &config, preview_limit, encoding) } fn parse_csv_reader( mut reader: csv::Reader, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, preview_limit: usize, effective_encoding: TableImportTextEncoding, ) -> Result { @@ -1164,7 +1196,7 @@ fn parse_csv_reader( fn parse_csv_reader_bounded( mut reader: csv::Reader, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, preview_limit: usize, effective_encoding: TableImportTextEncoding, ) -> Result { @@ -1173,7 +1205,7 @@ fn parse_csv_reader_bounded( fn parse_csv_reader_inner( reader: &mut csv::Reader, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, preview_limit: usize, effective_encoding: TableImportTextEncoding, count_all_rows: bool, @@ -1236,7 +1268,7 @@ fn parse_delimited_preview_file_with_options( explicit_options.encoding = Some(encoding); let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?; - return parse_csv_reader_bounded(reader, config, preview_limit, encoding); + return parse_csv_reader_bounded(reader, &config, preview_limit, encoding); } for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] { @@ -1244,7 +1276,7 @@ fn parse_delimited_preview_file_with_options( explicit_options.encoding = Some(encoding); let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?; - match parse_csv_reader_bounded(reader, config, preview_limit, encoding) { + match parse_csv_reader_bounded(reader, &config, preview_limit, encoding) { Ok(parsed) => return Ok(parsed), Err(error) if error.starts_with("Invalid byte sequence for ") => continue, Err(error) => return Err(error), @@ -1254,7 +1286,7 @@ fn parse_delimited_preview_file_with_options( } let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?; - parse_csv_reader_bounded(reader, config, preview_limit, encoding) + parse_csv_reader_bounded(reader, &config, preview_limit, encoding) } pub fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result { @@ -6291,7 +6323,7 @@ fn import_cancelled_progress( fn delimited_record_to_row( record: &csv::StringRecord, columns_len: usize, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, ) -> Vec { (0..columns_len) .map(|index| { @@ -6302,7 +6334,7 @@ fn delimited_record_to_row( fn delimited_columns_and_first_record( reader: &mut csv::Reader, - config: DelimitedParseConfig, + config: &DelimitedParseConfig, ) -> Result<(Vec, Option), String> { let mut columns = Vec::new(); for (index, record) in reader.records().enumerate() { @@ -6351,7 +6383,7 @@ fn stream_delimited_rows_to_channel( // Keep CSV parsing off the async executor while the bounded channel prevents unbounded // accumulation when the database consumer is under load. let (mut reader, config, _) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?; - let (columns, first_record) = delimited_columns_and_first_record(&mut reader, config)?; + let (columns, first_record) = delimited_columns_and_first_record(&mut reader, &config)?; sender .blocking_send(Ok(DelimitedStreamMessage::Header(columns.clone()))) .map_err(|_| "Delimited import consumer closed before the stream started".to_string())?; @@ -6374,7 +6406,7 @@ fn stream_delimited_rows_to_channel( if config.row_range.last_data_row.is_some_and(|last| source_row_number > last) { break; } - pending_rows.push(delimited_record_to_row(&record, columns.len(), config)); + pending_rows.push(delimited_record_to_row(&record, columns.len(), &config)); if pending_rows.len() >= batch_size { sender .blocking_send(Ok(DelimitedStreamMessage::Rows { @@ -9372,12 +9404,93 @@ mod tests { parsed.rows[1], vec![ serde_json::Value::String("2".to_string()), - serde_json::Value::Null, + // 默认 NULL 字面量生效后,空字段保留为空字符串,只有字面量才是 NULL + serde_json::Value::String(String::new()), serde_json::Value::String("false".to_string()), ] ); } + #[test] + fn effective_delimited_config_shares_the_export_null_literal_by_default() { + let config = + effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap(); + assert_eq!(config.null_literal.as_deref(), Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL)); + assert!(!config.empty_string_as_null, "NULL 字面量生效时不再把空字段当作 NULL"); + + // 显式空串 = 关闭字面量,退回「空字段即 NULL」的旧行为 + let legacy = + TableImportParseOptions { null_literal: Some(String::new()), ..TableImportParseOptions::default() }; + let config = effective_delimited_config(TableImportSourceFormat::Csv, &legacy).unwrap(); + assert_eq!(config.null_literal, None); + assert!(config.empty_string_as_null); + } + + #[test] + fn csv_null_literal_separates_null_from_an_empty_string() { + // `\N` 是 NULL;`""` 与裸空字段都是空字符串 + let parsed = parse_csv_bytes(b"id,name\n1,\\N\n2,\"\"\n3,\n", 10).unwrap(); + + assert_eq!(parsed.total_rows, 3); + assert_eq!(parsed.rows[0][1], serde_json::Value::Null); + assert_eq!(parsed.rows[1][1], serde_json::json!("")); + assert_eq!(parsed.rows[2][1], serde_json::json!("")); + } + + #[test] + fn tsv_export_with_null_literal_round_trips_through_the_importer() { + // 导出端(push_tsv_row + 默认字面量)写出的 TSV,按 TSV 规则导入: + // NULL 仍是 NULL、空字符串仍是空字符串 + let mut text = String::from("id\tnote\n"); + dbx_formats::csv_export::push_tsv_row( + &mut text, + &[serde_json::json!(1), serde_json::Value::Null], + Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL), + ); + text.push('\n'); + dbx_formats::csv_export::push_tsv_row( + &mut text, + &[serde_json::json!(2), serde_json::json!("")], + Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL), + ); + + let parsed = parse_delimited_bytes_with_options( + text.as_bytes(), + TableImportSourceFormat::Tsv, + &TableImportParseOptions::default(), + 10, + ) + .unwrap(); + assert_eq!(parsed.rows[0][1], serde_json::Value::Null); + assert_eq!(parsed.rows[1][1], serde_json::json!("")); + } + + #[test] + fn csv_null_literal_can_be_replaced_or_disabled() { + let custom = + TableImportParseOptions { null_literal: Some("(null)".to_string()), ..TableImportParseOptions::default() }; + let parsed = + parse_delimited_bytes_with_options(b"id,name\n1,(null)\n2,\n", TableImportSourceFormat::Csv, &custom, 10) + .unwrap(); + assert_eq!(parsed.rows[0][1], serde_json::Value::Null); + assert_eq!(parsed.rows[1][1], serde_json::json!("")); + + let legacy = + TableImportParseOptions { null_literal: Some(String::new()), ..TableImportParseOptions::default() }; + let parsed = + parse_delimited_bytes_with_options(b"id,name\n1,\n2,Ada\n", TableImportSourceFormat::Csv, &legacy, 10) + .unwrap(); + assert_eq!(parsed.rows[0][1], serde_json::Value::Null); + assert_eq!(parsed.rows[1][1], serde_json::json!("Ada")); + } + + #[test] + fn csv_null_literal_is_not_reported_as_a_missing_value() { + // 字面量只按整字段匹配:夹带空格的 `\N ` 不是 NULL,避免误伤真实数据 + let parsed = parse_csv_bytes(b"id,name\n1, \\N \n", 10).unwrap(); + assert_eq!(parsed.rows[0][1], serde_json::json!(" \\N ")); + } + #[test] fn duplicate_csv_headers_map_to_distinct_source_indexes() { let parsed = parse_csv_bytes(b"name,name\nAda,Lovelace\n", 10).unwrap(); @@ -9627,7 +9740,7 @@ mod tests { let config = effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap(); - let preview = parse_csv_reader_bounded(reader, config, 1, TableImportTextEncoding::Utf8).unwrap(); + let preview = parse_csv_reader_bounded(reader, &config, 1, TableImportTextEncoding::Utf8).unwrap(); assert_eq!(preview.columns, vec!["id", "name"]); assert_eq!(preview.rows, vec![vec![serde_json::json!("1"), serde_json::json!("Ada")]]); @@ -9754,7 +9867,9 @@ mod tests { assert_eq!(parsed.columns, vec!["column_1", "column_2"]); assert_eq!(parsed.total_rows, 2); assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("Ada")]); - assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::Value::Null]); + // 默认 NULL 字面量生效时,显式打开的「空字符串作为 NULL」不再接管空字段: + // 否则字面量刚把 NULL 和空串分开,这里又会把空串重新写成 NULL。 + assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::json!("")]); } #[test] diff --git a/crates/dbx-core/tests/live_clickhouse_query_result_export.rs b/crates/dbx-core/tests/live_clickhouse_query_result_export.rs index dd297f4126..6171d53a96 100644 --- a/crates/dbx-core/tests/live_clickhouse_query_result_export.rs +++ b/crates/dbx-core/tests/live_clickhouse_query_result_export.rs @@ -102,6 +102,7 @@ async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_onc execution_id: Some(format!("live-clickhouse-query-export-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, diff --git a/crates/dbx-core/tests/live_mysql57.rs b/crates/dbx-core/tests/live_mysql57.rs index 28096be3da..0bbdbd1213 100644 --- a/crates/dbx-core/tests/live_mysql57.rs +++ b/crates/dbx-core/tests/live_mysql57.rs @@ -477,6 +477,7 @@ async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplic execution_id: Some(format!("live-mysql-query-export-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -572,6 +573,7 @@ async fn live_mysql_csv_temporal_export_round_trip_preserves_dbx_force_text_valu execution_id: Some(format!("live-mysql-issue-8803-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -691,6 +693,7 @@ async fn live_mysql_xlsx_export_can_outlive_query_timeout_while_rows_keep_arrivi execution_id: Some(format!("live-mysql-query-export-timeout-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, diff --git a/crates/dbx-core/tests/live_postgres_query_result_export.rs b/crates/dbx-core/tests/live_postgres_query_result_export.rs index 1f2a84b319..97aa1a1d0f 100644 --- a/crates/dbx-core/tests/live_postgres_query_result_export.rs +++ b/crates/dbx-core/tests/live_postgres_query_result_export.rs @@ -150,6 +150,7 @@ async fn live_postgres_query_result_export_uses_single_streamed_query() { execution_id: Some(format!("live-postgres-query-export-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -238,6 +239,7 @@ async fn live_postgres_query_result_xlsx_preserves_temporal_cell_types() { execution_id: Some(format!("live-postgres-xlsx-temporal-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -315,6 +317,7 @@ async fn live_postgres_numeric_xlsx_ignores_fractional_trailing_zeros() { execution_id: Some(format!("live-postgres-xlsx-numeric-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -397,6 +400,7 @@ async fn live_postgres_truncated_batch_result_export_replays_safe_temp_setup() { execution_id: Some(format!("temp-export-csv-{short_suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -481,6 +485,7 @@ async fn live_postgres_xlsx_export_can_outlive_query_timeout_while_rows_keep_arr execution_id: Some(format!("live-postgres-query-export-timeout-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, @@ -556,6 +561,7 @@ async fn live_postgres_stream_still_times_out_without_progress_and_recovers() { execution_id: Some(format!("live-postgres-query-export-stall-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, diff --git a/crates/dbx-core/tests/live_sqlserver_completion.rs b/crates/dbx-core/tests/live_sqlserver_completion.rs index 2b94dceb43..5586e78744 100644 --- a/crates/dbx-core/tests/live_sqlserver_completion.rs +++ b/crates/dbx-core/tests/live_sqlserver_completion.rs @@ -1215,6 +1215,7 @@ async fn live_sqlserver_query_result_export_streams_cte_query_to_csv() { execution_id: Some(format!("live-sqlserver-export-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, diff --git a/crates/dbx-core/tests/live_sqlserver_query_result_export.rs b/crates/dbx-core/tests/live_sqlserver_query_result_export.rs index 3a072ca915..ef46d374b2 100644 --- a/crates/dbx-core/tests/live_sqlserver_query_result_export.rs +++ b/crates/dbx-core/tests/live_sqlserver_query_result_export.rs @@ -129,6 +129,7 @@ async fn live_sqlserver_xlsx_export_can_outlive_query_timeout_while_rows_keep_ar execution_id: Some(format!("live-sqlserver-xlsx-{suffix}")), date_time_format: None, csv_quote_mode: Default::default(), + null_literal: String::new(), export_table_name: None, export_column_types: None, selected_columns: None, diff --git a/crates/dbx-formats/src/csv_export.rs b/crates/dbx-formats/src/csv_export.rs index f21f52db2c..5c9cb0e5f5 100644 --- a/crates/dbx-formats/src/csv_export.rs +++ b/crates/dbx-formats/src/csv_export.rs @@ -11,6 +11,27 @@ pub enum CsvQuoteMode { Necessary, } +/// CSV 里 NULL 的默认字面量(`\N`,MySQL `SELECT ... INTO OUTFILE` 惯例)。 +/// +/// 默认值必须同时被导出与导入两侧引用:导出写出该字面量、导入按该字面量还原 NULL, +/// 空字符串则始终写成 `""`,这样「空字符串」与「NULL」在 CSV 中不再互相冒充。 +pub const DEFAULT_CSV_NULL_LITERAL: &str = "\\N"; + +/// 把导出/导入配置里的原始字面量解析成实际生效的值:空串表示「不写 NULL 标记」, +/// 即沿用历史行为(NULL 写成空字段,无法与空字符串区分)。 +pub fn csv_null_literal(raw: &str) -> Option<&str> { + if raw.is_empty() { + None + } else { + Some(raw) + } +} + +/// 导出/导入配置里 NULL 字面量的 serde 默认值:未显式配置时用 [`DEFAULT_CSV_NULL_LITERAL`]。 +pub fn default_csv_null_literal() -> String { + DEFAULT_CSV_NULL_LITERAL.to_string() +} + /// CSV 转义直写目标 buffer:包引号 + 内部 `"` 翻倍。值不含 `"` 时整段拷贝, /// 不做 replace 分配(逐批流式导出对每个单元格调用,是导出热路径)。 fn push_csv_escaped_content(out: &mut String, value: &str) { @@ -67,7 +88,26 @@ pub fn push_csv_text_value(out: &mut String, value: &Value) { out.push('"'); } -fn push_csv_value_with_quote_mode(out: &mut String, value: &Value, quote_mode: CsvQuoteMode, quote_null: bool) { +fn push_csv_value_with_quote_mode( + out: &mut String, + value: &Value, + quote_mode: CsvQuoteMode, + quote_null: bool, + null_literal: Option<&str>, +) { + if value.is_null() { + // 配置了 NULL 字面量时,NULL 写成该字面量;空字符串仍然写成 `""`。 + // 两者不再互相冒充,导出→导入才能无损往返。 + // + // 字面量必须尽量裸写:PostgreSQL 的 COPY ... FORMAT csv 从不把带引号的值当成 NULL + // (`"\N"` 是普通字符串,整数列直接报错),ClickHouse 的 CSV 读取更是在引号内遇到 + // `\N` 就解析失败;只有裸 `\N` 才是这两者与 MySQL 共同识别的 NULL。字面量自身含 + // 分隔符/引号/换行时才加引号(此时它本来也不再是标准 NULL 标记)。 + if let Some(literal) = null_literal { + push_csv_field(out, literal, CsvQuoteMode::Necessary); + return; + } + } if quote_mode == CsvQuoteMode::All { if value.is_null() && !quote_null { return; @@ -87,13 +127,6 @@ fn push_csv_value_with_quote_mode(out: &mut String, value: &Value, quote_mode: C } } -fn push_csv_value(out: &mut String, value: &Value) { - if value.is_null() { - return; - } - push_csv_text_value(out, value); -} - /// TSV 转义直写:仅含特殊字符时包引号(语义与原 escape_tsv 一致)。 fn push_tsv_escaped(out: &mut String, value: &str) { if value.contains('\t') || value.contains('\n') || value.contains('\r') || value.contains('"') { @@ -103,7 +136,14 @@ fn push_tsv_escaped(out: &mut String, value: &str) { } } -fn push_tsv_value(out: &mut String, value: &Value) { +fn push_tsv_value(out: &mut String, value: &Value, null_literal: Option<&str>) { + if value.is_null() { + // 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返。 + if let Some(literal) = null_literal { + push_tsv_escaped(out, literal); + return; + } + } match value { Value::Null => {} Value::String(v) => push_tsv_escaped(out, v), @@ -115,12 +155,12 @@ fn push_tsv_value(out: &mut String, value: &Value) { } } -pub fn push_tsv_row(out: &mut String, row: &[Value]) { +pub fn push_tsv_row(out: &mut String, row: &[Value], null_literal: Option<&str>) { for (cell_index, cell) in row.iter().enumerate() { if cell_index > 0 { out.push('\t'); } - push_tsv_value(out, cell); + push_tsv_value(out, cell, null_literal); } } @@ -146,12 +186,6 @@ fn escape_csv_with_quote_mode(value: &str, quote_mode: CsvQuoteMode) -> String { out } -fn format_csv_value(value: &Value) -> String { - let mut out = String::new(); - push_csv_value(&mut out, value); - out -} - #[cfg_attr(not(test), allow(dead_code))] pub fn escape_tsv(value: &str) -> String { let mut out = String::with_capacity(value.len()); @@ -159,19 +193,19 @@ pub fn escape_tsv(value: &str) -> String { out } -fn push_tsv_rows(out: &mut String, rows: &[Vec]) { +fn push_tsv_rows(out: &mut String, rows: &[Vec], null_literal: Option<&str>) { for (row_index, row) in rows.iter().enumerate() { if row_index > 0 { out.push('\n'); } - push_tsv_row(out, row); + push_tsv_row(out, row, null_literal); } } #[cfg_attr(not(test), allow(dead_code))] pub fn format_tsv_rows(rows: &[Vec]) -> String { let mut out = String::with_capacity(estimated_rows_capacity(rows)); - push_tsv_rows(&mut out, rows); + push_tsv_rows(&mut out, rows, None); out } @@ -186,7 +220,7 @@ pub fn format_tsv(columns: &[String], rows: &[Vec]) -> String { push_tsv_escaped(&mut out, column); } out.push('\n'); - push_tsv_rows(&mut out, rows); + push_tsv_rows(&mut out, rows, None); out } @@ -198,11 +232,21 @@ pub fn push_query_result_csv_row(out: &mut String, row: &[Value]) { } pub fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) { + push_query_result_csv_row_with_options(out, row, quote_mode, None); +} + +/// 与 [`push_query_result_csv_row_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。 +pub fn push_query_result_csv_row_with_options( + out: &mut String, + row: &[Value], + quote_mode: CsvQuoteMode, + null_literal: Option<&str>, +) { for (cell_index, cell) in row.iter().enumerate() { if cell_index > 0 { out.push(','); } - push_csv_value_with_quote_mode(out, cell, quote_mode, false); + push_csv_value_with_quote_mode(out, cell, quote_mode, false, null_literal); } } @@ -211,11 +255,21 @@ pub fn push_table_csv_row(out: &mut String, row: &[Value]) { } pub fn push_table_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) { + push_table_csv_row_with_options(out, row, quote_mode, None); +} + +/// 与 [`push_table_csv_row_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。 +pub fn push_table_csv_row_with_options( + out: &mut String, + row: &[Value], + quote_mode: CsvQuoteMode, + null_literal: Option<&str>, +) { for (cell_index, cell) in row.iter().enumerate() { if cell_index > 0 { out.push(','); } - push_csv_value_with_quote_mode(out, cell, quote_mode, true); + push_csv_value_with_quote_mode(out, cell, quote_mode, true, null_literal); } } @@ -254,6 +308,19 @@ pub fn format_csv(columns: &[String], rows: &[Vec]) -> String { } pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec], quote_mode: CsvQuoteMode) -> String { + format_csv_with_options(columns, rows, quote_mode, None) +} + +/// 与 [`format_csv_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。 +/// +/// 表数据导出的首批行走这里,后续分批走 [`push_table_csv_row_with_options`]: +/// 两处必须收到同一个字面量,否则同一个文件里 NULL 会前后不一致。 +pub fn format_csv_with_options( + columns: &[String], + rows: &[Vec], + quote_mode: CsvQuoteMode, + null_literal: Option<&str>, +) -> String { let mut out = String::with_capacity( estimated_rows_capacity(rows).saturating_add(columns.len().saturating_mul(12)).min(ROWS_CAPACITY_ESTIMATE_MAX), ); @@ -268,7 +335,7 @@ pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec], quote if row_index > 0 { out.push('\n'); } - push_query_result_csv_row_with_quote_mode(&mut out, row, quote_mode); + push_query_result_csv_row_with_options(&mut out, row, quote_mode, null_literal); } out } @@ -282,7 +349,17 @@ pub fn format_query_result_csv_with_quote_mode( rows: &[Vec], quote_mode: CsvQuoteMode, ) -> String { - format_csv_with_quote_mode(columns, rows, quote_mode) + format_csv_with_options(columns, rows, quote_mode, None) +} + +/// 与 [`format_query_result_csv_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。 +pub fn format_query_result_csv_with_options( + columns: &[String], + rows: &[Vec], + quote_mode: CsvQuoteMode, + null_literal: Option<&str>, +) -> String { + format_csv_with_options(columns, rows, quote_mode, null_literal) } pub fn write_csv_text_row( @@ -305,20 +382,26 @@ pub fn write_csv_value_row( writer: &mut impl Write, values: impl IntoIterator, quote_mode: CsvQuoteMode, +) -> Result<(), String> { + write_csv_value_row_with_options(writer, values, quote_mode, None) +} + +/// 与 [`write_csv_value_row`] 相同,但允许把 NULL 写成指定字面量。 +pub fn write_csv_value_row_with_options( + writer: &mut impl Write, + values: impl IntoIterator, + quote_mode: CsvQuoteMode, + null_literal: Option<&str>, ) -> Result<(), String> { let mut first = true; + let mut formatted = String::new(); for value in values { if !first { writer.write_all(b",").map_err(|err| err.to_string())?; } first = false; - let formatted = if quote_mode == CsvQuoteMode::All { - format_csv_value(&value) - } else { - let mut formatted = String::new(); - push_csv_value_with_quote_mode(&mut formatted, &value, quote_mode, false); - formatted - }; + formatted.clear(); + push_csv_value_with_quote_mode(&mut formatted, &value, quote_mode, false, null_literal); writer.write_all(formatted.as_bytes()).map_err(|err| err.to_string())?; } Ok(()) @@ -327,8 +410,8 @@ pub fn write_csv_value_row( #[cfg(test)] mod tests { use super::{ - format_csv, format_csv_with_quote_mode, format_query_result_csv, format_query_result_csv_rows, format_tsv, - CsvQuoteMode, + format_csv, format_csv_with_options, format_csv_with_quote_mode, format_query_result_csv, + format_query_result_csv_rows, format_tsv, CsvQuoteMode, DEFAULT_CSV_NULL_LITERAL, }; use serde_json::json; @@ -434,6 +517,104 @@ mod tests { } } + #[test] + fn default_null_literal_is_the_mysql_outfile_convention() { + assert_eq!(DEFAULT_CSV_NULL_LITERAL, "\\N"); + // 空串表示「不写 NULL 标记」,即沿用历史行为 + assert_eq!(super::csv_null_literal(""), None); + assert_eq!(super::csv_null_literal(DEFAULT_CSV_NULL_LITERAL), Some("\\N")); + } + + #[test] + fn tsv_null_literal_separates_null_from_an_empty_string() { + let row = vec![Value::Null, json!("")]; + + // 未配置字面量(旧行为):NULL 与空串都写空字段,二者在文件里无法区分 + let mut out = String::new(); + super::push_tsv_row(&mut out, &row, None); + assert_eq!(out, "\t"); + + // 配置字面量后:NULL 写成字面量,空字符串仍是空字段 + let mut out = String::new(); + super::push_tsv_row(&mut out, &row, Some(DEFAULT_CSV_NULL_LITERAL)); + assert_eq!(out, "\\N\t"); + } + + #[test] + fn null_literal_stops_null_from_impersonating_an_empty_string() { + let columns = vec!["a".to_string(), "b".to_string()]; + let row = vec![Value::Null, json!("")]; + + // 未配置字面量(旧行为):NULL 写成裸空字段,空串写成 `""`, + // 文件里看似可区分,但导入端拿不到引号信息,两者最终都会变成 NULL + assert_eq!( + format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, None), + "\"a\",\"b\"\n,\"\"" + ); + + // 配置字面量后:NULL 写成裸字面量,空串写成 `""`;即使 quote mode = All 也不给 + // 字面量加引号,否则 PostgreSQL/ClickHouse 的 CSV 读取不认它是 NULL + assert_eq!( + format_csv_with_options( + &columns, + std::slice::from_ref(&row), + CsvQuoteMode::All, + Some(DEFAULT_CSV_NULL_LITERAL) + ), + "\"a\",\"b\"\n\\N,\"\"" + ); + // Necessary 模式:空串裸写为空字段,但 NULL 有独立字面量,仍然可区分 + assert_eq!( + format_csv_with_options( + &columns, + std::slice::from_ref(&row), + CsvQuoteMode::Necessary, + Some(DEFAULT_CSV_NULL_LITERAL) + ), + "a,b\n\\N," + ); + } + + #[test] + fn null_literal_is_quoted_only_when_it_needs_quoting() { + let columns = vec!["a".to_string()]; + let row = vec![Value::Null]; + + // 标准字面量裸写(PostgreSQL/ClickHouse/MySQL 共同识别的 NULL 写法) + assert_eq!( + format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, Some("\\N")), + "\"a\"\n\\N" + ); + // 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段 + assert_eq!( + format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, Some("a,b")), + "\"a\"\n\"a,b\"" + ); + } + + #[test] + fn null_literal_applies_to_streaming_row_writers_too() { + let row = vec![Value::Null, json!("NULL"), json!("")]; + + let mut table = String::new(); + super::push_table_csv_row_with_options(&mut table, &row, CsvQuoteMode::All, Some(DEFAULT_CSV_NULL_LITERAL)); + assert_eq!(table, "\\N,\"NULL\",\"\""); + + let mut query = String::new(); + super::push_query_result_csv_row_with_options( + &mut query, + &row, + CsvQuoteMode::All, + Some(DEFAULT_CSV_NULL_LITERAL), + ); + assert_eq!(query, "\\N,\"NULL\",\"\""); + + let mut written = Vec::new(); + super::write_csv_value_row_with_options(&mut written, row, CsvQuoteMode::All, Some(DEFAULT_CSV_NULL_LITERAL)) + .unwrap(); + assert_eq!(String::from_utf8(written).unwrap(), "\\N,\"NULL\",\"\""); + } + #[test] fn reusable_row_buffers_match_batch_formatters() { let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)]; @@ -447,7 +628,7 @@ mod tests { assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\""); let mut tsv = String::new(); - super::push_tsv_row(&mut tsv, &row); + super::push_tsv_row(&mut tsv, &row, None); assert_eq!(tsv, super::format_tsv_rows(&[row])); } diff --git a/docs/screenshots/issue-10505-destination-rows.png b/docs/screenshots/issue-10505-destination-rows.png new file mode 100644 index 0000000000..6489b8c941 Binary files /dev/null and b/docs/screenshots/issue-10505-destination-rows.png differ diff --git a/docs/screenshots/issue-10505-export-complete.png b/docs/screenshots/issue-10505-export-complete.png new file mode 100644 index 0000000000..fcb3248a01 Binary files /dev/null and b/docs/screenshots/issue-10505-export-complete.png differ diff --git a/docs/screenshots/issue-10505-import-default-option.png b/docs/screenshots/issue-10505-import-default-option.png new file mode 100644 index 0000000000..cc7ce3b096 Binary files /dev/null and b/docs/screenshots/issue-10505-import-default-option.png differ diff --git a/docs/screenshots/issue-10505-import-done.png b/docs/screenshots/issue-10505-import-done.png new file mode 100644 index 0000000000..47e681ce7e Binary files /dev/null and b/docs/screenshots/issue-10505-import-done.png differ diff --git a/docs/screenshots/issue-10505-legacy-option-fails.png b/docs/screenshots/issue-10505-legacy-option-fails.png new file mode 100644 index 0000000000..ae3b37f1a0 Binary files /dev/null and b/docs/screenshots/issue-10505-legacy-option-fails.png differ diff --git a/docs/screenshots/issue-10505-null-mode-setting.png b/docs/screenshots/issue-10505-null-mode-setting.png new file mode 100644 index 0000000000..20dcbb0c62 Binary files /dev/null and b/docs/screenshots/issue-10505-null-mode-setting.png differ diff --git a/packages/app-tests/exportFormats.test.ts b/packages/app-tests/exportFormats.test.ts index 3339e5e3be..4e18290baa 100644 --- a/packages/app-tests/exportFormats.test.ts +++ b/packages/app-tests/exportFormats.test.ts @@ -2,7 +2,9 @@ import { strict as assert } from "node:assert"; import { test } from "vitest"; import { formatCsv } from "../../apps/desktop/src/lib/export/exportFormats.ts"; -test("formatCsv writes database null as an empty cell and preserves literal NULL", () => { +// NULL 必须写成独立字面量、空字符串才写成空字段,否则导出再导入时 '' 会被还原成 +// NULL,写回 NOT NULL DEFAULT '' 的列就会报「不允许为 null」。 +test("formatCsv writes database null as the shared NULL literal and keeps empty strings empty", () => { assert.equal( formatCsv( ["id", "note"], @@ -12,7 +14,22 @@ test("formatCsv writes database null as an empty cell and preserves literal NULL [3, "NULL"], ], ), - '"id","note"\n"1",\n"2",""\n"3","NULL"', + '"id","note"\n"1",\\N\n"2",""\n"3","NULL"', + ); +}); + +test("formatCsv keeps the legacy empty-cell NULL when the literal is explicitly disabled", () => { + assert.equal( + formatCsv( + ["id", "note"], + [ + [1, null], + [2, ""], + ], + "all", + "", + ), + '"id","note"\n"1",\n"2",""', ); }); @@ -23,9 +40,10 @@ test("formatCsv only quotes fields with CSV special characters in necessary mode [ [2085252644, "延庆县", "plain"], [2085252645, "门头沟区", 'line 1\n"line 2"'], + [2085252646, "昌平区", null], ], "necessary", ), - 'id,"district,name",note\n2085252644,延庆县,plain\n2085252645,门头沟区,"line 1\n""line 2"""', + 'id,"district,name",note\n2085252644,延庆县,plain\n2085252645,门头沟区,"line 1\n""line 2"""\n2085252646,昌平区,\\N', ); }); diff --git a/src-tauri/src/commands/csv_export.rs b/src-tauri/src/commands/csv_export.rs index 01bf8af608..d8598ceb2e 100644 --- a/src-tauri/src/commands/csv_export.rs +++ b/src-tauri/src/commands/csv_export.rs @@ -2,7 +2,8 @@ use std::sync::Arc; use crate::commands::connection::AppState; use dbx_core::csv_export::{ - export_table_data_csv_core, format_query_result_csv_with_quote_mode, CsvQuoteMode, TableCsvExportOptions, + csv_null_literal, default_csv_null_literal, export_table_data_csv_core, format_query_result_csv_with_options, + CsvQuoteMode, TableCsvExportOptions, }; use serde::Deserialize; use serde_json::Value; @@ -16,12 +17,21 @@ pub struct QueryResultCsvExportRequest { pub rows: Vec>, #[serde(default)] pub csv_quote_mode: CsvQuoteMode, + /// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量, + /// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。 + #[serde(default = "default_csv_null_literal")] + pub null_literal: String, } #[tauri::command] pub async fn export_query_result_csv(request: QueryResultCsvExportRequest) -> Result<(), String> { tauri::async_runtime::spawn_blocking(move || { - let csv = format_query_result_csv_with_quote_mode(&request.columns, &request.rows, request.csv_quote_mode); + let csv = format_query_result_csv_with_options( + &request.columns, + &request.rows, + request.csv_quote_mode, + csv_null_literal(&request.null_literal), + ); std::fs::write(&request.file_path, format!("\u{FEFF}{csv}")).map_err(|err| err.to_string()) }) .await