fix(export): mark NULL as \N in CSV and TSV exports

This commit is contained in:
zipg authored and GitHub committed 2026-09-28 22:35:43 +08:00
1 parent 02f27dc294
commit 1aaa5ca717
49 files changed
+1056 -130

No files matched your search

+27 -5
View File
@@ -15,12 +15,34 @@ import {
} from "@/lib/table/tableImport";
describe("tableImport", () => {
it("preserves explicit empty strings by default for Excel only", () => {
it("keeps empty fields as empty strings for every source format by default", () => {
// 导出端默认用 `\N` 表示 NULL,空字段代表空字符串;导入端默认必须与之一致,
// 否则 DBX 导出的 CSV 再导回时会把 '' 变成 NULL。
expect(defaultTableImportEmptyStringAsNull("excel")).toBe(false);
expect(defaultTableImportEmptyStringAsNull("csv")).toBe(true);
expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(true);
expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(true);
expect(defaultTableImportEmptyStringAsNull("json")).toBe(true);
expect(defaultTableImportEmptyStringAsNull("csv")).toBe(false);
expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(false);
expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(false);
expect(defaultTableImportEmptyStringAsNull("json")).toBe(false);
});
it("only disables the shared NULL literal when empty fields are treated as NULL", () => {
const baseSettings = {
delimiter: ",",
textEncoding: "auto" as const,
titleRow: 1,
dataStartRow: 2,
lastDataRow: 0,
trimValues: false,
jsonShape: "auto" as const,
};
const sharedLiteral = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: false });
expect(sharedLiteral.nullLiteral).toBeUndefined();
expect(sharedLiteral.emptyStringAsNull).toBe(false);
const legacyEmptyField = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: true });
expect(legacyEmptyField.nullLiteral).toBe("");
expect(legacyEmptyField.emptyStringAsNull).toBe(true);
});
it("formats import elapsed time for progress and terminal summaries", () => {
+2 -2
View File
@@ -3388,9 +3388,9 @@ export async function cancelQueryResultExport(exportId: string, executionId?: st
});
}
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise<void> {
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise<void> {
const { formatCsv } = await import("@/lib/export/exportFormats");
const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode);
const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode, nullLiteral);
const fileName = filePath.split(/[\\/]/).pop() || "export.csv";
const blob = new Blob(["\uFEFF", content], {
type: "text/csv;charset=utf-8",
+10 -1
View File
@@ -5466,6 +5466,8 @@ export interface TableImportParseOptions {
lastDataRow?: number | null;
trimValues?: boolean | null;
emptyStringAsNull?: boolean | null;
/** 分隔文本里代表 NULL 的字面量。缺省表示用后端默认值 `\N`;空串表示关闭字面量,退回「空字段即 NULL」。 */
nullLiteral?: string | null;
sheetName?: string | null;
sheetIndex?: number | null;
jsonShape?: TableImportJsonShape | null;
@@ -5861,6 +5863,8 @@ export interface TableExportRequest {
insertMode?: SqlInsertMode;
insertDialect?: SqlInsertDialect;
csvQuoteMode?: CsvQuoteMode;
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
nullLiteral?: string;
columns?: string[];
selectedColumns?: SqlExportColumnSelection[];
columnTypes?: Array<string | null | undefined>;
@@ -5891,6 +5895,8 @@ export interface TableCsvExportOptions {
pageSize?: number;
timeoutSecs?: number;
csvQuoteMode?: CsvQuoteMode;
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
nullLiteral?: string;
}
export interface TableExportProgress {
@@ -5917,6 +5923,8 @@ export interface QueryResultExportRequest {
format: "csv" | "xlsx" | "json" | "txt" | "sql";
insertMode?: SqlInsertMode;
csvQuoteMode?: CsvQuoteMode;
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
nullLiteral?: string;
includeSqlSheet?: boolean;
pageSize: number;
rowLimit?: number | null;
@@ -6074,13 +6082,14 @@ export async function recordDatabaseExportDestination(directory: string): Promis
await invoke("record_database_export_destination", { directory });
}
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise<void> {
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise<void> {
return invoke("export_query_result_csv", {
request: {
filePath,
columns,
rows,
csvQuoteMode,
...(nullLiteral === undefined ? {} : { nullLiteral }),
},
});
}
@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { csvNullLiteralForMode, DEFAULT_CSV_NULL_LITERAL, DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode } from "@/lib/export/csvNullMode";
describe("normalizeCsvNullMode", () => {
it("defaults to writing the NULL marker", () => {
expect(DEFAULT_CSV_NULL_MODE).toBe("marker");
expect(normalizeCsvNullMode(undefined)).toBe("marker");
expect(normalizeCsvNullMode(null)).toBe("marker");
expect(normalizeCsvNullMode("marker")).toBe("marker");
});
it("keeps an explicit request for the legacy empty field", () => {
expect(normalizeCsvNullMode("empty")).toBe("empty");
});
it("falls back to the default for unknown values", () => {
expect(normalizeCsvNullMode("")).toBe("marker");
expect(normalizeCsvNullMode("\\N")).toBe("marker");
expect(normalizeCsvNullMode(0)).toBe("marker");
});
});
describe("csvNullLiteralForMode", () => {
it("writes the MySQL-style NULL marker by default", () => {
expect(csvNullLiteralForMode("marker")).toBe(DEFAULT_CSV_NULL_LITERAL);
expect(csvNullLiteralForMode("marker")).toBe("\\N");
});
it("disables the literal with an empty string to restore the old export", () => {
expect(csvNullLiteralForMode("empty")).toBe("");
});
});
+68 -3
View File
@@ -1,5 +1,58 @@
import { describe, expect, it } from "vitest";
import { formatTsv } from "@/lib/export/exportFormats";
import { formatCsv, formatTsv } from "@/lib/export/exportFormats";
describe("formatCsv NULL handling", () => {
it("writes NULL as an unquoted marker and an empty string as an empty quoted field", () => {
// 即使 quote mode = all,NULL 字面量也必须裸写:PostgreSQL/ClickHouse 的 CSV 读取
// 不把带引号的 `"\N"` 当作 NULL,裸 `\N` 才是它们和 MySQL 共同识别的写法。
expect(
formatCsv(
["id", "name"],
[
[1, ""],
[2, null],
],
"all",
"\\N",
),
).toBe('"id","name"\n"1",""\n"2",\\N');
});
it("keeps the marker unquoted when only necessary fields are quoted", () => {
expect(
formatCsv(
["id", "name"],
[
[1, ""],
[2, null],
],
"necessary",
"\\N",
),
).toBe("id,name\n1,\n2,\\N");
});
it("quotes a marker that itself needs quoting", () => {
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
expect(formatCsv(["id", "name"], [[1, null]], "all", "a,b")).toBe('"id","name"\n"1","a,b"');
});
it("restores the legacy empty field when the marker is disabled", () => {
// 关闭字面量时 NULL 是完全空字段、空字符串是 `""`:CSV 引号语义上仍有区别,
// 但 DBX 的导入端比较的是去引号后的值,因此旧格式在往返时依赖不了这个区别。
expect(
formatCsv(
["id", "name"],
[
[1, ""],
[2, null],
],
"all",
"",
),
).toBe('"id","name"\n"1",""\n"2",');
});
});
describe("formatTsv", () => {
it("copies a value that only contains double quotes verbatim (#10087)", () => {
@@ -14,8 +67,20 @@ describe("formatTsv", () => {
expect(output).toBe('v\n"a\tb"\n"c\nd"');
});
it("renders NULL as an empty field without adding quotes", () => {
const output = formatTsv(["id", "name"], [[1, null]]);
it("renders NULL as the null literal and an empty string as an empty field", () => {
const output = formatTsv(
["id", "name"],
[
[1, null],
[2, ""],
],
);
expect(output).toBe("id\tname\n1\t\\N\n2\t");
});
it("falls back to an empty NULL field when the literal is disabled", () => {
const output = formatTsv(["id", "name"], [[1, null]], "");
expect(output).toBe("id\tname\n1\t");
});
@@ -0,0 +1,23 @@
/**
* 导出 CSV 时 NULL 的写法。与 Rust `dbx_formats::csv_export` 保持一致:
* `marker` 把 NULL 写成 `\N`(MySQL `SELECT ... INTO OUTFILE`、ClickHouse
* `format_csv_null_representation` 的既定写法),空字符串才写成空字段,两者在文件里可区分,
* 导回 NOT NULL DEFAULT '' 的列时不会被还原成 NULL;`empty` 是旧行为:NULL 与空字符串都写成空字段。
*/
export const DEFAULT_CSV_NULL_LITERAL = "\\N";
export const CSV_NULL_MODE_MARKER = "marker";
export const CSV_NULL_MODE_EMPTY = "empty";
export type CsvNullMode = typeof CSV_NULL_MODE_MARKER | typeof CSV_NULL_MODE_EMPTY;
export const DEFAULT_CSV_NULL_MODE: CsvNullMode = CSV_NULL_MODE_MARKER;
export function normalizeCsvNullMode(value: unknown): CsvNullMode {
return value === CSV_NULL_MODE_EMPTY ? CSV_NULL_MODE_EMPTY : DEFAULT_CSV_NULL_MODE;
}
/** 导出请求里的 NULL 字面量;空串表示关闭该字面量(旧行为)。 */
export function csvNullLiteralForMode(mode: CsvNullMode): string {
return mode === CSV_NULL_MODE_EMPTY ? "" : DEFAULT_CSV_NULL_LITERAL;
}
+19 -10
View File
@@ -1,26 +1,35 @@
import type { DatabaseType, QueryResult } from "@/types/database";
import * as api from "@/lib/backend/api";
import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
import { DEFAULT_CSV_NULL_LITERAL } from "@/lib/export/csvNullMode";
import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
export type ExportCellValue = string | number | boolean | null;
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all"): string {
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all", nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(",");
const body = rows.map((row) => row.map((cell) => (cell === null ? "" : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
// NULL 写成独立字面量、空字符串写成空字段,否则导入端无法把两者区分开
// (写回 NOT NULL DEFAULT '' 的列时会把空字符串还原成 NULL)。
//
// 字面量裸写:PostgreSQL 的 COPY ... FORMAT csv 不认带引号的 NULL(`"\N"` 是普通字符串,
// 整数列直接报错),ClickHouse 的 CSV 读取在引号内遇到 `\N` 会解析失败;只有裸 `\N` 是
// 它们与 MySQL 共同识别的 NULL。字面量自身含分隔符/引号/换行时才加引号。
const nullField = nullLiteral ? escapeCsvField(nullLiteral, "necessary") : "";
const body = rows.map((row) => row.map((cell) => (cell === null ? nullField : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
return `${header}\n${body}`;
}
// Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)"
// export: fields are joined by a tab and NULL becomes empty. A field is wrapped
// in double quotes only when it contains a tab or a line break - the characters
// that would otherwise corrupt the TSV row/column shape. A field that merely
// contains a double quote is emitted verbatim: TSV is parsed by splitting on the
// tab alone (there is no quote state machine on paste-back), so quoting such a
// value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
export function formatTsv(columns: string[], rows: ExportCellValue[][]): string {
// export: fields are joined by a tab and NULL becomes the null literal. A field
// is wrapped in double quotes only when it contains a tab or a line break - the
// characters that would otherwise corrupt the TSV row/column shape. A field that
// merely contains a double quote is emitted verbatim: TSV is parsed by splitting
// on the tab alone (there is no quote state machine on paste-back), so quoting
// such a value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
export function formatTsv(columns: string[], rows: ExportCellValue[][], nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
const esc = (value: ExportCellValue) => {
const text = value === null ? "" : String(value);
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返
const text = value === null ? nullLiteral || "" : String(value);
if (text.includes("\t") || text.includes("\n") || text.includes("\r")) {
return `"${text.replace(/"/g, '""')}"`;
}
@@ -236,6 +236,11 @@ describe("editorSettingsDraftFromSettings", () => {
expect(draft.completionTriggerMode).toBe("positional");
});
it("maps csvNullMode and normalizes an unknown value back to the marker", () => {
expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "empty" })).csvNullMode).toBe("empty");
expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "bogus" as unknown } as Partial<EditorSettings>)).csvNullMode).toBe("marker");
});
it("maps and normalizes tableCompletionSchemaQualification", () => {
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "always" })).tableCompletionSchemaQualification).toBe("always");
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "invalid" } as any)).tableCompletionSchemaQualification).toBe("collision");
@@ -413,4 +413,9 @@ describe("settingsTransfer", () => {
expect(transferCategoryForKey("csvQuoteMode")).toBe("data");
expect(collectTransferCategories(["csvQuoteMode", "pageSize"])).toEqual(["data"]);
});
it("maps csvNullMode into the data category", () => {
expect(transferCategoryForKey("csvNullMode")).toBe("data");
expect(collectTransferCategories(["csvNullMode", "csvQuoteMode"])).toEqual(["data"]);
});
});
@@ -7,6 +7,7 @@ import { normalizeCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPo
import { normalizeSqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification";
import { normalizeTableHoverLookupMode } from "@/lib/editor/hoverTableLookup";
import { normalizeRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates";
import { normalizeCsvNullMode } from "@/lib/export/csvNullMode";
export const EDITOR_SETTINGS_DRAFT_KEYS = [
"fontFamily",
@@ -120,6 +121,7 @@ export const EDITOR_SETTINGS_DRAFT_KEYS = [
"redisDatabaseDisplayLimit",
"exportBatchSize",
"csvQuoteMode",
"csvNullMode",
"exportRowLimitEnabled",
"exportRowLimit",
"queryExportKeysetOptimizationEnabled",
@@ -164,6 +166,7 @@ function normalizedDraftValue(key: EditorSettingsDraftKey, value: unknown): unkn
if (key === "tableCompletionSchemaQualification") return normalizeSqlTableCompletionSchemaQualification(value);
if (key === "tableHoverLookupMode") return normalizeTableHoverLookupMode(value);
if (key === "redisKeyTemplates") return normalizeRedisKeyTemplates(value);
if (key === "csvNullMode") return normalizeCsvNullMode(value);
if (key === "backgroundImage") return normalizeBackgroundImageSettings(value);
return value;
}
@@ -178,6 +178,7 @@ const SETTINGS_TRANSFER_CATEGORY_KEYS: Record<SettingsTransferCategoryId, readon
"redisDatabaseDisplayLimit",
"exportBatchSize",
"csvQuoteMode",
"csvNullMode",
"exportRowLimitEnabled",
"exportRowLimit",
"queryExportKeysetOptimizationEnabled",
+12 -2
View File
@@ -97,8 +97,15 @@ export interface TableImportParseSettings {
databaseType?: DatabaseType | null;
}
export function defaultTableImportEmptyStringAsNull(format: TableImportSourceFormat): boolean {
return format !== "excel";
/**
* 导入时分隔文本的「空字段即 NULL」默认值。
*
* 导出端默认把 NULL 写成 `\N` 字面量、空字符串才写成空字段,所以导入端默认必须
* 保留空字段(不当作 NULL),否则 DBX 自己导出的 CSV 在导入时会把空字符串还原成
* NULL,写回 `NOT NULL DEFAULT ''` 的列就会报「不允许为 null」。
*/
export function defaultTableImportEmptyStringAsNull(_format: TableImportSourceFormat): boolean {
return false;
}
export function buildTableImportParseOptions(settings: TableImportParseSettings): TableImportParseOptions {
@@ -113,6 +120,9 @@ export function buildTableImportParseOptions(settings: TableImportParseSettings)
lastDataRow: settings.lastDataRow,
trimValues: settings.trimValues,
emptyStringAsNull: settings.emptyStringAsNull,
// 勾选「空字符串作为 NULL」= 退回旧行为:关闭 NULL 字面量,空字段一律当 NULL。
// 不勾选时留空,由后端使用与导出端一致的默认字面量 `\N`。
nullLiteral: settings.emptyStringAsNull ? "" : undefined,
sheetName: settings.format === "excel" ? settings.sheetName || null : null,
jsonShape: settings.format === "json" ? settings.jsonShape : null,
// SQL 脚本需要按源方言解析字符串转义与标识符大小写(取目标连接的数据库类型)