fix(export): mark NULL as \N in CSV and TSV exports

This commit is contained in:
zipg authored and GitHub committed 2026-09-28 22:35:43 +08:00
1 parent 02f27dc294
commit 1aaa5ca717
49 files changed
+1056 -130

No files matched your search

@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { csvNullLiteralForMode, DEFAULT_CSV_NULL_LITERAL, DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode } from "@/lib/export/csvNullMode";
describe("normalizeCsvNullMode", () => {
it("defaults to writing the NULL marker", () => {
expect(DEFAULT_CSV_NULL_MODE).toBe("marker");
expect(normalizeCsvNullMode(undefined)).toBe("marker");
expect(normalizeCsvNullMode(null)).toBe("marker");
expect(normalizeCsvNullMode("marker")).toBe("marker");
});
it("keeps an explicit request for the legacy empty field", () => {
expect(normalizeCsvNullMode("empty")).toBe("empty");
});
it("falls back to the default for unknown values", () => {
expect(normalizeCsvNullMode("")).toBe("marker");
expect(normalizeCsvNullMode("\\N")).toBe("marker");
expect(normalizeCsvNullMode(0)).toBe("marker");
});
});
describe("csvNullLiteralForMode", () => {
it("writes the MySQL-style NULL marker by default", () => {
expect(csvNullLiteralForMode("marker")).toBe(DEFAULT_CSV_NULL_LITERAL);
expect(csvNullLiteralForMode("marker")).toBe("\\N");
});
it("disables the literal with an empty string to restore the old export", () => {
expect(csvNullLiteralForMode("empty")).toBe("");
});
});
+68 -3
View File
@@ -1,5 +1,58 @@
import { describe, expect, it } from "vitest";
import { formatTsv } from "@/lib/export/exportFormats";
import { formatCsv, formatTsv } from "@/lib/export/exportFormats";
describe("formatCsv NULL handling", () => {
it("writes NULL as an unquoted marker and an empty string as an empty quoted field", () => {
// 即使 quote mode = all,NULL 字面量也必须裸写:PostgreSQL/ClickHouse 的 CSV 读取
// 不把带引号的 `"\N"` 当作 NULL,裸 `\N` 才是它们和 MySQL 共同识别的写法。
expect(
formatCsv(
["id", "name"],
[
[1, ""],
[2, null],
],
"all",
"\\N",
),
).toBe('"id","name"\n"1",""\n"2",\\N');
});
it("keeps the marker unquoted when only necessary fields are quoted", () => {
expect(
formatCsv(
["id", "name"],
[
[1, ""],
[2, null],
],
"necessary",
"\\N",
),
).toBe("id,name\n1,\n2,\\N");
});
it("quotes a marker that itself needs quoting", () => {
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
expect(formatCsv(["id", "name"], [[1, null]], "all", "a,b")).toBe('"id","name"\n"1","a,b"');
});
it("restores the legacy empty field when the marker is disabled", () => {
// 关闭字面量时 NULL 是完全空字段、空字符串是 `""`:CSV 引号语义上仍有区别,
// 但 DBX 的导入端比较的是去引号后的值,因此旧格式在往返时依赖不了这个区别。
expect(
formatCsv(
["id", "name"],
[
[1, ""],
[2, null],
],
"all",
"",
),
).toBe('"id","name"\n"1",""\n"2",');
});
});
describe("formatTsv", () => {
it("copies a value that only contains double quotes verbatim (#10087)", () => {
@@ -14,8 +67,20 @@ describe("formatTsv", () => {
expect(output).toBe('v\n"a\tb"\n"c\nd"');
});
it("renders NULL as an empty field without adding quotes", () => {
const output = formatTsv(["id", "name"], [[1, null]]);
it("renders NULL as the null literal and an empty string as an empty field", () => {
const output = formatTsv(
["id", "name"],
[
[1, null],
[2, ""],
],
);
expect(output).toBe("id\tname\n1\t\\N\n2\t");
});
it("falls back to an empty NULL field when the literal is disabled", () => {
const output = formatTsv(["id", "name"], [[1, null]], "");
expect(output).toBe("id\tname\n1\t");
});
@@ -0,0 +1,23 @@
/**
* 导出 CSV 时 NULL 的写法。与 Rust `dbx_formats::csv_export` 保持一致:
* `marker` 把 NULL 写成 `\N`(MySQL `SELECT ... INTO OUTFILE`、ClickHouse
* `format_csv_null_representation` 的既定写法),空字符串才写成空字段,两者在文件里可区分,
* 导回 NOT NULL DEFAULT '' 的列时不会被还原成 NULL;`empty` 是旧行为:NULL 与空字符串都写成空字段。
*/
export const DEFAULT_CSV_NULL_LITERAL = "\\N";
export const CSV_NULL_MODE_MARKER = "marker";
export const CSV_NULL_MODE_EMPTY = "empty";
export type CsvNullMode = typeof CSV_NULL_MODE_MARKER | typeof CSV_NULL_MODE_EMPTY;
export const DEFAULT_CSV_NULL_MODE: CsvNullMode = CSV_NULL_MODE_MARKER;
export function normalizeCsvNullMode(value: unknown): CsvNullMode {
return value === CSV_NULL_MODE_EMPTY ? CSV_NULL_MODE_EMPTY : DEFAULT_CSV_NULL_MODE;
}
/** 导出请求里的 NULL 字面量;空串表示关闭该字面量(旧行为)。 */
export function csvNullLiteralForMode(mode: CsvNullMode): string {
return mode === CSV_NULL_MODE_EMPTY ? "" : DEFAULT_CSV_NULL_LITERAL;
}
+19 -10
View File
@@ -1,26 +1,35 @@
import type { DatabaseType, QueryResult } from "@/types/database";
import * as api from "@/lib/backend/api";
import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
import { DEFAULT_CSV_NULL_LITERAL } from "@/lib/export/csvNullMode";
import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
export type ExportCellValue = string | number | boolean | null;
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all"): string {
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all", nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(",");
const body = rows.map((row) => row.map((cell) => (cell === null ? "" : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
// NULL 写成独立字面量、空字符串写成空字段,否则导入端无法把两者区分开
// (写回 NOT NULL DEFAULT '' 的列时会把空字符串还原成 NULL)。
//
// 字面量裸写:PostgreSQL 的 COPY ... FORMAT csv 不认带引号的 NULL(`"\N"` 是普通字符串,
// 整数列直接报错),ClickHouse 的 CSV 读取在引号内遇到 `\N` 会解析失败;只有裸 `\N` 是
// 它们与 MySQL 共同识别的 NULL。字面量自身含分隔符/引号/换行时才加引号。
const nullField = nullLiteral ? escapeCsvField(nullLiteral, "necessary") : "";
const body = rows.map((row) => row.map((cell) => (cell === null ? nullField : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
return `${header}\n${body}`;
}
// Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)"
// export: fields are joined by a tab and NULL becomes empty. A field is wrapped
// in double quotes only when it contains a tab or a line break - the characters
// that would otherwise corrupt the TSV row/column shape. A field that merely
// contains a double quote is emitted verbatim: TSV is parsed by splitting on the
// tab alone (there is no quote state machine on paste-back), so quoting such a
// value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
export function formatTsv(columns: string[], rows: ExportCellValue[][]): string {
// export: fields are joined by a tab and NULL becomes the null literal. A field
// is wrapped in double quotes only when it contains a tab or a line break - the
// characters that would otherwise corrupt the TSV row/column shape. A field that
// merely contains a double quote is emitted verbatim: TSV is parsed by splitting
// on the tab alone (there is no quote state machine on paste-back), so quoting
// such a value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
export function formatTsv(columns: string[], rows: ExportCellValue[][], nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
const esc = (value: ExportCellValue) => {
const text = value === null ? "" : String(value);
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返
const text = value === null ? nullLiteral || "" : String(value);
if (text.includes("\t") || text.includes("\n") || text.includes("\r")) {
return `"${text.replace(/"/g, '""')}"`;
}