fix(export): mark NULL as \N in CSV and TSV exports
This commit is contained in:
49 files changed
+1056
-130
No files matched your search
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { csvNullLiteralForMode, DEFAULT_CSV_NULL_LITERAL, DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode } from "@/lib/export/csvNullMode";
|
||||
|
||||
describe("normalizeCsvNullMode", () => {
|
||||
it("defaults to writing the NULL marker", () => {
|
||||
expect(DEFAULT_CSV_NULL_MODE).toBe("marker");
|
||||
expect(normalizeCsvNullMode(undefined)).toBe("marker");
|
||||
expect(normalizeCsvNullMode(null)).toBe("marker");
|
||||
expect(normalizeCsvNullMode("marker")).toBe("marker");
|
||||
});
|
||||
|
||||
it("keeps an explicit request for the legacy empty field", () => {
|
||||
expect(normalizeCsvNullMode("empty")).toBe("empty");
|
||||
});
|
||||
|
||||
it("falls back to the default for unknown values", () => {
|
||||
expect(normalizeCsvNullMode("")).toBe("marker");
|
||||
expect(normalizeCsvNullMode("\\N")).toBe("marker");
|
||||
expect(normalizeCsvNullMode(0)).toBe("marker");
|
||||
});
|
||||
});
|
||||
|
||||
describe("csvNullLiteralForMode", () => {
|
||||
it("writes the MySQL-style NULL marker by default", () => {
|
||||
expect(csvNullLiteralForMode("marker")).toBe(DEFAULT_CSV_NULL_LITERAL);
|
||||
expect(csvNullLiteralForMode("marker")).toBe("\\N");
|
||||
});
|
||||
|
||||
it("disables the literal with an empty string to restore the old export", () => {
|
||||
expect(csvNullLiteralForMode("empty")).toBe("");
|
||||
});
|
||||
});
|
||||
+68
-3
@@ -1,5 +1,58 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatTsv } from "@/lib/export/exportFormats";
|
||||
import { formatCsv, formatTsv } from "@/lib/export/exportFormats";
|
||||
|
||||
describe("formatCsv NULL handling", () => {
|
||||
it("writes NULL as an unquoted marker and an empty string as an empty quoted field", () => {
|
||||
// 即使 quote mode = all,NULL 字面量也必须裸写:PostgreSQL/ClickHouse 的 CSV 读取
|
||||
// 不把带引号的 `"\N"` 当作 NULL,裸 `\N` 才是它们和 MySQL 共同识别的写法。
|
||||
expect(
|
||||
formatCsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, ""],
|
||||
[2, null],
|
||||
],
|
||||
"all",
|
||||
"\\N",
|
||||
),
|
||||
).toBe('"id","name"\n"1",""\n"2",\\N');
|
||||
});
|
||||
|
||||
it("keeps the marker unquoted when only necessary fields are quoted", () => {
|
||||
expect(
|
||||
formatCsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, ""],
|
||||
[2, null],
|
||||
],
|
||||
"necessary",
|
||||
"\\N",
|
||||
),
|
||||
).toBe("id,name\n1,\n2,\\N");
|
||||
});
|
||||
|
||||
it("quotes a marker that itself needs quoting", () => {
|
||||
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
|
||||
expect(formatCsv(["id", "name"], [[1, null]], "all", "a,b")).toBe('"id","name"\n"1","a,b"');
|
||||
});
|
||||
|
||||
it("restores the legacy empty field when the marker is disabled", () => {
|
||||
// 关闭字面量时 NULL 是完全空字段、空字符串是 `""`:CSV 引号语义上仍有区别,
|
||||
// 但 DBX 的导入端比较的是去引号后的值,因此旧格式在往返时依赖不了这个区别。
|
||||
expect(
|
||||
formatCsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, ""],
|
||||
[2, null],
|
||||
],
|
||||
"all",
|
||||
"",
|
||||
),
|
||||
).toBe('"id","name"\n"1",""\n"2",');
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatTsv", () => {
|
||||
it("copies a value that only contains double quotes verbatim (#10087)", () => {
|
||||
@@ -14,8 +67,20 @@ describe("formatTsv", () => {
|
||||
expect(output).toBe('v\n"a\tb"\n"c\nd"');
|
||||
});
|
||||
|
||||
it("renders NULL as an empty field without adding quotes", () => {
|
||||
const output = formatTsv(["id", "name"], [[1, null]]);
|
||||
it("renders NULL as the null literal and an empty string as an empty field", () => {
|
||||
const output = formatTsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, null],
|
||||
[2, ""],
|
||||
],
|
||||
);
|
||||
|
||||
expect(output).toBe("id\tname\n1\t\\N\n2\t");
|
||||
});
|
||||
|
||||
it("falls back to an empty NULL field when the literal is disabled", () => {
|
||||
const output = formatTsv(["id", "name"], [[1, null]], "");
|
||||
|
||||
expect(output).toBe("id\tname\n1\t");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* 导出 CSV 时 NULL 的写法。与 Rust `dbx_formats::csv_export` 保持一致:
|
||||
* `marker` 把 NULL 写成 `\N`(MySQL `SELECT ... INTO OUTFILE`、ClickHouse
|
||||
* `format_csv_null_representation` 的既定写法),空字符串才写成空字段,两者在文件里可区分,
|
||||
* 导回 NOT NULL DEFAULT '' 的列时不会被还原成 NULL;`empty` 是旧行为:NULL 与空字符串都写成空字段。
|
||||
*/
|
||||
export const DEFAULT_CSV_NULL_LITERAL = "\\N";
|
||||
|
||||
export const CSV_NULL_MODE_MARKER = "marker";
|
||||
export const CSV_NULL_MODE_EMPTY = "empty";
|
||||
|
||||
export type CsvNullMode = typeof CSV_NULL_MODE_MARKER | typeof CSV_NULL_MODE_EMPTY;
|
||||
|
||||
export const DEFAULT_CSV_NULL_MODE: CsvNullMode = CSV_NULL_MODE_MARKER;
|
||||
|
||||
export function normalizeCsvNullMode(value: unknown): CsvNullMode {
|
||||
return value === CSV_NULL_MODE_EMPTY ? CSV_NULL_MODE_EMPTY : DEFAULT_CSV_NULL_MODE;
|
||||
}
|
||||
|
||||
/** 导出请求里的 NULL 字面量;空串表示关闭该字面量(旧行为)。 */
|
||||
export function csvNullLiteralForMode(mode: CsvNullMode): string {
|
||||
return mode === CSV_NULL_MODE_EMPTY ? "" : DEFAULT_CSV_NULL_LITERAL;
|
||||
}
|
||||
@@ -1,26 +1,35 @@
|
||||
import type { DatabaseType, QueryResult } from "@/types/database";
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
||||
import { DEFAULT_CSV_NULL_LITERAL } from "@/lib/export/csvNullMode";
|
||||
import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||
|
||||
export type ExportCellValue = string | number | boolean | null;
|
||||
|
||||
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all"): string {
|
||||
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all", nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
|
||||
const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(",");
|
||||
const body = rows.map((row) => row.map((cell) => (cell === null ? "" : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
|
||||
// NULL 写成独立字面量、空字符串写成空字段,否则导入端无法把两者区分开
|
||||
// (写回 NOT NULL DEFAULT '' 的列时会把空字符串还原成 NULL)。
|
||||
//
|
||||
// 字面量裸写:PostgreSQL 的 COPY ... FORMAT csv 不认带引号的 NULL(`"\N"` 是普通字符串,
|
||||
// 整数列直接报错),ClickHouse 的 CSV 读取在引号内遇到 `\N` 会解析失败;只有裸 `\N` 是
|
||||
// 它们与 MySQL 共同识别的 NULL。字面量自身含分隔符/引号/换行时才加引号。
|
||||
const nullField = nullLiteral ? escapeCsvField(nullLiteral, "necessary") : "";
|
||||
const body = rows.map((row) => row.map((cell) => (cell === null ? nullField : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
|
||||
return `${header}\n${body}`;
|
||||
}
|
||||
|
||||
// Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)"
|
||||
// export: fields are joined by a tab and NULL becomes empty. A field is wrapped
|
||||
// in double quotes only when it contains a tab or a line break - the characters
|
||||
// that would otherwise corrupt the TSV row/column shape. A field that merely
|
||||
// contains a double quote is emitted verbatim: TSV is parsed by splitting on the
|
||||
// tab alone (there is no quote state machine on paste-back), so quoting such a
|
||||
// value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
|
||||
export function formatTsv(columns: string[], rows: ExportCellValue[][]): string {
|
||||
// export: fields are joined by a tab and NULL becomes the null literal. A field
|
||||
// is wrapped in double quotes only when it contains a tab or a line break - the
|
||||
// characters that would otherwise corrupt the TSV row/column shape. A field that
|
||||
// merely contains a double quote is emitted verbatim: TSV is parsed by splitting
|
||||
// on the tab alone (there is no quote state machine on paste-back), so quoting
|
||||
// such a value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
|
||||
export function formatTsv(columns: string[], rows: ExportCellValue[][], nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
|
||||
const esc = (value: ExportCellValue) => {
|
||||
const text = value === null ? "" : String(value);
|
||||
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返
|
||||
const text = value === null ? nullLiteral || "" : String(value);
|
||||
if (text.includes("\t") || text.includes("\n") || text.includes("\r")) {
|
||||
return `"${text.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user