fix(export): mark NULL as \N in CSV and TSV exports
This commit is contained in:
49 files changed
+1056
-130
No files matched your search
+27
-5
@@ -15,12 +15,34 @@ import {
|
||||
} from "@/lib/table/tableImport";
|
||||
|
||||
describe("tableImport", () => {
|
||||
it("preserves explicit empty strings by default for Excel only", () => {
|
||||
it("keeps empty fields as empty strings for every source format by default", () => {
|
||||
// 导出端默认用 `\N` 表示 NULL,空字段代表空字符串;导入端默认必须与之一致,
|
||||
// 否则 DBX 导出的 CSV 再导回时会把 '' 变成 NULL。
|
||||
expect(defaultTableImportEmptyStringAsNull("excel")).toBe(false);
|
||||
expect(defaultTableImportEmptyStringAsNull("csv")).toBe(true);
|
||||
expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(true);
|
||||
expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(true);
|
||||
expect(defaultTableImportEmptyStringAsNull("json")).toBe(true);
|
||||
expect(defaultTableImportEmptyStringAsNull("csv")).toBe(false);
|
||||
expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(false);
|
||||
expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(false);
|
||||
expect(defaultTableImportEmptyStringAsNull("json")).toBe(false);
|
||||
});
|
||||
|
||||
it("only disables the shared NULL literal when empty fields are treated as NULL", () => {
|
||||
const baseSettings = {
|
||||
delimiter: ",",
|
||||
textEncoding: "auto" as const,
|
||||
titleRow: 1,
|
||||
dataStartRow: 2,
|
||||
lastDataRow: 0,
|
||||
trimValues: false,
|
||||
jsonShape: "auto" as const,
|
||||
};
|
||||
|
||||
const sharedLiteral = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: false });
|
||||
expect(sharedLiteral.nullLiteral).toBeUndefined();
|
||||
expect(sharedLiteral.emptyStringAsNull).toBe(false);
|
||||
|
||||
const legacyEmptyField = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: true });
|
||||
expect(legacyEmptyField.nullLiteral).toBe("");
|
||||
expect(legacyEmptyField.emptyStringAsNull).toBe(true);
|
||||
});
|
||||
|
||||
it("formats import elapsed time for progress and terminal summaries", () => {
|
||||
|
||||
@@ -3388,9 +3388,9 @@ export async function cancelQueryResultExport(exportId: string, executionId?: st
|
||||
});
|
||||
}
|
||||
|
||||
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise<void> {
|
||||
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise<void> {
|
||||
const { formatCsv } = await import("@/lib/export/exportFormats");
|
||||
const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode);
|
||||
const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode, nullLiteral);
|
||||
const fileName = filePath.split(/[\\/]/).pop() || "export.csv";
|
||||
const blob = new Blob(["\uFEFF", content], {
|
||||
type: "text/csv;charset=utf-8",
|
||||
|
||||
@@ -5466,6 +5466,8 @@ export interface TableImportParseOptions {
|
||||
lastDataRow?: number | null;
|
||||
trimValues?: boolean | null;
|
||||
emptyStringAsNull?: boolean | null;
|
||||
/** 分隔文本里代表 NULL 的字面量。缺省表示用后端默认值 `\N`;空串表示关闭字面量,退回「空字段即 NULL」。 */
|
||||
nullLiteral?: string | null;
|
||||
sheetName?: string | null;
|
||||
sheetIndex?: number | null;
|
||||
jsonShape?: TableImportJsonShape | null;
|
||||
@@ -5861,6 +5863,8 @@ export interface TableExportRequest {
|
||||
insertMode?: SqlInsertMode;
|
||||
insertDialect?: SqlInsertDialect;
|
||||
csvQuoteMode?: CsvQuoteMode;
|
||||
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
|
||||
nullLiteral?: string;
|
||||
columns?: string[];
|
||||
selectedColumns?: SqlExportColumnSelection[];
|
||||
columnTypes?: Array<string | null | undefined>;
|
||||
@@ -5891,6 +5895,8 @@ export interface TableCsvExportOptions {
|
||||
pageSize?: number;
|
||||
timeoutSecs?: number;
|
||||
csvQuoteMode?: CsvQuoteMode;
|
||||
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
|
||||
nullLiteral?: string;
|
||||
}
|
||||
|
||||
export interface TableExportProgress {
|
||||
@@ -5917,6 +5923,8 @@ export interface QueryResultExportRequest {
|
||||
format: "csv" | "xlsx" | "json" | "txt" | "sql";
|
||||
insertMode?: SqlInsertMode;
|
||||
csvQuoteMode?: CsvQuoteMode;
|
||||
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
|
||||
nullLiteral?: string;
|
||||
includeSqlSheet?: boolean;
|
||||
pageSize: number;
|
||||
rowLimit?: number | null;
|
||||
@@ -6074,13 +6082,14 @@ export async function recordDatabaseExportDestination(directory: string): Promis
|
||||
await invoke("record_database_export_destination", { directory });
|
||||
}
|
||||
|
||||
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise<void> {
|
||||
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise<void> {
|
||||
return invoke("export_query_result_csv", {
|
||||
request: {
|
||||
filePath,
|
||||
columns,
|
||||
rows,
|
||||
csvQuoteMode,
|
||||
...(nullLiteral === undefined ? {} : { nullLiteral }),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { csvNullLiteralForMode, DEFAULT_CSV_NULL_LITERAL, DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode } from "@/lib/export/csvNullMode";
|
||||
|
||||
describe("normalizeCsvNullMode", () => {
|
||||
it("defaults to writing the NULL marker", () => {
|
||||
expect(DEFAULT_CSV_NULL_MODE).toBe("marker");
|
||||
expect(normalizeCsvNullMode(undefined)).toBe("marker");
|
||||
expect(normalizeCsvNullMode(null)).toBe("marker");
|
||||
expect(normalizeCsvNullMode("marker")).toBe("marker");
|
||||
});
|
||||
|
||||
it("keeps an explicit request for the legacy empty field", () => {
|
||||
expect(normalizeCsvNullMode("empty")).toBe("empty");
|
||||
});
|
||||
|
||||
it("falls back to the default for unknown values", () => {
|
||||
expect(normalizeCsvNullMode("")).toBe("marker");
|
||||
expect(normalizeCsvNullMode("\\N")).toBe("marker");
|
||||
expect(normalizeCsvNullMode(0)).toBe("marker");
|
||||
});
|
||||
});
|
||||
|
||||
describe("csvNullLiteralForMode", () => {
|
||||
it("writes the MySQL-style NULL marker by default", () => {
|
||||
expect(csvNullLiteralForMode("marker")).toBe(DEFAULT_CSV_NULL_LITERAL);
|
||||
expect(csvNullLiteralForMode("marker")).toBe("\\N");
|
||||
});
|
||||
|
||||
it("disables the literal with an empty string to restore the old export", () => {
|
||||
expect(csvNullLiteralForMode("empty")).toBe("");
|
||||
});
|
||||
});
|
||||
+68
-3
@@ -1,5 +1,58 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatTsv } from "@/lib/export/exportFormats";
|
||||
import { formatCsv, formatTsv } from "@/lib/export/exportFormats";
|
||||
|
||||
describe("formatCsv NULL handling", () => {
|
||||
it("writes NULL as an unquoted marker and an empty string as an empty quoted field", () => {
|
||||
// 即使 quote mode = all,NULL 字面量也必须裸写:PostgreSQL/ClickHouse 的 CSV 读取
|
||||
// 不把带引号的 `"\N"` 当作 NULL,裸 `\N` 才是它们和 MySQL 共同识别的写法。
|
||||
expect(
|
||||
formatCsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, ""],
|
||||
[2, null],
|
||||
],
|
||||
"all",
|
||||
"\\N",
|
||||
),
|
||||
).toBe('"id","name"\n"1",""\n"2",\\N');
|
||||
});
|
||||
|
||||
it("keeps the marker unquoted when only necessary fields are quoted", () => {
|
||||
expect(
|
||||
formatCsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, ""],
|
||||
[2, null],
|
||||
],
|
||||
"necessary",
|
||||
"\\N",
|
||||
),
|
||||
).toBe("id,name\n1,\n2,\\N");
|
||||
});
|
||||
|
||||
it("quotes a marker that itself needs quoting", () => {
|
||||
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
|
||||
expect(formatCsv(["id", "name"], [[1, null]], "all", "a,b")).toBe('"id","name"\n"1","a,b"');
|
||||
});
|
||||
|
||||
it("restores the legacy empty field when the marker is disabled", () => {
|
||||
// 关闭字面量时 NULL 是完全空字段、空字符串是 `""`:CSV 引号语义上仍有区别,
|
||||
// 但 DBX 的导入端比较的是去引号后的值,因此旧格式在往返时依赖不了这个区别。
|
||||
expect(
|
||||
formatCsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, ""],
|
||||
[2, null],
|
||||
],
|
||||
"all",
|
||||
"",
|
||||
),
|
||||
).toBe('"id","name"\n"1",""\n"2",');
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatTsv", () => {
|
||||
it("copies a value that only contains double quotes verbatim (#10087)", () => {
|
||||
@@ -14,8 +67,20 @@ describe("formatTsv", () => {
|
||||
expect(output).toBe('v\n"a\tb"\n"c\nd"');
|
||||
});
|
||||
|
||||
it("renders NULL as an empty field without adding quotes", () => {
|
||||
const output = formatTsv(["id", "name"], [[1, null]]);
|
||||
it("renders NULL as the null literal and an empty string as an empty field", () => {
|
||||
const output = formatTsv(
|
||||
["id", "name"],
|
||||
[
|
||||
[1, null],
|
||||
[2, ""],
|
||||
],
|
||||
);
|
||||
|
||||
expect(output).toBe("id\tname\n1\t\\N\n2\t");
|
||||
});
|
||||
|
||||
it("falls back to an empty NULL field when the literal is disabled", () => {
|
||||
const output = formatTsv(["id", "name"], [[1, null]], "");
|
||||
|
||||
expect(output).toBe("id\tname\n1\t");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* 导出 CSV 时 NULL 的写法。与 Rust `dbx_formats::csv_export` 保持一致:
|
||||
* `marker` 把 NULL 写成 `\N`(MySQL `SELECT ... INTO OUTFILE`、ClickHouse
|
||||
* `format_csv_null_representation` 的既定写法),空字符串才写成空字段,两者在文件里可区分,
|
||||
* 导回 NOT NULL DEFAULT '' 的列时不会被还原成 NULL;`empty` 是旧行为:NULL 与空字符串都写成空字段。
|
||||
*/
|
||||
export const DEFAULT_CSV_NULL_LITERAL = "\\N";
|
||||
|
||||
export const CSV_NULL_MODE_MARKER = "marker";
|
||||
export const CSV_NULL_MODE_EMPTY = "empty";
|
||||
|
||||
export type CsvNullMode = typeof CSV_NULL_MODE_MARKER | typeof CSV_NULL_MODE_EMPTY;
|
||||
|
||||
export const DEFAULT_CSV_NULL_MODE: CsvNullMode = CSV_NULL_MODE_MARKER;
|
||||
|
||||
export function normalizeCsvNullMode(value: unknown): CsvNullMode {
|
||||
return value === CSV_NULL_MODE_EMPTY ? CSV_NULL_MODE_EMPTY : DEFAULT_CSV_NULL_MODE;
|
||||
}
|
||||
|
||||
/** 导出请求里的 NULL 字面量;空串表示关闭该字面量(旧行为)。 */
|
||||
export function csvNullLiteralForMode(mode: CsvNullMode): string {
|
||||
return mode === CSV_NULL_MODE_EMPTY ? "" : DEFAULT_CSV_NULL_LITERAL;
|
||||
}
|
||||
@@ -1,26 +1,35 @@
|
||||
import type { DatabaseType, QueryResult } from "@/types/database";
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
||||
import { DEFAULT_CSV_NULL_LITERAL } from "@/lib/export/csvNullMode";
|
||||
import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||
|
||||
export type ExportCellValue = string | number | boolean | null;
|
||||
|
||||
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all"): string {
|
||||
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all", nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
|
||||
const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(",");
|
||||
const body = rows.map((row) => row.map((cell) => (cell === null ? "" : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
|
||||
// NULL 写成独立字面量、空字符串写成空字段,否则导入端无法把两者区分开
|
||||
// (写回 NOT NULL DEFAULT '' 的列时会把空字符串还原成 NULL)。
|
||||
//
|
||||
// 字面量裸写:PostgreSQL 的 COPY ... FORMAT csv 不认带引号的 NULL(`"\N"` 是普通字符串,
|
||||
// 整数列直接报错),ClickHouse 的 CSV 读取在引号内遇到 `\N` 会解析失败;只有裸 `\N` 是
|
||||
// 它们与 MySQL 共同识别的 NULL。字面量自身含分隔符/引号/换行时才加引号。
|
||||
const nullField = nullLiteral ? escapeCsvField(nullLiteral, "necessary") : "";
|
||||
const body = rows.map((row) => row.map((cell) => (cell === null ? nullField : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
|
||||
return `${header}\n${body}`;
|
||||
}
|
||||
|
||||
// Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)"
|
||||
// export: fields are joined by a tab and NULL becomes empty. A field is wrapped
|
||||
// in double quotes only when it contains a tab or a line break - the characters
|
||||
// that would otherwise corrupt the TSV row/column shape. A field that merely
|
||||
// contains a double quote is emitted verbatim: TSV is parsed by splitting on the
|
||||
// tab alone (there is no quote state machine on paste-back), so quoting such a
|
||||
// value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
|
||||
export function formatTsv(columns: string[], rows: ExportCellValue[][]): string {
|
||||
// export: fields are joined by a tab and NULL becomes the null literal. A field
|
||||
// is wrapped in double quotes only when it contains a tab or a line break - the
|
||||
// characters that would otherwise corrupt the TSV row/column shape. A field that
|
||||
// merely contains a double quote is emitted verbatim: TSV is parsed by splitting
|
||||
// on the tab alone (there is no quote state machine on paste-back), so quoting
|
||||
// such a value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
|
||||
export function formatTsv(columns: string[], rows: ExportCellValue[][], nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
|
||||
const esc = (value: ExportCellValue) => {
|
||||
const text = value === null ? "" : String(value);
|
||||
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返
|
||||
const text = value === null ? nullLiteral || "" : String(value);
|
||||
if (text.includes("\t") || text.includes("\n") || text.includes("\r")) {
|
||||
return `"${text.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
@@ -236,6 +236,11 @@ describe("editorSettingsDraftFromSettings", () => {
|
||||
expect(draft.completionTriggerMode).toBe("positional");
|
||||
});
|
||||
|
||||
it("maps csvNullMode and normalizes an unknown value back to the marker", () => {
|
||||
expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "empty" })).csvNullMode).toBe("empty");
|
||||
expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "bogus" as unknown } as Partial<EditorSettings>)).csvNullMode).toBe("marker");
|
||||
});
|
||||
|
||||
it("maps and normalizes tableCompletionSchemaQualification", () => {
|
||||
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "always" })).tableCompletionSchemaQualification).toBe("always");
|
||||
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "invalid" } as any)).tableCompletionSchemaQualification).toBe("collision");
|
||||
|
||||
@@ -413,4 +413,9 @@ describe("settingsTransfer", () => {
|
||||
expect(transferCategoryForKey("csvQuoteMode")).toBe("data");
|
||||
expect(collectTransferCategories(["csvQuoteMode", "pageSize"])).toEqual(["data"]);
|
||||
});
|
||||
|
||||
it("maps csvNullMode into the data category", () => {
|
||||
expect(transferCategoryForKey("csvNullMode")).toBe("data");
|
||||
expect(collectTransferCategories(["csvNullMode", "csvQuoteMode"])).toEqual(["data"]);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import { normalizeCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPo
|
||||
import { normalizeSqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification";
|
||||
import { normalizeTableHoverLookupMode } from "@/lib/editor/hoverTableLookup";
|
||||
import { normalizeRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates";
|
||||
import { normalizeCsvNullMode } from "@/lib/export/csvNullMode";
|
||||
|
||||
export const EDITOR_SETTINGS_DRAFT_KEYS = [
|
||||
"fontFamily",
|
||||
@@ -120,6 +121,7 @@ export const EDITOR_SETTINGS_DRAFT_KEYS = [
|
||||
"redisDatabaseDisplayLimit",
|
||||
"exportBatchSize",
|
||||
"csvQuoteMode",
|
||||
"csvNullMode",
|
||||
"exportRowLimitEnabled",
|
||||
"exportRowLimit",
|
||||
"queryExportKeysetOptimizationEnabled",
|
||||
@@ -164,6 +166,7 @@ function normalizedDraftValue(key: EditorSettingsDraftKey, value: unknown): unkn
|
||||
if (key === "tableCompletionSchemaQualification") return normalizeSqlTableCompletionSchemaQualification(value);
|
||||
if (key === "tableHoverLookupMode") return normalizeTableHoverLookupMode(value);
|
||||
if (key === "redisKeyTemplates") return normalizeRedisKeyTemplates(value);
|
||||
if (key === "csvNullMode") return normalizeCsvNullMode(value);
|
||||
if (key === "backgroundImage") return normalizeBackgroundImageSettings(value);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -178,6 +178,7 @@ const SETTINGS_TRANSFER_CATEGORY_KEYS: Record<SettingsTransferCategoryId, readon
|
||||
"redisDatabaseDisplayLimit",
|
||||
"exportBatchSize",
|
||||
"csvQuoteMode",
|
||||
"csvNullMode",
|
||||
"exportRowLimitEnabled",
|
||||
"exportRowLimit",
|
||||
"queryExportKeysetOptimizationEnabled",
|
||||
|
||||
@@ -97,8 +97,15 @@ export interface TableImportParseSettings {
|
||||
databaseType?: DatabaseType | null;
|
||||
}
|
||||
|
||||
export function defaultTableImportEmptyStringAsNull(format: TableImportSourceFormat): boolean {
|
||||
return format !== "excel";
|
||||
/**
|
||||
* 导入时分隔文本的「空字段即 NULL」默认值。
|
||||
*
|
||||
* 导出端默认把 NULL 写成 `\N` 字面量、空字符串才写成空字段,所以导入端默认必须
|
||||
* 保留空字段(不当作 NULL),否则 DBX 自己导出的 CSV 在导入时会把空字符串还原成
|
||||
* NULL,写回 `NOT NULL DEFAULT ''` 的列就会报「不允许为 null」。
|
||||
*/
|
||||
export function defaultTableImportEmptyStringAsNull(_format: TableImportSourceFormat): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
export function buildTableImportParseOptions(settings: TableImportParseSettings): TableImportParseOptions {
|
||||
@@ -113,6 +120,9 @@ export function buildTableImportParseOptions(settings: TableImportParseSettings)
|
||||
lastDataRow: settings.lastDataRow,
|
||||
trimValues: settings.trimValues,
|
||||
emptyStringAsNull: settings.emptyStringAsNull,
|
||||
// 勾选「空字符串作为 NULL」= 退回旧行为:关闭 NULL 字面量,空字段一律当 NULL。
|
||||
// 不勾选时留空,由后端使用与导出端一致的默认字面量 `\N`。
|
||||
nullLiteral: settings.emptyStringAsNull ? "" : undefined,
|
||||
sheetName: settings.format === "excel" ? settings.sheetName || null : null,
|
||||
jsonShape: settings.format === "json" ? settings.jsonShape : null,
|
||||
// SQL 脚本需要按源方言解析字符串转义与标识符大小写(取目标连接的数据库类型)
|
||||
|
||||
Reference in new issue
Block a user