fix(export): mark NULL as \N in CSV and TSV exports
This commit is contained in:
49 files changed
+1052
-126
No files matched your search
@@ -101,6 +101,7 @@ import {
|
|||||||
type TabSortMode,
|
type TabSortMode,
|
||||||
type UpdateDownloadSource,
|
type UpdateDownloadSource,
|
||||||
type CsvQuoteMode,
|
type CsvQuoteMode,
|
||||||
|
type CsvNullMode,
|
||||||
type CustomThemeColors,
|
type CustomThemeColors,
|
||||||
type CustomTheme,
|
type CustomTheme,
|
||||||
type McpConnectionPolicy,
|
type McpConnectionPolicy,
|
||||||
@@ -854,6 +855,7 @@ const editSidebarIndent = ref(settingsStore.editorSettings.sidebarIndent);
|
|||||||
const editSidebarFontSize = ref(settingsStore.editorSettings.sidebarFontSize);
|
const editSidebarFontSize = ref(settingsStore.editorSettings.sidebarFontSize);
|
||||||
const editExportBatchSize = ref(settingsStore.editorSettings.exportBatchSize);
|
const editExportBatchSize = ref(settingsStore.editorSettings.exportBatchSize);
|
||||||
const editCsvQuoteMode = ref<CsvQuoteMode>(settingsStore.editorSettings.csvQuoteMode);
|
const editCsvQuoteMode = ref<CsvQuoteMode>(settingsStore.editorSettings.csvQuoteMode);
|
||||||
|
const editCsvNullMode = ref<CsvNullMode>(settingsStore.editorSettings.csvNullMode);
|
||||||
const editGlobalDateTimeDisplayFormat = ref(settingsStore.editorSettings.globalDateTimeDisplayFormat);
|
const editGlobalDateTimeDisplayFormat = ref(settingsStore.editorSettings.globalDateTimeDisplayFormat);
|
||||||
const editGlobalDateTimeExportFormat = ref(settingsStore.editorSettings.globalDateTimeExportFormat);
|
const editGlobalDateTimeExportFormat = ref(settingsStore.editorSettings.globalDateTimeExportFormat);
|
||||||
const editGlobalDateTimeImportFormat = ref(settingsStore.editorSettings.globalDateTimeImportFormat);
|
const editGlobalDateTimeImportFormat = ref(settingsStore.editorSettings.globalDateTimeImportFormat);
|
||||||
@@ -1117,6 +1119,7 @@ function currentEditorSettingsDraft(): EditorSettingsDraft {
|
|||||||
redisDatabaseDisplayLimit: editRedisDatabaseDisplayLimit.value,
|
redisDatabaseDisplayLimit: editRedisDatabaseDisplayLimit.value,
|
||||||
exportBatchSize: editExportBatchSize.value,
|
exportBatchSize: editExportBatchSize.value,
|
||||||
csvQuoteMode: editCsvQuoteMode.value,
|
csvQuoteMode: editCsvQuoteMode.value,
|
||||||
|
csvNullMode: editCsvNullMode.value,
|
||||||
globalDateTimeDisplayFormat: editGlobalDateTimeDisplayFormat.value,
|
globalDateTimeDisplayFormat: editGlobalDateTimeDisplayFormat.value,
|
||||||
globalDateTimeExportFormat: editGlobalDateTimeExportFormat.value,
|
globalDateTimeExportFormat: editGlobalDateTimeExportFormat.value,
|
||||||
globalDateTimeImportFormat: editGlobalDateTimeImportFormat.value,
|
globalDateTimeImportFormat: editGlobalDateTimeImportFormat.value,
|
||||||
@@ -1779,6 +1782,7 @@ function syncEditorSettingsDraftFromStore() {
|
|||||||
editSidebarFontSize.value = settingsStore.editorSettings.sidebarFontSize;
|
editSidebarFontSize.value = settingsStore.editorSettings.sidebarFontSize;
|
||||||
editExportBatchSize.value = settingsStore.editorSettings.exportBatchSize;
|
editExportBatchSize.value = settingsStore.editorSettings.exportBatchSize;
|
||||||
editCsvQuoteMode.value = settingsStore.editorSettings.csvQuoteMode;
|
editCsvQuoteMode.value = settingsStore.editorSettings.csvQuoteMode;
|
||||||
|
editCsvNullMode.value = settingsStore.editorSettings.csvNullMode;
|
||||||
editGlobalDateTimeDisplayFormat.value = settingsStore.editorSettings.globalDateTimeDisplayFormat;
|
editGlobalDateTimeDisplayFormat.value = settingsStore.editorSettings.globalDateTimeDisplayFormat;
|
||||||
editGlobalDateTimeExportFormat.value = settingsStore.editorSettings.globalDateTimeExportFormat;
|
editGlobalDateTimeExportFormat.value = settingsStore.editorSettings.globalDateTimeExportFormat;
|
||||||
editGlobalDateTimeImportFormat.value = settingsStore.editorSettings.globalDateTimeImportFormat;
|
editGlobalDateTimeImportFormat.value = settingsStore.editorSettings.globalDateTimeImportFormat;
|
||||||
@@ -1915,6 +1919,7 @@ const editorSettingsDraftRefs: EditorSettingsDraftRefMap = {
|
|||||||
redisDatabaseDisplayLimit: editRedisDatabaseDisplayLimit,
|
redisDatabaseDisplayLimit: editRedisDatabaseDisplayLimit,
|
||||||
exportBatchSize: editExportBatchSize,
|
exportBatchSize: editExportBatchSize,
|
||||||
csvQuoteMode: editCsvQuoteMode,
|
csvQuoteMode: editCsvQuoteMode,
|
||||||
|
csvNullMode: editCsvNullMode,
|
||||||
exportRowLimitEnabled: editExportRowLimitEnabled,
|
exportRowLimitEnabled: editExportRowLimitEnabled,
|
||||||
exportRowLimit: editExportRowLimit,
|
exportRowLimit: editExportRowLimit,
|
||||||
queryExportKeysetOptimizationEnabled: editQueryExportKeysetOptimizationEnabled,
|
queryExportKeysetOptimizationEnabled: editQueryExportKeysetOptimizationEnabled,
|
||||||
@@ -2428,6 +2433,7 @@ function resetDefaultsForTab(tab: SettingsCategory) {
|
|||||||
editRedisDatabaseDisplayLimit.value = DEFAULT_EDITOR_SETTINGS.redisDatabaseDisplayLimit;
|
editRedisDatabaseDisplayLimit.value = DEFAULT_EDITOR_SETTINGS.redisDatabaseDisplayLimit;
|
||||||
editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize;
|
editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize;
|
||||||
editCsvQuoteMode.value = DEFAULT_EDITOR_SETTINGS.csvQuoteMode;
|
editCsvQuoteMode.value = DEFAULT_EDITOR_SETTINGS.csvQuoteMode;
|
||||||
|
editCsvNullMode.value = DEFAULT_EDITOR_SETTINGS.csvNullMode;
|
||||||
editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat;
|
editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat;
|
||||||
editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat;
|
editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat;
|
||||||
editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat;
|
editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat;
|
||||||
@@ -2572,6 +2578,7 @@ function resetAllDefaults() {
|
|||||||
editRedisDatabaseDisplayLimit.value = DEFAULT_EDITOR_SETTINGS.redisDatabaseDisplayLimit;
|
editRedisDatabaseDisplayLimit.value = DEFAULT_EDITOR_SETTINGS.redisDatabaseDisplayLimit;
|
||||||
editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize;
|
editExportBatchSize.value = DEFAULT_EDITOR_SETTINGS.exportBatchSize;
|
||||||
editCsvQuoteMode.value = DEFAULT_EDITOR_SETTINGS.csvQuoteMode;
|
editCsvQuoteMode.value = DEFAULT_EDITOR_SETTINGS.csvQuoteMode;
|
||||||
|
editCsvNullMode.value = DEFAULT_EDITOR_SETTINGS.csvNullMode;
|
||||||
editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat;
|
editGlobalDateTimeDisplayFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeDisplayFormat;
|
||||||
editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat;
|
editGlobalDateTimeExportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeExportFormat;
|
||||||
editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat;
|
editGlobalDateTimeImportFormat.value = DEFAULT_EDITOR_SETTINGS.globalDateTimeImportFormat;
|
||||||
@@ -8550,6 +8557,23 @@ onUnmounted(() => {
|
|||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="flex items-start justify-between gap-4">
|
||||||
|
<div class="min-w-0 space-y-0.5">
|
||||||
|
<Label for="csv-null-mode">{{ t("settings.csvNullMode") }}</Label>
|
||||||
|
<p class="text-xs text-muted-foreground">
|
||||||
|
{{ t("settings.csvNullModeDescription") }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<Select v-model="editCsvNullMode">
|
||||||
|
<SelectTrigger id="csv-null-mode" class="h-8 w-44 shrink-0">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="marker">{{ t("settings.csvNullModeMarker") }}</SelectItem>
|
||||||
|
<SelectItem value="empty">{{ t("settings.csvNullModeEmpty") }}</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
<div class="space-y-2">
|
<div class="space-y-2">
|
||||||
<Label>{{ t("settings.exportBatchSize") }}</Label>
|
<Label>{{ t("settings.exportBatchSize") }}</Label>
|
||||||
<div class="flex items-center gap-3">
|
<div class="flex items-center gap-3">
|
||||||
|
|||||||
@@ -769,7 +769,7 @@ async function prepareBatchSources(sources: ImportSource[]) {
|
|||||||
loadingPreview.value = false;
|
loadingPreview.value = false;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : true;
|
emptyStringAsNull.value = formats.length && formats.every((format) => format === formats[0]) ? defaultTableImportEmptyStringAsNull(formats[0]!) : defaultTableImportEmptyStringAsNull(formats[0] ?? "csv");
|
||||||
try {
|
try {
|
||||||
for (const [index, source] of sources.entries()) {
|
for (const [index, source] of sources.entries()) {
|
||||||
const format = formats[index]!;
|
const format = formats[index]!;
|
||||||
@@ -1471,10 +1471,13 @@ watch(rawProgressPercent, (percent) => {
|
|||||||
<input v-model="trimValues" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
<input v-model="trimValues" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
||||||
{{ t("tableImport.trimValues") }}
|
{{ t("tableImport.trimValues") }}
|
||||||
</label>
|
</label>
|
||||||
|
<div class="space-y-1">
|
||||||
<label class="flex items-center gap-2 text-xs">
|
<label class="flex items-center gap-2 text-xs">
|
||||||
<input v-model="emptyStringAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
<input v-model="emptyStringAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" />
|
||||||
{{ t("tableImport.emptyStringAsNull") }}
|
{{ t("tableImport.emptyStringAsNull") }}
|
||||||
</label>
|
</label>
|
||||||
|
<p class="text-[11px] text-muted-foreground">{{ t("tableImport.emptyStringAsNullHint") }}</p>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else-if="sourceFormat === 'sql'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
<div v-else-if="sourceFormat === 'sql'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
|
|||||||
import { generateDatabaseExportId } from "@/lib/export/databaseExport";
|
import { generateDatabaseExportId } from "@/lib/export/databaseExport";
|
||||||
import { buildXlsxHeaderOverrides, hasXlsxHeaderComments, type XlsxExportOptions, type XlsxHeaderMode } from "@/lib/export/xlsxHeader";
|
import { buildXlsxHeaderOverrides, hasXlsxHeaderComments, type XlsxExportOptions, type XlsxHeaderMode } from "@/lib/export/xlsxHeader";
|
||||||
import { showSqlInsertModeDialog, type SqlInsertDialect, type SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
import { showSqlInsertModeDialog, type SqlInsertDialect, type SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||||
|
import { csvNullLiteralForMode } from "@/lib/export/csvNullMode";
|
||||||
import { copyToClipboard, eventTargetAllowsAppClipboardShortcut } from "@/lib/common/clipboard";
|
import { copyToClipboard, eventTargetAllowsAppClipboardShortcut } from "@/lib/common/clipboard";
|
||||||
import {
|
import {
|
||||||
defaultPasteTableMode,
|
defaultPasteTableMode,
|
||||||
@@ -2423,7 +2424,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "
|
|||||||
executePage: (sql) => api.executeQuery(props.connection.id, props.database, sql),
|
executePage: (sql) => api.executeQuery(props.connection.id, props.database, sql),
|
||||||
});
|
});
|
||||||
if (format === "csv") {
|
if (format === "csv") {
|
||||||
await api.exportQueryResultCsv(filePath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), settingsStore.editorSettings.csvQuoteMode);
|
await api.exportQueryResultCsv(filePath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), settingsStore.editorSettings.csvQuoteMode, csvNullLiteralForMode(settingsStore.editorSettings.csvNullMode));
|
||||||
} else {
|
} else {
|
||||||
const comments = result.columns.map((name) => columnInfos?.find((column) => column.name.toLocaleLowerCase() === name.toLocaleLowerCase())?.comment);
|
const comments = result.columns.map((name) => columnInfos?.find((column) => column.name.toLocaleLowerCase() === name.toLocaleLowerCase())?.comment);
|
||||||
const headerOverrides = buildXlsxHeaderOverrides(result.columns, comments, headerMode);
|
const headerOverrides = buildXlsxHeaderOverrides(result.columns, comments, headerMode);
|
||||||
@@ -2463,6 +2464,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "
|
|||||||
format,
|
format,
|
||||||
...(format === "sql" ? { insertMode, insertDialect, splitMaxMb } : {}),
|
...(format === "sql" ? { insertMode, insertDialect, splitMaxMb } : {}),
|
||||||
csvQuoteMode: settingsStore.editorSettings.csvQuoteMode,
|
csvQuoteMode: settingsStore.editorSettings.csvQuoteMode,
|
||||||
|
nullLiteral: csvNullLiteralForMode(settingsStore.editorSettings.csvNullMode),
|
||||||
columns,
|
columns,
|
||||||
columnComments: format === "xlsx" ? columnComments : undefined,
|
columnComments: format === "xlsx" ? columnComments : undefined,
|
||||||
autoFilter: format === "xlsx" ? autoFilter : undefined,
|
autoFilter: format === "xlsx" ? autoFilter : undefined,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { DEFAULT_DATA_GRID_EXTRACTOR_OPTIONS } from "@/lib/dataGrid/dataGridCopy
|
|||||||
import { clearDataGridClipboardCopy, parseDataGridClipboard } from "@/lib/dataGrid/dataGridClipboard";
|
import { clearDataGridClipboardCopy, parseDataGridClipboard } from "@/lib/dataGrid/dataGridClipboard";
|
||||||
import { MONGO_DOCUMENT_GRID_NULL, mongoDocumentGridExternalValue } from "@/lib/mongo/mongoDocumentValues";
|
import { MONGO_DOCUMENT_GRID_NULL, mongoDocumentGridExternalValue } from "@/lib/mongo/mongoDocumentValues";
|
||||||
import { saveTextFile } from "@/lib/export/saveTextFile";
|
import { saveTextFile } from "@/lib/export/saveTextFile";
|
||||||
|
import { useSettingsStore } from "@/stores/settingsStore";
|
||||||
|
|
||||||
const toast = vi.fn();
|
const toast = vi.fn();
|
||||||
|
|
||||||
@@ -780,8 +781,8 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
expect(copyToClipboard).toHaveBeenCalledWith("2020-12-02 15:18:29");
|
expect(copyToClipboard).toHaveBeenCalledWith("2020-12-02 15:18:29");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("copies all rows with empty fields for NULL cells", async () => {
|
it("copies all rows with the null literal for NULL cells", async () => {
|
||||||
const text = "id\tname\n1\t\n2\tAda";
|
const text = "id\tname\n1\t\\N\n2\tAda";
|
||||||
const state = createExportState(editableTable, ["id", "name"], undefined, undefined, undefined, [
|
const state = createExportState(editableTable, ["id", "name"], undefined, undefined, undefined, [
|
||||||
[1, null],
|
[1, null],
|
||||||
[2, "Ada"],
|
[2, "Ada"],
|
||||||
@@ -790,6 +791,7 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
await state.copyAll();
|
await state.copyAll();
|
||||||
|
|
||||||
expect(copyToClipboard).toHaveBeenCalledWith(text);
|
expect(copyToClipboard).toHaveBeenCalledWith(text);
|
||||||
|
// 内部剪贴板按文本全等匹配保留逻辑矩阵:粘贴回网格时 NULL 语义无损还原
|
||||||
expect(parseDataGridClipboard(text)).toEqual([
|
expect(parseDataGridClipboard(text)).toEqual([
|
||||||
["id", "name"],
|
["id", "name"],
|
||||||
["1", null],
|
["1", null],
|
||||||
@@ -1672,7 +1674,7 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
await state.copyAll();
|
await state.copyAll();
|
||||||
expect(copyToClipboard).toHaveBeenCalledWith("_id\tnullable\n1\t");
|
expect(copyToClipboard).toHaveBeenCalledWith("_id\tnullable\n1\t\\N");
|
||||||
|
|
||||||
vi.mocked(extractDataGridSelection).mockResolvedValueOnce({ text: "", mimeType: "text/tab-separated-values", fileExtension: "tsv", rowCount: 1, columnCount: 1 });
|
vi.mocked(extractDataGridSelection).mockResolvedValueOnce({ text: "", mimeType: "text/tab-separated-values", fileExtension: "tsv", rowCount: 1, columnCount: 1 });
|
||||||
await expect(state.copyWithExtractor("tsv")).resolves.toBe(true);
|
await expect(state.copyWithExtractor("tsv")).resolves.toBe(true);
|
||||||
@@ -1680,7 +1682,7 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
|
|
||||||
setActivePinia(createPinia());
|
setActivePinia(createPinia());
|
||||||
await state.exportCurrentPageCsv();
|
await state.exportCurrentPageCsv();
|
||||||
expect(exportQueryResultCsv).toHaveBeenCalledWith(expect.any(String), ["_id", "nullable"], [["1", null]], expect.anything());
|
expect(exportQueryResultCsv).toHaveBeenCalledWith(expect.any(String), ["_id", "nullable"], [["1", null]], expect.anything(), expect.anything());
|
||||||
|
|
||||||
const reservedString = MONGO_DOCUMENT_GRID_NULL;
|
const reservedString = MONGO_DOCUMENT_GRID_NULL;
|
||||||
const fullExportState = createMongoExportState({
|
const fullExportState = createMongoExportState({
|
||||||
@@ -1699,7 +1701,7 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
await fullExportState.exportCsv();
|
await fullExportState.exportCsv();
|
||||||
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["_id", "value"], [["1", reservedString]], expect.anything());
|
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["_id", "value"], [["1", reservedString]], expect.anything(), expect.anything());
|
||||||
});
|
});
|
||||||
|
|
||||||
it("exports only visible Mongo columns from the full result set", async () => {
|
it("exports only visible Mongo columns from the full result set", async () => {
|
||||||
@@ -1722,7 +1724,7 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
|
|
||||||
await state.exportCsv();
|
await state.exportCsv();
|
||||||
|
|
||||||
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything());
|
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything(), expect.anything());
|
||||||
});
|
});
|
||||||
|
|
||||||
it("applies visible Mongo columns when the complete result is already local", async () => {
|
it("applies visible Mongo columns when the complete result is already local", async () => {
|
||||||
@@ -1751,7 +1753,23 @@ describe("useDataGridExport prepared row statements", () => {
|
|||||||
|
|
||||||
await state.exportCsv();
|
await state.exportCsv();
|
||||||
|
|
||||||
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything());
|
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"], ["Other"]], expect.anything(), expect.anything());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes the CSV NULL setting to the export call", async () => {
|
||||||
|
setActivePinia(createPinia());
|
||||||
|
const state = createMongoExportState({
|
||||||
|
columns: ["name"],
|
||||||
|
item: { ...row(["Visible"]), sourceIndex: 0 },
|
||||||
|
mongoDocuments: [{ name: "Visible" }],
|
||||||
|
});
|
||||||
|
|
||||||
|
await state.exportCsv();
|
||||||
|
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"]], expect.anything(), "\\N");
|
||||||
|
|
||||||
|
useSettingsStore().editorSettings.csvNullMode = "empty";
|
||||||
|
await state.exportCsv();
|
||||||
|
expect(exportQueryResultCsv).toHaveBeenLastCalledWith(expect.any(String), ["name"], [["Visible"]], expect.anything(), "");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("exports missing Mongo fields as null while retaining explicit empty strings", async () => {
|
it("exports missing Mongo fields as null while retaining explicit empty strings", async () => {
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import { summarizeExportRows } from "@/lib/export/exportDiagnostics";
|
|||||||
import { appendDebugLog, appendNativeProcessMemoryLog, getBrowserMemorySnapshot, isDebugLoggingEnabled } from "@/lib/backend/debugLog";
|
import { appendDebugLog, appendNativeProcessMemoryLog, getBrowserMemorySnapshot, isDebugLoggingEnabled } from "@/lib/backend/debugLog";
|
||||||
import { uuid } from "@/lib/common/utils";
|
import { uuid } from "@/lib/common/utils";
|
||||||
import { useSettingsStore } from "@/stores/settingsStore";
|
import { useSettingsStore } from "@/stores/settingsStore";
|
||||||
|
import { csvNullLiteralForMode } from "@/lib/export/csvNullMode";
|
||||||
import { expandNestedJsonStringsForCopy } from "@/lib/common/jsonCopyValue";
|
import { expandNestedJsonStringsForCopy } from "@/lib/common/jsonCopyValue";
|
||||||
import { buildMongoCopyDocumentFromOriginal, buildMongoCopyInsertDocument, buildMongoCopyUpdateDocument, formatMongoShellLiteral, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
|
import { buildMongoCopyDocumentFromOriginal, buildMongoCopyInsertDocument, buildMongoCopyUpdateDocument, formatMongoShellLiteral, type MongoInputValue } from "@/lib/mongo/mongoDocumentValues";
|
||||||
import { formatMongoShellText } from "@/lib/mongo/mongoFormatter";
|
import { formatMongoShellText } from "@/lib/mongo/mongoFormatter";
|
||||||
@@ -912,7 +913,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
|
|||||||
}
|
}
|
||||||
outputPath = path as string;
|
outputPath = path as string;
|
||||||
}
|
}
|
||||||
await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode);
|
await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode, csvNullLiteralForMode(useSettingsStore().editorSettings.csvNullMode));
|
||||||
if (needsFullExport && exportProgressState) {
|
if (needsFullExport && exportProgressState) {
|
||||||
exportProgressState.value = {
|
exportProgressState.value = {
|
||||||
...exportProgressState.value,
|
...exportProgressState.value,
|
||||||
@@ -952,7 +953,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
|
|||||||
}
|
}
|
||||||
const result = await resultToExport(undefined, undefined, false);
|
const result = await resultToExport(undefined, undefined, false);
|
||||||
const rows = forceCsvTextForTemporalColumns(result.rows, result.columnTypes);
|
const rows = forceCsvTextForTemporalColumns(result.rows, result.columnTypes);
|
||||||
await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode);
|
await api.exportQueryResultCsv(outputPath, result.columns, rows, useSettingsStore().editorSettings.csvQuoteMode, csvNullLiteralForMode(useSettingsStore().editorSettings.csvNullMode));
|
||||||
toast(t("grid.exported"));
|
toast(t("grid.exported"));
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
toast(t("grid.exportFailed", { message: translateBackendError(t, e) }), 5000);
|
toast(t("grid.exportFailed", { message: translateBackendError(t, e) }), 5000);
|
||||||
@@ -1389,6 +1390,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
|
|||||||
format,
|
format,
|
||||||
...(format === "sql" && sqlExportOptions ? { insertMode: sqlExportOptions.insertMode, splitMaxMb: sqlExportOptions.splitMaxMb, selectedColumns: sqlExportOptions.selectedColumns } : {}),
|
...(format === "sql" && sqlExportOptions ? { insertMode: sqlExportOptions.insertMode, splitMaxMb: sqlExportOptions.splitMaxMb, selectedColumns: sqlExportOptions.selectedColumns } : {}),
|
||||||
csvQuoteMode: editorSettings.csvQuoteMode,
|
csvQuoteMode: editorSettings.csvQuoteMode,
|
||||||
|
nullLiteral: csvNullLiteralForMode(editorSettings.csvNullMode),
|
||||||
columns: format === "sql" ? effectiveColumns(sourceColumns.value, columns.value).map((column, index) => column ?? columns.value[index]!) : columns.value,
|
columns: format === "sql" ? effectiveColumns(sourceColumns.value, columns.value).map((column, index) => column ?? columns.value[index]!) : columns.value,
|
||||||
columnTypes: columnTypes.value,
|
columnTypes: columnTypes.value,
|
||||||
...(format === "sql" ? { columnExtras: sqlExportColumnExtras(effectiveColumns(sourceColumns.value, columns.value).map((column, index) => column ?? columns.value[index]!)) } : {}),
|
...(format === "sql" ? { columnExtras: sqlExportColumnExtras(effectiveColumns(sourceColumns.value, columns.value).map((column, index) => column ?? columns.value[index]!)) } : {}),
|
||||||
@@ -1470,6 +1472,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
|
|||||||
...baseRequest,
|
...baseRequest,
|
||||||
...(format === "sql" ? { selectedColumns: sqlExportOptions?.selectedColumns } : {}),
|
...(format === "sql" ? { selectedColumns: sqlExportOptions?.selectedColumns } : {}),
|
||||||
csvQuoteMode: useSettingsStore().editorSettings.csvQuoteMode,
|
csvQuoteMode: useSettingsStore().editorSettings.csvQuoteMode,
|
||||||
|
nullLiteral: csvNullLiteralForMode(useSettingsStore().editorSettings.csvNullMode),
|
||||||
...sqlExportPrimaryKeyOptions(),
|
...sqlExportPrimaryKeyOptions(),
|
||||||
dateTimeFormat: useSettingsStore().editorSettings.globalDateTimeExportFormat || undefined,
|
dateTimeFormat: useSettingsStore().editorSettings.globalDateTimeExportFormat || undefined,
|
||||||
numericColumnRightAlign: useSettingsStore().editorSettings.numericColumnRightAlign ?? true,
|
numericColumnRightAlign: useSettingsStore().editorSettings.numericColumnRightAlign ?? true,
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ import {
|
|||||||
structurePreviewDdlStorageType,
|
structurePreviewDdlStorageType,
|
||||||
} from "@/components/sidebar/sidebarTreeDialogState";
|
} from "@/components/sidebar/sidebarTreeDialogState";
|
||||||
import type { CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
import type { CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
||||||
|
import { csvNullLiteralForMode } from "@/lib/export/csvNullMode";
|
||||||
import type { SqlInsertDialect } from "@/lib/export/sqlInsertMode";
|
import type { SqlInsertDialect } from "@/lib/export/sqlInsertMode";
|
||||||
import { uuid } from "@/lib/common/utils";
|
import { uuid } from "@/lib/common/utils";
|
||||||
|
|
||||||
@@ -59,6 +60,8 @@ interface SidebarTableExportTarget {
|
|||||||
batchSize: number;
|
batchSize: number;
|
||||||
rowLimit: number | null;
|
rowLimit: number | null;
|
||||||
csvQuoteMode: CsvQuoteMode;
|
csvQuoteMode: CsvQuoteMode;
|
||||||
|
/** 导出 CSV 时 NULL 的字面量;空串表示关闭(旧行为)。 */
|
||||||
|
nullLiteral: string;
|
||||||
fileNameBase?: string;
|
fileNameBase?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -437,6 +440,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
|||||||
batchSize: editorSettings.exportBatchSize,
|
batchSize: editorSettings.exportBatchSize,
|
||||||
rowLimit: editorSettings.exportRowLimitEnabled ? editorSettings.exportRowLimit : null,
|
rowLimit: editorSettings.exportRowLimitEnabled ? editorSettings.exportRowLimit : null,
|
||||||
csvQuoteMode: editorSettings.csvQuoteMode,
|
csvQuoteMode: editorSettings.csvQuoteMode,
|
||||||
|
nullLiteral: csvNullLiteralForMode(editorSettings.csvNullMode),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -472,7 +476,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
|||||||
executePage: (sql) => api.executeQuery(connectionId, database, sql),
|
executePage: (sql) => api.executeQuery(connectionId, database, sql),
|
||||||
});
|
});
|
||||||
if (format === "csv") {
|
if (format === "csv") {
|
||||||
await api.exportQueryResultCsv(outputPath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), target.csvQuoteMode);
|
await api.exportQueryResultCsv(outputPath, result.columns, forceCsvTextForTemporalColumns(result.rows, result.column_types ?? []), target.csvQuoteMode, target.nullLiteral);
|
||||||
} else {
|
} else {
|
||||||
const comments = result.columns.map((name) => exportColumnInfos?.find((column) => column.name.toLocaleLowerCase() === name.toLocaleLowerCase())?.comment);
|
const comments = result.columns.map((name) => exportColumnInfos?.find((column) => column.name.toLocaleLowerCase() === name.toLocaleLowerCase())?.comment);
|
||||||
const headerOverrides = buildXlsxHeaderOverrides(result.columns, comments, headerMode);
|
const headerOverrides = buildXlsxHeaderOverrides(result.columns, comments, headerMode);
|
||||||
@@ -503,6 +507,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
|||||||
format,
|
format,
|
||||||
...(format === "sql" ? { insertDialect } : {}),
|
...(format === "sql" ? { insertDialect } : {}),
|
||||||
csvQuoteMode: target.csvQuoteMode,
|
csvQuoteMode: target.csvQuoteMode,
|
||||||
|
nullLiteral: target.nullLiteral,
|
||||||
columns: queryColumns,
|
columns: queryColumns,
|
||||||
columnComments,
|
columnComments,
|
||||||
autoFilter: format === "xlsx" ? autoFilter : undefined,
|
autoFilter: format === "xlsx" ? autoFilter : undefined,
|
||||||
|
|||||||
@@ -6291,6 +6291,7 @@ export default withEnglishFallback({
|
|||||||
lastDataRow: "Son məlumat sətri (sona qədər üçün 0)",
|
lastDataRow: "Son məlumat sətri (sona qədər üçün 0)",
|
||||||
trimValues: "Dəyərlərin əvvəlindəki və sonundakı boşluqları sil",
|
trimValues: "Dəyərlərin əvvəlindəki və sonundakı boşluqları sil",
|
||||||
emptyStringAsNull: "Boş sətri NULL kimi qəbul et",
|
emptyStringAsNull: "Boş sətri NULL kimi qəbul et",
|
||||||
|
emptyStringAsNullHint: "İşarələndikdə boş sahələr NULL kimi idxal olunur və \\N adi mətn olur; əks halda boş sahələr boş sətir qalır və \\N NULL kimi idxal olunur.",
|
||||||
jsonShape: "JSON sətirləri",
|
jsonShape: "JSON sətirləri",
|
||||||
jsonShapeAuto: "Avtomatik müəyyən et",
|
jsonShapeAuto: "Avtomatik müəyyən et",
|
||||||
jsonShapeObjects: "Obyektlər",
|
jsonShapeObjects: "Obyektlər",
|
||||||
@@ -7822,6 +7823,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "CSV sahələrində qoşa dırnaqların necə istifadə olunacağını seçin.",
|
csvQuoteModeDescription: "CSV sahələrində qoşa dırnaqların necə istifadə olunacağını seçin.",
|
||||||
csvQuoteModeAll: "Bütün sahələr",
|
csvQuoteModeAll: "Bütün sahələr",
|
||||||
csvQuoteModeNecessary: "Yalnız lazım olduqda",
|
csvQuoteModeNecessary: "Yalnız lazım olduqda",
|
||||||
|
csvNullMode: "CSV NULL təsviri",
|
||||||
|
csvNullModeDescription: "CSV ixracında NULL-un yazılma üsulu. «\\N» NULL ilə boş sətri fərqləndirir, yenidən idxalda boş sətirlər NULL olmur; «Boş» köhnə davranışı saxlayır, lakin onları fərqləndirmir.",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Boş",
|
||||||
performanceSection: "Məhsuldarlıq",
|
performanceSection: "Məhsuldarlıq",
|
||||||
shortcutExecuteSql: "SQL-i icra et",
|
shortcutExecuteSql: "SQL-i icra et",
|
||||||
shortcutExecuteSqlInNewResultTab: "SQL-i yeni nəticə vərəqində icra et",
|
shortcutExecuteSqlInNewResultTab: "SQL-i yeni nəticə vərəqində icra et",
|
||||||
|
|||||||
@@ -6899,6 +6899,7 @@ export default {
|
|||||||
lastDataRow: "Last data row (0 for end)",
|
lastDataRow: "Last data row (0 for end)",
|
||||||
trimValues: "Trim values",
|
trimValues: "Trim values",
|
||||||
emptyStringAsNull: "Empty string as NULL",
|
emptyStringAsNull: "Empty string as NULL",
|
||||||
|
emptyStringAsNullHint: "When checked, empty fields import as NULL and \\N becomes plain text; otherwise empty fields stay as empty strings and \\N imports as NULL.",
|
||||||
conflictPolicy: "On primary-key conflict",
|
conflictPolicy: "On primary-key conflict",
|
||||||
conflictError: "Stop with an error",
|
conflictError: "Stop with an error",
|
||||||
skipDuplicateRows: "Skip duplicate rows",
|
skipDuplicateRows: "Skip duplicate rows",
|
||||||
@@ -8550,6 +8551,10 @@ export default {
|
|||||||
csvQuoteModeDescription: "Choose how CSV fields use double quotes.",
|
csvQuoteModeDescription: "Choose how CSV fields use double quotes.",
|
||||||
csvQuoteModeAll: "All fields",
|
csvQuoteModeAll: "All fields",
|
||||||
csvQuoteModeNecessary: "Only when needed",
|
csvQuoteModeNecessary: "Only when needed",
|
||||||
|
csvNullMode: "CSV NULL representation",
|
||||||
|
csvNullModeDescription: 'How NULL is written when exporting CSV. "\\N" keeps NULL distinguishable from an empty string, so re-importing does not turn empty strings into NULL; "Empty" matches older versions but leaves the two indistinguishable.',
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Empty",
|
||||||
performanceSection: "Performance",
|
performanceSection: "Performance",
|
||||||
shortcutExecuteSql: "Execute SQL",
|
shortcutExecuteSql: "Execute SQL",
|
||||||
shortcutExecuteSqlInNewResultTab: "Execute SQL in new result tab",
|
shortcutExecuteSqlInNewResultTab: "Execute SQL in new result tab",
|
||||||
|
|||||||
@@ -6326,6 +6326,7 @@ export default withEnglishFallback({
|
|||||||
hasHeader: "First row is header",
|
hasHeader: "First row is header",
|
||||||
trimValues: "Trim values",
|
trimValues: "Trim values",
|
||||||
emptyStringAsNull: "Empty string as NULL",
|
emptyStringAsNull: "Empty string as NULL",
|
||||||
|
emptyStringAsNullHint: "Si se activa, los campos vacíos se importan como NULL y \\N pasa a ser texto; si no, los campos vacíos se mantienen como cadenas vacías y \\N se importa como NULL.",
|
||||||
jsonShape: "JSON rows",
|
jsonShape: "JSON rows",
|
||||||
jsonShapeAuto: "Auto-detect",
|
jsonShapeAuto: "Auto-detect",
|
||||||
jsonShapeObjects: "Objects",
|
jsonShapeObjects: "Objects",
|
||||||
@@ -7853,6 +7854,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "Elige cómo usar comillas dobles en los campos CSV.",
|
csvQuoteModeDescription: "Elige cómo usar comillas dobles en los campos CSV.",
|
||||||
csvQuoteModeAll: "Todos los campos",
|
csvQuoteModeAll: "Todos los campos",
|
||||||
csvQuoteModeNecessary: "Solo cuando sea necesario",
|
csvQuoteModeNecessary: "Solo cuando sea necesario",
|
||||||
|
csvNullMode: "Representación de NULL en CSV",
|
||||||
|
csvNullModeDescription: "Cómo se escribe NULL al exportar CSV. «\\N» distingue NULL de la cadena vacía, de modo que al reimportar las cadenas vacías no se convierten en NULL; «Vacío» mantiene el comportamiento anterior, pero no los distingue.",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Vacío",
|
||||||
performanceSection: "Rendimiento",
|
performanceSection: "Rendimiento",
|
||||||
shortcutExecuteSql: "Ejecutar SQL",
|
shortcutExecuteSql: "Ejecutar SQL",
|
||||||
shortcutExecuteSqlInNewResultTab: "Ejecutar SQL en una nueva pestaña de resultados",
|
shortcutExecuteSqlInNewResultTab: "Ejecutar SQL en una nueva pestaña de resultados",
|
||||||
|
|||||||
@@ -6248,6 +6248,7 @@ export default withEnglishFallback({
|
|||||||
hasHeader: "First row is header",
|
hasHeader: "First row is header",
|
||||||
trimValues: "Trim values",
|
trimValues: "Trim values",
|
||||||
emptyStringAsNull: "Empty string as NULL",
|
emptyStringAsNull: "Empty string as NULL",
|
||||||
|
emptyStringAsNullHint: "Se attivo, i campi vuoti vengono importati come NULL e \\N diventa testo normale; altrimenti restano stringhe vuote e \\N viene importato come NULL.",
|
||||||
jsonShape: "JSON rows",
|
jsonShape: "JSON rows",
|
||||||
jsonShapeAuto: "Auto-detect",
|
jsonShapeAuto: "Auto-detect",
|
||||||
jsonShapeObjects: "Objects",
|
jsonShapeObjects: "Objects",
|
||||||
@@ -7777,6 +7778,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "Scegli come usare le virgolette doppie nei campi CSV.",
|
csvQuoteModeDescription: "Scegli come usare le virgolette doppie nei campi CSV.",
|
||||||
csvQuoteModeAll: "Tutti i campi",
|
csvQuoteModeAll: "Tutti i campi",
|
||||||
csvQuoteModeNecessary: "Solo quando necessario",
|
csvQuoteModeNecessary: "Solo quando necessario",
|
||||||
|
csvNullMode: "Rappresentazione di NULL nel CSV",
|
||||||
|
csvNullModeDescription: "Come viene scritto NULL durante l'esportazione CSV. «\\N» distingue NULL dalla stringa vuota, così al reimport le stringhe vuote non diventano NULL; «Vuoto» mantiene il comportamento precedente, ma non li distingue.",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Vuoto",
|
||||||
performanceSection: "Prestazioni",
|
performanceSection: "Prestazioni",
|
||||||
shortcutExecuteSql: "Esegui SQL",
|
shortcutExecuteSql: "Esegui SQL",
|
||||||
shortcutExecuteSqlInNewResultTab: "Esegui SQL in una nuova scheda dei risultati",
|
shortcutExecuteSqlInNewResultTab: "Esegui SQL in una nuova scheda dei risultati",
|
||||||
|
|||||||
@@ -6267,6 +6267,7 @@ export default withEnglishFallback({
|
|||||||
hasHeader: "First row is header",
|
hasHeader: "First row is header",
|
||||||
trimValues: "Trim values",
|
trimValues: "Trim values",
|
||||||
emptyStringAsNull: "Empty string as NULL",
|
emptyStringAsNull: "Empty string as NULL",
|
||||||
|
emptyStringAsNullHint: "チェックすると空フィールドを NULL として取り込み、\\N は通常のテキストになります。チェックしない場合は空フィールドは空文字列のままとなり、\\N を NULL として取り込みます。",
|
||||||
jsonShape: "JSON rows",
|
jsonShape: "JSON rows",
|
||||||
jsonShapeAuto: "Auto-detect",
|
jsonShapeAuto: "Auto-detect",
|
||||||
jsonShapeObjects: "Objects",
|
jsonShapeObjects: "Objects",
|
||||||
@@ -7776,6 +7777,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "CSV フィールドを二重引用符で囲む方法を選択します。",
|
csvQuoteModeDescription: "CSV フィールドを二重引用符で囲む方法を選択します。",
|
||||||
csvQuoteModeAll: "すべてのフィールド",
|
csvQuoteModeAll: "すべてのフィールド",
|
||||||
csvQuoteModeNecessary: "必要な場合のみ",
|
csvQuoteModeNecessary: "必要な場合のみ",
|
||||||
|
csvNullMode: "CSV の NULL 表記",
|
||||||
|
csvNullModeDescription: "CSV エクスポート時に NULL を書く方式です。「\\N」は NULL と空文字列を区別でき、再インポートしても空文字列が NULL になりません。「空」は以前の動作と同じですが区別できません。",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "空",
|
||||||
performanceSection: "パフォーマンス",
|
performanceSection: "パフォーマンス",
|
||||||
shortcutExecuteSql: "SQLを実行",
|
shortcutExecuteSql: "SQLを実行",
|
||||||
shortcutExecuteSqlInNewResultTab: "新しい結果タブでSQLを実行",
|
shortcutExecuteSqlInNewResultTab: "新しい結果タブでSQLを実行",
|
||||||
|
|||||||
@@ -5981,6 +5981,7 @@ export default withEnglishFallback({
|
|||||||
lastDataRow: "마지막 데이터 행 (끝까지면 0)",
|
lastDataRow: "마지막 데이터 행 (끝까지면 0)",
|
||||||
trimValues: "값 양끝 공백 제거",
|
trimValues: "값 양끝 공백 제거",
|
||||||
emptyStringAsNull: "빈 문자열을 NULL로",
|
emptyStringAsNull: "빈 문자열을 NULL로",
|
||||||
|
emptyStringAsNullHint: "체크하면 빈 필드를 NULL로 가져오고 \\N은 일반 텍스트가 됩니다. 해제하면 빈 필드는 빈 문자열로 유지되고 \\N은 NULL로 가져옵니다.",
|
||||||
jsonShape: "JSON 행",
|
jsonShape: "JSON 행",
|
||||||
jsonShapeAuto: "자동 감지",
|
jsonShapeAuto: "자동 감지",
|
||||||
jsonShapeObjects: "객체",
|
jsonShapeObjects: "객체",
|
||||||
@@ -8083,6 +8084,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "CSV 내보내기 시 필드에 큰따옴표를 사용하는 방식을 선택합니다.",
|
csvQuoteModeDescription: "CSV 내보내기 시 필드에 큰따옴표를 사용하는 방식을 선택합니다.",
|
||||||
csvQuoteModeAll: "모든 필드",
|
csvQuoteModeAll: "모든 필드",
|
||||||
csvQuoteModeNecessary: "필수 필드만",
|
csvQuoteModeNecessary: "필수 필드만",
|
||||||
|
csvNullMode: "CSV NULL 표기",
|
||||||
|
csvNullModeDescription: 'CSV 내보내기 시 NULL을 쓰는 방식입니다. "\\N"은 NULL과 빈 문자열을 구분하므로 다시 가져올 때 빈 문자열이 NULL이 되지 않습니다. "비움"은 이전 동작과 같지만 구분할 수 없습니다.',
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "비움",
|
||||||
shortcutConvertNamingStyle: "네이밍 스타일 전환",
|
shortcutConvertNamingStyle: "네이밍 스타일 전환",
|
||||||
dataGridKeepFilterEditorExpanded: "필터 편집기 기본 펼치기",
|
dataGridKeepFilterEditorExpanded: "필터 편집기 기본 펼치기",
|
||||||
dataGridKeepFilterEditorExpandedDescription: "조건 및 텍스트 보기에서 편집기를 처음에 펼칩니다. 수동으로 접을 수도 있습니다.",
|
dataGridKeepFilterEditorExpandedDescription: "조건 및 텍스트 보기에서 편집기를 처음에 펼칩니다. 수동으로 접을 수도 있습니다.",
|
||||||
|
|||||||
@@ -6250,6 +6250,7 @@ export default withEnglishFallback({
|
|||||||
hasHeader: "First row is header",
|
hasHeader: "First row is header",
|
||||||
trimValues: "Trim values",
|
trimValues: "Trim values",
|
||||||
emptyStringAsNull: "Empty string as NULL",
|
emptyStringAsNull: "Empty string as NULL",
|
||||||
|
emptyStringAsNullHint: "Quando marcado, campos vazios são importados como NULL e \\N passa a ser texto; caso contrário, permanecem como strings vazias e \\N é importado como NULL.",
|
||||||
jsonShape: "JSON rows",
|
jsonShape: "JSON rows",
|
||||||
jsonShapeAuto: "Auto-detect",
|
jsonShapeAuto: "Auto-detect",
|
||||||
jsonShapeObjects: "Objects",
|
jsonShapeObjects: "Objects",
|
||||||
@@ -7779,6 +7780,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "Escolha como usar aspas duplas nos campos CSV.",
|
csvQuoteModeDescription: "Escolha como usar aspas duplas nos campos CSV.",
|
||||||
csvQuoteModeAll: "Todos os campos",
|
csvQuoteModeAll: "Todos os campos",
|
||||||
csvQuoteModeNecessary: "Somente quando necessário",
|
csvQuoteModeNecessary: "Somente quando necessário",
|
||||||
|
csvNullMode: "Representação de NULL no CSV",
|
||||||
|
csvNullModeDescription: "Como NULL é escrito ao exportar CSV. «\\N» distingue NULL da string vazia, então ao reimportar as strings vazias não viram NULL; «Vazio» mantém o comportamento antigo, mas não os distingue.",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Vazio",
|
||||||
performanceSection: "Desempenho",
|
performanceSection: "Desempenho",
|
||||||
shortcutExecuteSql: "Executar SQL",
|
shortcutExecuteSql: "Executar SQL",
|
||||||
shortcutExecuteSqlInNewResultTab: "Executar SQL em uma nova aba de resultados",
|
shortcutExecuteSqlInNewResultTab: "Executar SQL em uma nova aba de resultados",
|
||||||
|
|||||||
@@ -7418,6 +7418,7 @@ export default withEnglishFallback({
|
|||||||
lastDataRow: "Последняя строка данных (0 — до конца)",
|
lastDataRow: "Последняя строка данных (0 — до конца)",
|
||||||
trimValues: "Обрезать значения",
|
trimValues: "Обрезать значения",
|
||||||
emptyStringAsNull: "Пустая строка как NULL",
|
emptyStringAsNull: "Пустая строка как NULL",
|
||||||
|
emptyStringAsNullHint: "Если включено, пустые поля импортируются как NULL, а \\N считается обычным текстом; иначе пустые поля остаются пустыми строками, а \\N импортируется как NULL.",
|
||||||
jsonShape: "Строки JSON",
|
jsonShape: "Строки JSON",
|
||||||
jsonShapeAuto: "Автоопределение",
|
jsonShapeAuto: "Автоопределение",
|
||||||
jsonShapeObjects: "Объекты",
|
jsonShapeObjects: "Объекты",
|
||||||
@@ -9049,6 +9050,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "Выберите, как поля CSV используют двойные кавычки.",
|
csvQuoteModeDescription: "Выберите, как поля CSV используют двойные кавычки.",
|
||||||
csvQuoteModeAll: "Все поля",
|
csvQuoteModeAll: "Все поля",
|
||||||
csvQuoteModeNecessary: "Только когда нужно",
|
csvQuoteModeNecessary: "Только когда нужно",
|
||||||
|
csvNullMode: "Представление NULL в CSV",
|
||||||
|
csvNullModeDescription: "Как записывается NULL при экспорте CSV. «\\N» отличает NULL от пустой строки, поэтому при повторном импорте пустые строки не станут NULL; «Пусто» сохраняет прежнее поведение, но не различает их.",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Пусто",
|
||||||
performanceSection: "Производительность",
|
performanceSection: "Производительность",
|
||||||
shortcutExecuteSql: "Выполнить SQL",
|
shortcutExecuteSql: "Выполнить SQL",
|
||||||
shortcutExecuteSqlInNewResultTab: "Выполнить SQL в новой вкладке результатов",
|
shortcutExecuteSqlInNewResultTab: "Выполнить SQL в новой вкладке результатов",
|
||||||
|
|||||||
@@ -6173,6 +6173,7 @@ export default withEnglishFallback({
|
|||||||
lastDataRow: "Son veri satırı (sona kadar için 0)",
|
lastDataRow: "Son veri satırı (sona kadar için 0)",
|
||||||
trimValues: "Değerleri kırp",
|
trimValues: "Değerleri kırp",
|
||||||
emptyStringAsNull: "Boş dizeyi NULL say",
|
emptyStringAsNull: "Boş dizeyi NULL say",
|
||||||
|
emptyStringAsNullHint: "İşaretlendiğinde boş alanlar NULL olarak içe aktarılır ve \\N düz metin olur; aksi halde boş alanlar boş dize kalır ve \\N NULL olarak içe aktarılır.",
|
||||||
jsonShape: "JSON satırları",
|
jsonShape: "JSON satırları",
|
||||||
jsonShapeAuto: "Otomatik algıla",
|
jsonShapeAuto: "Otomatik algıla",
|
||||||
jsonShapeObjects: "Nesneler",
|
jsonShapeObjects: "Nesneler",
|
||||||
@@ -7721,6 +7722,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "CSV alanlarının çift tırnağı nasıl kullanacağını seçin.",
|
csvQuoteModeDescription: "CSV alanlarının çift tırnağı nasıl kullanacağını seçin.",
|
||||||
csvQuoteModeAll: "Tüm alanlar",
|
csvQuoteModeAll: "Tüm alanlar",
|
||||||
csvQuoteModeNecessary: "Yalnızca gerektiğinde",
|
csvQuoteModeNecessary: "Yalnızca gerektiğinde",
|
||||||
|
csvNullMode: "CSV NULL gösterimi",
|
||||||
|
csvNullModeDescription: "CSV dışa aktarılırken NULL nasıl yazılır. «\\N» NULL ile boş dizeyi ayırır, böylece yeniden içe aktarımda boş dizeler NULL olmaz; «Boş» eski davranışı korur ancak ikisini ayırmaz.",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "Boş",
|
||||||
performanceSection: "Başarım",
|
performanceSection: "Başarım",
|
||||||
shortcutExecuteSql: "SQL'i çalıştır",
|
shortcutExecuteSql: "SQL'i çalıştır",
|
||||||
shortcutExecuteSqlInNewResultTab: "SQL'i yeni sonuç sekmesinde çalıştır",
|
shortcutExecuteSqlInNewResultTab: "SQL'i yeni sonuç sekmesinde çalıştır",
|
||||||
|
|||||||
@@ -6866,6 +6866,7 @@ export default withEnglishFallback({
|
|||||||
lastDataRow: "最后数据行(0 表示末尾)",
|
lastDataRow: "最后数据行(0 表示末尾)",
|
||||||
trimValues: "裁剪空白",
|
trimValues: "裁剪空白",
|
||||||
emptyStringAsNull: "空字符串作为 NULL",
|
emptyStringAsNull: "空字符串作为 NULL",
|
||||||
|
emptyStringAsNullHint: "勾选后空字段一律按 NULL 导入(此时文件里的 \\N 按普通文本导入);不勾选则空字段保留为空字符串,并把 \\N 按 NULL 导入。",
|
||||||
conflictPolicy: "主键冲突时",
|
conflictPolicy: "主键冲突时",
|
||||||
conflictError: "停止并报错",
|
conflictError: "停止并报错",
|
||||||
skipDuplicateRows: "跳过重复行",
|
skipDuplicateRows: "跳过重复行",
|
||||||
@@ -8513,6 +8514,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "选择 CSV 导出时字段使用双引号的方式。",
|
csvQuoteModeDescription: "选择 CSV 导出时字段使用双引号的方式。",
|
||||||
csvQuoteModeAll: "全部字段",
|
csvQuoteModeAll: "全部字段",
|
||||||
csvQuoteModeNecessary: "仅必要字段",
|
csvQuoteModeNecessary: "仅必要字段",
|
||||||
|
csvNullMode: "CSV NULL 表示",
|
||||||
|
csvNullModeDescription: "导出 CSV 时 NULL 的写法。选「\\N」可区分 NULL 与空字符串,重新导入时不会把空字符串写成 NULL;选「留空」与旧版本一致,但两者在文件里无法区分。",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "留空",
|
||||||
performanceSection: "性能",
|
performanceSection: "性能",
|
||||||
shortcutExecuteSql: "执行 SQL",
|
shortcutExecuteSql: "执行 SQL",
|
||||||
shortcutExecuteSqlInNewResultTab: "在新结果标签页中执行 SQL",
|
shortcutExecuteSqlInNewResultTab: "在新结果标签页中执行 SQL",
|
||||||
|
|||||||
@@ -5557,6 +5557,7 @@ export default withEnglishFallback({
|
|||||||
hasHeader: "首列為標頭",
|
hasHeader: "首列為標頭",
|
||||||
trimValues: "裁剪空白",
|
trimValues: "裁剪空白",
|
||||||
emptyStringAsNull: "空字串作為 NULL",
|
emptyStringAsNull: "空字串作為 NULL",
|
||||||
|
emptyStringAsNullHint: "勾選後空欄位一律以 NULL 匯入(此時檔案裡的 \\N 以一般文字匯入);不勾選則空欄位保留為空字串,並把 \\N 以 NULL 匯入。",
|
||||||
conflictPolicy: "主鍵衝突時",
|
conflictPolicy: "主鍵衝突時",
|
||||||
conflictError: "停止並回報錯誤",
|
conflictError: "停止並回報錯誤",
|
||||||
skipDuplicateRows: "略過重複列",
|
skipDuplicateRows: "略過重複列",
|
||||||
@@ -7610,6 +7611,10 @@ export default withEnglishFallback({
|
|||||||
csvQuoteModeDescription: "選擇 CSV 匯出時欄位使用雙引號的方式。",
|
csvQuoteModeDescription: "選擇 CSV 匯出時欄位使用雙引號的方式。",
|
||||||
csvQuoteModeAll: "全部欄位",
|
csvQuoteModeAll: "全部欄位",
|
||||||
csvQuoteModeNecessary: "僅必要欄位",
|
csvQuoteModeNecessary: "僅必要欄位",
|
||||||
|
csvNullMode: "CSV NULL 表示",
|
||||||
|
csvNullModeDescription: "匯出 CSV 時 NULL 的寫法。選「\\N」可區分 NULL 與空字串,重新匯入時不會把空字串寫成 NULL;選「留空」與舊版本一致,但兩者在檔案裡無法區分。",
|
||||||
|
csvNullModeMarker: "\\N",
|
||||||
|
csvNullModeEmpty: "留空",
|
||||||
performanceSection: "效能",
|
performanceSection: "效能",
|
||||||
exportBatchSize: "匯出批次大小",
|
exportBatchSize: "匯出批次大小",
|
||||||
dateTimeSection: "日期與時間",
|
dateTimeSection: "日期與時間",
|
||||||
|
|||||||
+27
-5
@@ -15,12 +15,34 @@ import {
|
|||||||
} from "@/lib/table/tableImport";
|
} from "@/lib/table/tableImport";
|
||||||
|
|
||||||
describe("tableImport", () => {
|
describe("tableImport", () => {
|
||||||
it("preserves explicit empty strings by default for Excel only", () => {
|
it("keeps empty fields as empty strings for every source format by default", () => {
|
||||||
|
// 导出端默认用 `\N` 表示 NULL,空字段代表空字符串;导入端默认必须与之一致,
|
||||||
|
// 否则 DBX 导出的 CSV 再导回时会把 '' 变成 NULL。
|
||||||
expect(defaultTableImportEmptyStringAsNull("excel")).toBe(false);
|
expect(defaultTableImportEmptyStringAsNull("excel")).toBe(false);
|
||||||
expect(defaultTableImportEmptyStringAsNull("csv")).toBe(true);
|
expect(defaultTableImportEmptyStringAsNull("csv")).toBe(false);
|
||||||
expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(true);
|
expect(defaultTableImportEmptyStringAsNull("tsv")).toBe(false);
|
||||||
expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(true);
|
expect(defaultTableImportEmptyStringAsNull("delimited")).toBe(false);
|
||||||
expect(defaultTableImportEmptyStringAsNull("json")).toBe(true);
|
expect(defaultTableImportEmptyStringAsNull("json")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("only disables the shared NULL literal when empty fields are treated as NULL", () => {
|
||||||
|
const baseSettings = {
|
||||||
|
delimiter: ",",
|
||||||
|
textEncoding: "auto" as const,
|
||||||
|
titleRow: 1,
|
||||||
|
dataStartRow: 2,
|
||||||
|
lastDataRow: 0,
|
||||||
|
trimValues: false,
|
||||||
|
jsonShape: "auto" as const,
|
||||||
|
};
|
||||||
|
|
||||||
|
const sharedLiteral = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: false });
|
||||||
|
expect(sharedLiteral.nullLiteral).toBeUndefined();
|
||||||
|
expect(sharedLiteral.emptyStringAsNull).toBe(false);
|
||||||
|
|
||||||
|
const legacyEmptyField = buildTableImportParseOptions({ ...baseSettings, format: "csv", emptyStringAsNull: true });
|
||||||
|
expect(legacyEmptyField.nullLiteral).toBe("");
|
||||||
|
expect(legacyEmptyField.emptyStringAsNull).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("formats import elapsed time for progress and terminal summaries", () => {
|
it("formats import elapsed time for progress and terminal summaries", () => {
|
||||||
|
|||||||
@@ -3388,9 +3388,9 @@ export async function cancelQueryResultExport(exportId: string, executionId?: st
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise<void> {
|
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise<void> {
|
||||||
const { formatCsv } = await import("@/lib/export/exportFormats");
|
const { formatCsv } = await import("@/lib/export/exportFormats");
|
||||||
const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode);
|
const content = formatCsv(columns, rows as (string | number | boolean | null)[][], csvQuoteMode, nullLiteral);
|
||||||
const fileName = filePath.split(/[\\/]/).pop() || "export.csv";
|
const fileName = filePath.split(/[\\/]/).pop() || "export.csv";
|
||||||
const blob = new Blob(["\uFEFF", content], {
|
const blob = new Blob(["\uFEFF", content], {
|
||||||
type: "text/csv;charset=utf-8",
|
type: "text/csv;charset=utf-8",
|
||||||
|
|||||||
@@ -5466,6 +5466,8 @@ export interface TableImportParseOptions {
|
|||||||
lastDataRow?: number | null;
|
lastDataRow?: number | null;
|
||||||
trimValues?: boolean | null;
|
trimValues?: boolean | null;
|
||||||
emptyStringAsNull?: boolean | null;
|
emptyStringAsNull?: boolean | null;
|
||||||
|
/** 分隔文本里代表 NULL 的字面量。缺省表示用后端默认值 `\N`;空串表示关闭字面量,退回「空字段即 NULL」。 */
|
||||||
|
nullLiteral?: string | null;
|
||||||
sheetName?: string | null;
|
sheetName?: string | null;
|
||||||
sheetIndex?: number | null;
|
sheetIndex?: number | null;
|
||||||
jsonShape?: TableImportJsonShape | null;
|
jsonShape?: TableImportJsonShape | null;
|
||||||
@@ -5861,6 +5863,8 @@ export interface TableExportRequest {
|
|||||||
insertMode?: SqlInsertMode;
|
insertMode?: SqlInsertMode;
|
||||||
insertDialect?: SqlInsertDialect;
|
insertDialect?: SqlInsertDialect;
|
||||||
csvQuoteMode?: CsvQuoteMode;
|
csvQuoteMode?: CsvQuoteMode;
|
||||||
|
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
|
||||||
|
nullLiteral?: string;
|
||||||
columns?: string[];
|
columns?: string[];
|
||||||
selectedColumns?: SqlExportColumnSelection[];
|
selectedColumns?: SqlExportColumnSelection[];
|
||||||
columnTypes?: Array<string | null | undefined>;
|
columnTypes?: Array<string | null | undefined>;
|
||||||
@@ -5891,6 +5895,8 @@ export interface TableCsvExportOptions {
|
|||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
timeoutSecs?: number;
|
timeoutSecs?: number;
|
||||||
csvQuoteMode?: CsvQuoteMode;
|
csvQuoteMode?: CsvQuoteMode;
|
||||||
|
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
|
||||||
|
nullLiteral?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TableExportProgress {
|
export interface TableExportProgress {
|
||||||
@@ -5917,6 +5923,8 @@ export interface QueryResultExportRequest {
|
|||||||
format: "csv" | "xlsx" | "json" | "txt" | "sql";
|
format: "csv" | "xlsx" | "json" | "txt" | "sql";
|
||||||
insertMode?: SqlInsertMode;
|
insertMode?: SqlInsertMode;
|
||||||
csvQuoteMode?: CsvQuoteMode;
|
csvQuoteMode?: CsvQuoteMode;
|
||||||
|
/** CSV 里 NULL 写成什么。缺省表示用后端默认值 `\N`;空串表示关闭字面量。 */
|
||||||
|
nullLiteral?: string;
|
||||||
includeSqlSheet?: boolean;
|
includeSqlSheet?: boolean;
|
||||||
pageSize: number;
|
pageSize: number;
|
||||||
rowLimit?: number | null;
|
rowLimit?: number | null;
|
||||||
@@ -6074,13 +6082,14 @@ export async function recordDatabaseExportDestination(directory: string): Promis
|
|||||||
await invoke("record_database_export_destination", { directory });
|
await invoke("record_database_export_destination", { directory });
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all"): Promise<void> {
|
export async function exportQueryResultCsv(filePath: string, columns: string[], rows: readonly (readonly XlsxCellValue[])[], csvQuoteMode: CsvQuoteMode = "all", nullLiteral?: string): Promise<void> {
|
||||||
return invoke("export_query_result_csv", {
|
return invoke("export_query_result_csv", {
|
||||||
request: {
|
request: {
|
||||||
filePath,
|
filePath,
|
||||||
columns,
|
columns,
|
||||||
rows,
|
rows,
|
||||||
csvQuoteMode,
|
csvQuoteMode,
|
||||||
|
...(nullLiteral === undefined ? {} : { nullLiteral }),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { csvNullLiteralForMode, DEFAULT_CSV_NULL_LITERAL, DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode } from "@/lib/export/csvNullMode";
|
||||||
|
|
||||||
|
describe("normalizeCsvNullMode", () => {
|
||||||
|
it("defaults to writing the NULL marker", () => {
|
||||||
|
expect(DEFAULT_CSV_NULL_MODE).toBe("marker");
|
||||||
|
expect(normalizeCsvNullMode(undefined)).toBe("marker");
|
||||||
|
expect(normalizeCsvNullMode(null)).toBe("marker");
|
||||||
|
expect(normalizeCsvNullMode("marker")).toBe("marker");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps an explicit request for the legacy empty field", () => {
|
||||||
|
expect(normalizeCsvNullMode("empty")).toBe("empty");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to the default for unknown values", () => {
|
||||||
|
expect(normalizeCsvNullMode("")).toBe("marker");
|
||||||
|
expect(normalizeCsvNullMode("\\N")).toBe("marker");
|
||||||
|
expect(normalizeCsvNullMode(0)).toBe("marker");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("csvNullLiteralForMode", () => {
|
||||||
|
it("writes the MySQL-style NULL marker by default", () => {
|
||||||
|
expect(csvNullLiteralForMode("marker")).toBe(DEFAULT_CSV_NULL_LITERAL);
|
||||||
|
expect(csvNullLiteralForMode("marker")).toBe("\\N");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("disables the literal with an empty string to restore the old export", () => {
|
||||||
|
expect(csvNullLiteralForMode("empty")).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
+68
-3
@@ -1,5 +1,58 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { formatTsv } from "@/lib/export/exportFormats";
|
import { formatCsv, formatTsv } from "@/lib/export/exportFormats";
|
||||||
|
|
||||||
|
describe("formatCsv NULL handling", () => {
|
||||||
|
it("writes NULL as an unquoted marker and an empty string as an empty quoted field", () => {
|
||||||
|
// 即使 quote mode = all,NULL 字面量也必须裸写:PostgreSQL/ClickHouse 的 CSV 读取
|
||||||
|
// 不把带引号的 `"\N"` 当作 NULL,裸 `\N` 才是它们和 MySQL 共同识别的写法。
|
||||||
|
expect(
|
||||||
|
formatCsv(
|
||||||
|
["id", "name"],
|
||||||
|
[
|
||||||
|
[1, ""],
|
||||||
|
[2, null],
|
||||||
|
],
|
||||||
|
"all",
|
||||||
|
"\\N",
|
||||||
|
),
|
||||||
|
).toBe('"id","name"\n"1",""\n"2",\\N');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the marker unquoted when only necessary fields are quoted", () => {
|
||||||
|
expect(
|
||||||
|
formatCsv(
|
||||||
|
["id", "name"],
|
||||||
|
[
|
||||||
|
[1, ""],
|
||||||
|
[2, null],
|
||||||
|
],
|
||||||
|
"necessary",
|
||||||
|
"\\N",
|
||||||
|
),
|
||||||
|
).toBe("id,name\n1,\n2,\\N");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("quotes a marker that itself needs quoting", () => {
|
||||||
|
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
|
||||||
|
expect(formatCsv(["id", "name"], [[1, null]], "all", "a,b")).toBe('"id","name"\n"1","a,b"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restores the legacy empty field when the marker is disabled", () => {
|
||||||
|
// 关闭字面量时 NULL 是完全空字段、空字符串是 `""`:CSV 引号语义上仍有区别,
|
||||||
|
// 但 DBX 的导入端比较的是去引号后的值,因此旧格式在往返时依赖不了这个区别。
|
||||||
|
expect(
|
||||||
|
formatCsv(
|
||||||
|
["id", "name"],
|
||||||
|
[
|
||||||
|
[1, ""],
|
||||||
|
[2, null],
|
||||||
|
],
|
||||||
|
"all",
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
).toBe('"id","name"\n"1",""\n"2",');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("formatTsv", () => {
|
describe("formatTsv", () => {
|
||||||
it("copies a value that only contains double quotes verbatim (#10087)", () => {
|
it("copies a value that only contains double quotes verbatim (#10087)", () => {
|
||||||
@@ -14,8 +67,20 @@ describe("formatTsv", () => {
|
|||||||
expect(output).toBe('v\n"a\tb"\n"c\nd"');
|
expect(output).toBe('v\n"a\tb"\n"c\nd"');
|
||||||
});
|
});
|
||||||
|
|
||||||
it("renders NULL as an empty field without adding quotes", () => {
|
it("renders NULL as the null literal and an empty string as an empty field", () => {
|
||||||
const output = formatTsv(["id", "name"], [[1, null]]);
|
const output = formatTsv(
|
||||||
|
["id", "name"],
|
||||||
|
[
|
||||||
|
[1, null],
|
||||||
|
[2, ""],
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(output).toBe("id\tname\n1\t\\N\n2\t");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to an empty NULL field when the literal is disabled", () => {
|
||||||
|
const output = formatTsv(["id", "name"], [[1, null]], "");
|
||||||
|
|
||||||
expect(output).toBe("id\tname\n1\t");
|
expect(output).toBe("id\tname\n1\t");
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
/**
|
||||||
|
* 导出 CSV 时 NULL 的写法。与 Rust `dbx_formats::csv_export` 保持一致:
|
||||||
|
* `marker` 把 NULL 写成 `\N`(MySQL `SELECT ... INTO OUTFILE`、ClickHouse
|
||||||
|
* `format_csv_null_representation` 的既定写法),空字符串才写成空字段,两者在文件里可区分,
|
||||||
|
* 导回 NOT NULL DEFAULT '' 的列时不会被还原成 NULL;`empty` 是旧行为:NULL 与空字符串都写成空字段。
|
||||||
|
*/
|
||||||
|
export const DEFAULT_CSV_NULL_LITERAL = "\\N";
|
||||||
|
|
||||||
|
export const CSV_NULL_MODE_MARKER = "marker";
|
||||||
|
export const CSV_NULL_MODE_EMPTY = "empty";
|
||||||
|
|
||||||
|
export type CsvNullMode = typeof CSV_NULL_MODE_MARKER | typeof CSV_NULL_MODE_EMPTY;
|
||||||
|
|
||||||
|
export const DEFAULT_CSV_NULL_MODE: CsvNullMode = CSV_NULL_MODE_MARKER;
|
||||||
|
|
||||||
|
export function normalizeCsvNullMode(value: unknown): CsvNullMode {
|
||||||
|
return value === CSV_NULL_MODE_EMPTY ? CSV_NULL_MODE_EMPTY : DEFAULT_CSV_NULL_MODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 导出请求里的 NULL 字面量;空串表示关闭该字面量(旧行为)。 */
|
||||||
|
export function csvNullLiteralForMode(mode: CsvNullMode): string {
|
||||||
|
return mode === CSV_NULL_MODE_EMPTY ? "" : DEFAULT_CSV_NULL_LITERAL;
|
||||||
|
}
|
||||||
@@ -1,26 +1,35 @@
|
|||||||
import type { DatabaseType, QueryResult } from "@/types/database";
|
import type { DatabaseType, QueryResult } from "@/types/database";
|
||||||
import * as api from "@/lib/backend/api";
|
import * as api from "@/lib/backend/api";
|
||||||
import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
import { escapeCsvField, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
||||||
|
import { DEFAULT_CSV_NULL_LITERAL } from "@/lib/export/csvNullMode";
|
||||||
import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||||
|
|
||||||
export type ExportCellValue = string | number | boolean | null;
|
export type ExportCellValue = string | number | boolean | null;
|
||||||
|
|
||||||
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all"): string {
|
export function formatCsv(columns: string[], rows: ExportCellValue[][], quoteMode: CsvQuoteMode = "all", nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
|
||||||
const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(",");
|
const header = columns.map((column) => escapeCsvField(column, quoteMode)).join(",");
|
||||||
const body = rows.map((row) => row.map((cell) => (cell === null ? "" : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
|
// NULL 写成独立字面量、空字符串写成空字段,否则导入端无法把两者区分开
|
||||||
|
// (写回 NOT NULL DEFAULT '' 的列时会把空字符串还原成 NULL)。
|
||||||
|
//
|
||||||
|
// 字面量裸写:PostgreSQL 的 COPY ... FORMAT csv 不认带引号的 NULL(`"\N"` 是普通字符串,
|
||||||
|
// 整数列直接报错),ClickHouse 的 CSV 读取在引号内遇到 `\N` 会解析失败;只有裸 `\N` 是
|
||||||
|
// 它们与 MySQL 共同识别的 NULL。字面量自身含分隔符/引号/换行时才加引号。
|
||||||
|
const nullField = nullLiteral ? escapeCsvField(nullLiteral, "necessary") : "";
|
||||||
|
const body = rows.map((row) => row.map((cell) => (cell === null ? nullField : escapeCsvField(String(cell), quoteMode))).join(",")).join("\n");
|
||||||
return `${header}\n${body}`;
|
return `${header}\n${body}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)"
|
// Tab-separated values with a header row, mirroring Navicat's "Text File (*.txt)"
|
||||||
// export: fields are joined by a tab and NULL becomes empty. A field is wrapped
|
// export: fields are joined by a tab and NULL becomes the null literal. A field
|
||||||
// in double quotes only when it contains a tab or a line break - the characters
|
// is wrapped in double quotes only when it contains a tab or a line break - the
|
||||||
// that would otherwise corrupt the TSV row/column shape. A field that merely
|
// characters that would otherwise corrupt the TSV row/column shape. A field that
|
||||||
// contains a double quote is emitted verbatim: TSV is parsed by splitting on the
|
// merely contains a double quote is emitted verbatim: TSV is parsed by splitting
|
||||||
// tab alone (there is no quote state machine on paste-back), so quoting such a
|
// on the tab alone (there is no quote state machine on paste-back), so quoting
|
||||||
// value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
|
// such a value would corrupt it (e.g. a value of `"abc"` must not become `"""abc"""`).
|
||||||
export function formatTsv(columns: string[], rows: ExportCellValue[][]): string {
|
export function formatTsv(columns: string[], rows: ExportCellValue[][], nullLiteral: string = DEFAULT_CSV_NULL_LITERAL): string {
|
||||||
const esc = (value: ExportCellValue) => {
|
const esc = (value: ExportCellValue) => {
|
||||||
const text = value === null ? "" : String(value);
|
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返
|
||||||
|
const text = value === null ? nullLiteral || "" : String(value);
|
||||||
if (text.includes("\t") || text.includes("\n") || text.includes("\r")) {
|
if (text.includes("\t") || text.includes("\n") || text.includes("\r")) {
|
||||||
return `"${text.replace(/"/g, '""')}"`;
|
return `"${text.replace(/"/g, '""')}"`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -236,6 +236,11 @@ describe("editorSettingsDraftFromSettings", () => {
|
|||||||
expect(draft.completionTriggerMode).toBe("positional");
|
expect(draft.completionTriggerMode).toBe("positional");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("maps csvNullMode and normalizes an unknown value back to the marker", () => {
|
||||||
|
expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "empty" })).csvNullMode).toBe("empty");
|
||||||
|
expect(editorSettingsDraftFromSettings(makeSettings({ csvNullMode: "bogus" as unknown } as Partial<EditorSettings>)).csvNullMode).toBe("marker");
|
||||||
|
});
|
||||||
|
|
||||||
it("maps and normalizes tableCompletionSchemaQualification", () => {
|
it("maps and normalizes tableCompletionSchemaQualification", () => {
|
||||||
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "always" })).tableCompletionSchemaQualification).toBe("always");
|
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "always" })).tableCompletionSchemaQualification).toBe("always");
|
||||||
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "invalid" } as any)).tableCompletionSchemaQualification).toBe("collision");
|
expect(editorSettingsDraftFromSettings(makeSettings({ tableCompletionSchemaQualification: "invalid" } as any)).tableCompletionSchemaQualification).toBe("collision");
|
||||||
|
|||||||
@@ -413,4 +413,9 @@ describe("settingsTransfer", () => {
|
|||||||
expect(transferCategoryForKey("csvQuoteMode")).toBe("data");
|
expect(transferCategoryForKey("csvQuoteMode")).toBe("data");
|
||||||
expect(collectTransferCategories(["csvQuoteMode", "pageSize"])).toEqual(["data"]);
|
expect(collectTransferCategories(["csvQuoteMode", "pageSize"])).toEqual(["data"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("maps csvNullMode into the data category", () => {
|
||||||
|
expect(transferCategoryForKey("csvNullMode")).toBe("data");
|
||||||
|
expect(collectTransferCategories(["csvNullMode", "csvQuoteMode"])).toEqual(["data"]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -7,6 +7,7 @@ import { normalizeCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPo
|
|||||||
import { normalizeSqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification";
|
import { normalizeSqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification";
|
||||||
import { normalizeTableHoverLookupMode } from "@/lib/editor/hoverTableLookup";
|
import { normalizeTableHoverLookupMode } from "@/lib/editor/hoverTableLookup";
|
||||||
import { normalizeRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates";
|
import { normalizeRedisKeyTemplates } from "@/lib/redis/redisKeyTemplates";
|
||||||
|
import { normalizeCsvNullMode } from "@/lib/export/csvNullMode";
|
||||||
|
|
||||||
export const EDITOR_SETTINGS_DRAFT_KEYS = [
|
export const EDITOR_SETTINGS_DRAFT_KEYS = [
|
||||||
"fontFamily",
|
"fontFamily",
|
||||||
@@ -120,6 +121,7 @@ export const EDITOR_SETTINGS_DRAFT_KEYS = [
|
|||||||
"redisDatabaseDisplayLimit",
|
"redisDatabaseDisplayLimit",
|
||||||
"exportBatchSize",
|
"exportBatchSize",
|
||||||
"csvQuoteMode",
|
"csvQuoteMode",
|
||||||
|
"csvNullMode",
|
||||||
"exportRowLimitEnabled",
|
"exportRowLimitEnabled",
|
||||||
"exportRowLimit",
|
"exportRowLimit",
|
||||||
"queryExportKeysetOptimizationEnabled",
|
"queryExportKeysetOptimizationEnabled",
|
||||||
@@ -164,6 +166,7 @@ function normalizedDraftValue(key: EditorSettingsDraftKey, value: unknown): unkn
|
|||||||
if (key === "tableCompletionSchemaQualification") return normalizeSqlTableCompletionSchemaQualification(value);
|
if (key === "tableCompletionSchemaQualification") return normalizeSqlTableCompletionSchemaQualification(value);
|
||||||
if (key === "tableHoverLookupMode") return normalizeTableHoverLookupMode(value);
|
if (key === "tableHoverLookupMode") return normalizeTableHoverLookupMode(value);
|
||||||
if (key === "redisKeyTemplates") return normalizeRedisKeyTemplates(value);
|
if (key === "redisKeyTemplates") return normalizeRedisKeyTemplates(value);
|
||||||
|
if (key === "csvNullMode") return normalizeCsvNullMode(value);
|
||||||
if (key === "backgroundImage") return normalizeBackgroundImageSettings(value);
|
if (key === "backgroundImage") return normalizeBackgroundImageSettings(value);
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,6 +178,7 @@ const SETTINGS_TRANSFER_CATEGORY_KEYS: Record<SettingsTransferCategoryId, readon
|
|||||||
"redisDatabaseDisplayLimit",
|
"redisDatabaseDisplayLimit",
|
||||||
"exportBatchSize",
|
"exportBatchSize",
|
||||||
"csvQuoteMode",
|
"csvQuoteMode",
|
||||||
|
"csvNullMode",
|
||||||
"exportRowLimitEnabled",
|
"exportRowLimitEnabled",
|
||||||
"exportRowLimit",
|
"exportRowLimit",
|
||||||
"queryExportKeysetOptimizationEnabled",
|
"queryExportKeysetOptimizationEnabled",
|
||||||
|
|||||||
@@ -97,8 +97,15 @@ export interface TableImportParseSettings {
|
|||||||
databaseType?: DatabaseType | null;
|
databaseType?: DatabaseType | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function defaultTableImportEmptyStringAsNull(format: TableImportSourceFormat): boolean {
|
/**
|
||||||
return format !== "excel";
|
* 导入时分隔文本的「空字段即 NULL」默认值。
|
||||||
|
*
|
||||||
|
* 导出端默认把 NULL 写成 `\N` 字面量、空字符串才写成空字段,所以导入端默认必须
|
||||||
|
* 保留空字段(不当作 NULL),否则 DBX 自己导出的 CSV 在导入时会把空字符串还原成
|
||||||
|
* NULL,写回 `NOT NULL DEFAULT ''` 的列就会报「不允许为 null」。
|
||||||
|
*/
|
||||||
|
export function defaultTableImportEmptyStringAsNull(_format: TableImportSourceFormat): boolean {
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildTableImportParseOptions(settings: TableImportParseSettings): TableImportParseOptions {
|
export function buildTableImportParseOptions(settings: TableImportParseSettings): TableImportParseOptions {
|
||||||
@@ -113,6 +120,9 @@ export function buildTableImportParseOptions(settings: TableImportParseSettings)
|
|||||||
lastDataRow: settings.lastDataRow,
|
lastDataRow: settings.lastDataRow,
|
||||||
trimValues: settings.trimValues,
|
trimValues: settings.trimValues,
|
||||||
emptyStringAsNull: settings.emptyStringAsNull,
|
emptyStringAsNull: settings.emptyStringAsNull,
|
||||||
|
// 勾选「空字符串作为 NULL」= 退回旧行为:关闭 NULL 字面量,空字段一律当 NULL。
|
||||||
|
// 不勾选时留空,由后端使用与导出端一致的默认字面量 `\N`。
|
||||||
|
nullLiteral: settings.emptyStringAsNull ? "" : undefined,
|
||||||
sheetName: settings.format === "excel" ? settings.sheetName || null : null,
|
sheetName: settings.format === "excel" ? settings.sheetName || null : null,
|
||||||
jsonShape: settings.format === "json" ? settings.jsonShape : null,
|
jsonShape: settings.format === "json" ? settings.jsonShape : null,
|
||||||
// SQL 脚本需要按源方言解析字符串转义与标识符大小写(取目标连接的数据库类型)
|
// SQL 脚本需要按源方言解析字符串转义与标识符大小写(取目标连接的数据库类型)
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ import { TABLE_DATA_EXPORT_PAGE_SIZE } from "@/lib/table/tableDataExport";
|
|||||||
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
|
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
|
||||||
import { isDataTabMetadataLifecycleStale } from "@/lib/sidebar/dataTabOpenPolicy";
|
import { isDataTabMetadataLifecycleStale } from "@/lib/sidebar/dataTabOpenPolicy";
|
||||||
import type { SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
import type { SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||||
|
import { csvNullLiteralForMode } from "@/lib/export/csvNullMode";
|
||||||
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
|
||||||
import { getCachedTableMetadata, loadTableColumns, loadTableIndexes, loadTableMetadata, tableMetadataToDataTabMeta, updateCachedTableMetadataType, type TableMetadataRequest } from "@/lib/metadata/tableMetadataCache";
|
import { getCachedTableMetadata, loadTableColumns, loadTableIndexes, loadTableMetadata, tableMetadataToDataTabMeta, updateCachedTableMetadataType, type TableMetadataRequest } from "@/lib/metadata/tableMetadataCache";
|
||||||
import { loadObjectDdl } from "@/lib/metadata/objectDdlCache";
|
import { loadObjectDdl } from "@/lib/metadata/objectDdlCache";
|
||||||
@@ -9237,6 +9238,7 @@ export const useQueryStore = defineStore("query", () => {
|
|||||||
keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled,
|
keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled,
|
||||||
csvQuoteMode: settings.csvQuoteMode,
|
csvQuoteMode: settings.csvQuoteMode,
|
||||||
clientSessionId,
|
clientSessionId,
|
||||||
|
nullLiteral: csvNullLiteralForMode(settings.csvNullMode),
|
||||||
executionId: uuid(),
|
executionId: uuid(),
|
||||||
exportTableName: options.exportTableName,
|
exportTableName: options.exportTableName,
|
||||||
exportColumnTypes: options.exportColumnTypes,
|
exportColumnTypes: options.exportColumnTypes,
|
||||||
@@ -9277,6 +9279,7 @@ export const useQueryStore = defineStore("query", () => {
|
|||||||
keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled,
|
keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled,
|
||||||
csvQuoteMode: settings.csvQuoteMode,
|
csvQuoteMode: settings.csvQuoteMode,
|
||||||
clientSessionId: `${tabClientSessionId(tab, "export")}:${exportId}`,
|
clientSessionId: `${tabClientSessionId(tab, "export")}:${exportId}`,
|
||||||
|
nullLiteral: csvNullLiteralForMode(settings.csvNullMode),
|
||||||
executionId: uuid(),
|
executionId: uuid(),
|
||||||
numericColumnRightAlign: settings.numericColumnRightAlign,
|
numericColumnRightAlign: settings.numericColumnRightAlign,
|
||||||
columnComments,
|
columnComments,
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ import { normalizeTableHoverLookupMode, type TableHoverLookupMode } from "@/lib/
|
|||||||
import { normalizeCompletionTriggerMode, type SqlCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPolicy";
|
import { normalizeCompletionTriggerMode, type SqlCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPolicy";
|
||||||
import { DEFAULT_SQL_TABLE_COMPLETION_SCHEMA_QUALIFICATION, normalizeSqlTableCompletionSchemaQualification, type SqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification";
|
import { DEFAULT_SQL_TABLE_COMPLETION_SCHEMA_QUALIFICATION, normalizeSqlTableCompletionSchemaQualification, type SqlTableCompletionSchemaQualification } from "@/lib/sql/sqlCompletionSchemaQualification";
|
||||||
import { DEFAULT_CSV_QUOTE_MODE, normalizeCsvQuoteMode, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
import { DEFAULT_CSV_QUOTE_MODE, normalizeCsvQuoteMode, type CsvQuoteMode } from "@/lib/export/csvQuoteMode";
|
||||||
|
import { DEFAULT_CSV_NULL_MODE, normalizeCsvNullMode, type CsvNullMode } from "@/lib/export/csvNullMode";
|
||||||
import { configureMetadataRuntimeCache, METADATA_CACHE_DEFAULT_MEMORY_MB, normalizeMetadataCacheMemoryMb } from "@/lib/metadata/metadataRuntimeCache";
|
import { configureMetadataRuntimeCache, METADATA_CACHE_DEFAULT_MEMORY_MB, normalizeMetadataCacheMemoryMb } from "@/lib/metadata/metadataRuntimeCache";
|
||||||
import type { AiApiStyle, AiAssistantMode, AiAuthMethod, AiChatSelectionState, AiConfig, AiConfigItem, AiConfiguredModel, AiEffortLevel, AiEffortSelection, AiModelEffortPreference, AiProvider, AiReasoningLevel, AiTestConnectionResult } from "@/types/ai";
|
import type { AiApiStyle, AiAssistantMode, AiAuthMethod, AiChatSelectionState, AiConfig, AiConfigItem, AiConfiguredModel, AiEffortLevel, AiEffortSelection, AiModelEffortPreference, AiProvider, AiReasoningLevel, AiTestConnectionResult } from "@/types/ai";
|
||||||
import type { SqlShortcutAction, SqlSnippet, TableInfoTab } from "@/types/database";
|
import type { SqlShortcutAction, SqlSnippet, TableInfoTab } from "@/types/database";
|
||||||
@@ -53,6 +54,7 @@ export type {
|
|||||||
AiReasoningLevel,
|
AiReasoningLevel,
|
||||||
AiTestConnectionResult,
|
AiTestConnectionResult,
|
||||||
CsvQuoteMode,
|
CsvQuoteMode,
|
||||||
|
CsvNullMode,
|
||||||
DataTabReuseMode,
|
DataTabReuseMode,
|
||||||
SavedSqlOpenTargetMode,
|
SavedSqlOpenTargetMode,
|
||||||
SqlCompletionTriggerMode,
|
SqlCompletionTriggerMode,
|
||||||
@@ -998,6 +1000,7 @@ export interface EditorSettings {
|
|||||||
tableColumnTemplateFields: string[];
|
tableColumnTemplateFields: string[];
|
||||||
exportBatchSize: number;
|
exportBatchSize: number;
|
||||||
csvQuoteMode: CsvQuoteMode;
|
csvQuoteMode: CsvQuoteMode;
|
||||||
|
csvNullMode: CsvNullMode;
|
||||||
/** Global Redis key-search templates; overridden by non-empty connection templates. */
|
/** Global Redis key-search templates; overridden by non-empty connection templates. */
|
||||||
redisKeyTemplates: string[];
|
redisKeyTemplates: string[];
|
||||||
/** Sidebar database-list cap for Redis connections; the rest are revealed via "load more". */
|
/** Sidebar database-list cap for Redis connections; the rest are revealed via "load more". */
|
||||||
@@ -1279,6 +1282,7 @@ export const DEFAULT_EDITOR_SETTINGS: EditorSettings = {
|
|||||||
tableColumnTemplateFields: [...DEFAULT_TABLE_COLUMN_TEMPLATE_FIELDS],
|
tableColumnTemplateFields: [...DEFAULT_TABLE_COLUMN_TEMPLATE_FIELDS],
|
||||||
exportBatchSize: 2000,
|
exportBatchSize: 2000,
|
||||||
csvQuoteMode: DEFAULT_CSV_QUOTE_MODE,
|
csvQuoteMode: DEFAULT_CSV_QUOTE_MODE,
|
||||||
|
csvNullMode: DEFAULT_CSV_NULL_MODE,
|
||||||
redisKeyTemplates: [],
|
redisKeyTemplates: [],
|
||||||
redisDatabaseDisplayLimit: REDIS_DATABASE_DISPLAY_LIMIT_DEFAULT,
|
redisDatabaseDisplayLimit: REDIS_DATABASE_DISPLAY_LIMIT_DEFAULT,
|
||||||
exportRowLimitEnabled: false,
|
exportRowLimitEnabled: false,
|
||||||
@@ -1896,6 +1900,7 @@ export function normalizeEditorSettings(settings: Partial<EditorSettings>, exist
|
|||||||
tableColumnTemplateFields: normalizeTableColumnTemplateFields(settings.tableColumnTemplateFields),
|
tableColumnTemplateFields: normalizeTableColumnTemplateFields(settings.tableColumnTemplateFields),
|
||||||
exportBatchSize: typeof settings.exportBatchSize === "number" && settings.exportBatchSize >= 100 && settings.exportBatchSize <= 100000 ? Math.round(settings.exportBatchSize) : DEFAULT_EDITOR_SETTINGS.exportBatchSize,
|
exportBatchSize: typeof settings.exportBatchSize === "number" && settings.exportBatchSize >= 100 && settings.exportBatchSize <= 100000 ? Math.round(settings.exportBatchSize) : DEFAULT_EDITOR_SETTINGS.exportBatchSize,
|
||||||
csvQuoteMode: normalizeCsvQuoteMode(settings.csvQuoteMode),
|
csvQuoteMode: normalizeCsvQuoteMode(settings.csvQuoteMode),
|
||||||
|
csvNullMode: normalizeCsvNullMode(settings.csvNullMode),
|
||||||
redisKeyTemplates: normalizeRedisKeyTemplates(settings.redisKeyTemplates),
|
redisKeyTemplates: normalizeRedisKeyTemplates(settings.redisKeyTemplates),
|
||||||
redisDatabaseDisplayLimit:
|
redisDatabaseDisplayLimit:
|
||||||
typeof settings.redisDatabaseDisplayLimit === "number" && settings.redisDatabaseDisplayLimit >= REDIS_DATABASE_DISPLAY_LIMIT_MIN && settings.redisDatabaseDisplayLimit <= REDIS_DATABASE_DISPLAY_LIMIT_MAX
|
typeof settings.redisDatabaseDisplayLimit === "number" && settings.redisDatabaseDisplayLimit >= REDIS_DATABASE_DISPLAY_LIMIT_MIN && settings.redisDatabaseDisplayLimit <= REDIS_DATABASE_DISPLAY_LIMIT_MAX
|
||||||
@@ -2703,6 +2708,7 @@ export const useSettingsStore = defineStore("settings", () => {
|
|||||||
if (partial.tableColumnTemplateFields !== undefined) editorSettings.value.tableColumnTemplateFields = normalizeTableColumnTemplateFields(partial.tableColumnTemplateFields);
|
if (partial.tableColumnTemplateFields !== undefined) editorSettings.value.tableColumnTemplateFields = normalizeTableColumnTemplateFields(partial.tableColumnTemplateFields);
|
||||||
if (partial.exportBatchSize !== undefined) editorSettings.value.exportBatchSize = Math.min(100000, Math.max(100, Math.round(partial.exportBatchSize)));
|
if (partial.exportBatchSize !== undefined) editorSettings.value.exportBatchSize = Math.min(100000, Math.max(100, Math.round(partial.exportBatchSize)));
|
||||||
if (partial.csvQuoteMode !== undefined) editorSettings.value.csvQuoteMode = normalizeCsvQuoteMode(partial.csvQuoteMode);
|
if (partial.csvQuoteMode !== undefined) editorSettings.value.csvQuoteMode = normalizeCsvQuoteMode(partial.csvQuoteMode);
|
||||||
|
if (partial.csvNullMode !== undefined) editorSettings.value.csvNullMode = normalizeCsvNullMode(partial.csvNullMode);
|
||||||
if (partial.redisKeyTemplates !== undefined) editorSettings.value.redisKeyTemplates = normalizeRedisKeyTemplates(partial.redisKeyTemplates);
|
if (partial.redisKeyTemplates !== undefined) editorSettings.value.redisKeyTemplates = normalizeRedisKeyTemplates(partial.redisKeyTemplates);
|
||||||
if (partial.redisDatabaseDisplayLimit !== undefined) editorSettings.value.redisDatabaseDisplayLimit = Math.min(REDIS_DATABASE_DISPLAY_LIMIT_MAX, Math.max(REDIS_DATABASE_DISPLAY_LIMIT_MIN, Math.round(partial.redisDatabaseDisplayLimit)));
|
if (partial.redisDatabaseDisplayLimit !== undefined) editorSettings.value.redisDatabaseDisplayLimit = Math.min(REDIS_DATABASE_DISPLAY_LIMIT_MAX, Math.max(REDIS_DATABASE_DISPLAY_LIMIT_MIN, Math.round(partial.redisDatabaseDisplayLimit)));
|
||||||
if (partial.exportRowLimitEnabled !== undefined) editorSettings.value.exportRowLimitEnabled = partial.exportRowLimitEnabled;
|
if (partial.exportRowLimitEnabled !== undefined) editorSettings.value.exportRowLimitEnabled = partial.exportRowLimitEnabled;
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ pub struct TableCsvExportOptions {
|
|||||||
pub timeout_secs: Option<u64>,
|
pub timeout_secs: Option<u64>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub csv_quote_mode: CsvQuoteMode,
|
pub csv_quote_mode: CsvQuoteMode,
|
||||||
|
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||||
|
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串无法区分)。
|
||||||
|
#[serde(default = "default_csv_null_literal")]
|
||||||
|
pub null_literal: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn connection_database_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
|
async fn connection_database_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
|
||||||
@@ -156,7 +160,12 @@ where
|
|||||||
|
|
||||||
for row in result.rows {
|
for row in result.rows {
|
||||||
writer.write_all(b"\n").map_err(|err| err.to_string())?;
|
writer.write_all(b"\n").map_err(|err| err.to_string())?;
|
||||||
write_csv_value_row(writer, row, options.csv_quote_mode)?;
|
write_csv_value_row_with_options(
|
||||||
|
writer,
|
||||||
|
row,
|
||||||
|
options.csv_quote_mode,
|
||||||
|
csv_null_literal(&options.null_literal),
|
||||||
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
rows_exported += fetched as u64;
|
rows_exported += fetched as u64;
|
||||||
@@ -191,6 +200,7 @@ mod tests {
|
|||||||
page_size: Some(page_size),
|
page_size: Some(page_size),
|
||||||
timeout_secs: Some(30),
|
timeout_secs: Some(30),
|
||||||
csv_quote_mode: CsvQuoteMode::default(),
|
csv_quote_mode: CsvQuoteMode::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -354,6 +364,194 @@ mod tests {
|
|||||||
config
|
config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 表数据导出必须把 NULL 写成字面量、空字符串写成 `""`,否则导入端拿不到引号信息,
|
||||||
|
/// 两者都会变成 NULL(正是「NOT NULL DEFAULT '' 的列导入报错」的成因)。
|
||||||
|
#[tokio::test]
|
||||||
|
async fn streamed_table_export_writes_null_literal_and_keeps_empty_strings() {
|
||||||
|
fn result_with_a_null_and_an_empty_string() -> QueryResult {
|
||||||
|
serde_json::from_value(serde_json::json!({
|
||||||
|
"columns": ["name", "note"],
|
||||||
|
"rows": [["a", null], ["b", ""]],
|
||||||
|
"affected_rows": 0,
|
||||||
|
"execution_time_ms": 1,
|
||||||
|
"session_id": null,
|
||||||
|
"has_more": false
|
||||||
|
}))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut options = export_options(10);
|
||||||
|
options.null_literal = dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string();
|
||||||
|
let mut pages = vec![result_with_a_null_and_an_empty_string()].into_iter();
|
||||||
|
let mut output = Vec::new();
|
||||||
|
write_table_csv_pages(&mut output, DatabaseType::Postgres, &options, None, |_, _| {
|
||||||
|
std::future::ready(Ok(pages.next().expect("single page")))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(String::from_utf8(output).unwrap(), "\"name\",\"note\"\n\"a\",\\N\n\"b\",\"\"");
|
||||||
|
|
||||||
|
// 关闭字面量(旧行为):NULL 与空串都写空字段,二者在文件里已无法区分
|
||||||
|
options.null_literal = String::new();
|
||||||
|
let mut pages = vec![result_with_a_null_and_an_empty_string()].into_iter();
|
||||||
|
let mut output = Vec::new();
|
||||||
|
write_table_csv_pages(&mut output, DatabaseType::Postgres, &options, None, |_, _| {
|
||||||
|
std::future::ready(Ok(pages.next().expect("single page")))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(String::from_utf8(output).unwrap(), "\"name\",\"note\"\n\"a\",\n\"b\",\"\"");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 用户反馈:`NOT NULL DEFAULT ''` 的列导出成 CSV、再导入到另一个环境的同名表时报
|
||||||
|
/// 「不允许为 null」。用真实 SQLite 库走一遍「导出 → 导入」,确认空字符串仍是空字符串、
|
||||||
|
/// NULL 仍是 NULL(而不是双双塌成 NULL)。
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exported_csv_round_trips_empty_strings_and_nulls_through_a_real_import() {
|
||||||
|
use crate::data::table_import::{
|
||||||
|
import_table_file_core, TableImportColumnMapping, TableImportMode, TableImportParseOptions,
|
||||||
|
TableImportRequest, TableImportSourceFormat,
|
||||||
|
};
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
const DDL: &str = "CREATE TABLE contacts (id INTEGER, name TEXT NOT NULL DEFAULT '', note TEXT)";
|
||||||
|
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let source_path = directory.path().join("source.db");
|
||||||
|
let sqlite = crate::db::sqlite::connect_path_create_if_missing(source_path.to_str().unwrap()).await.unwrap();
|
||||||
|
crate::db::sqlite::execute_query(&sqlite, DDL).await.unwrap();
|
||||||
|
crate::db::sqlite::execute_query(
|
||||||
|
&sqlite,
|
||||||
|
"INSERT INTO contacts (id, name, note) VALUES (1, '', NULL), (2, 'Ada', 'x')",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut options = export_options(100);
|
||||||
|
options.table_name = "contacts".to_string();
|
||||||
|
options.columns = vec!["id".to_string(), "name".to_string(), "note".to_string()];
|
||||||
|
options.null_literal = dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string();
|
||||||
|
|
||||||
|
let mut exported_bytes = "\u{FEFF}".as_bytes().to_vec();
|
||||||
|
let exported = write_table_csv_pages(&mut exported_bytes, DatabaseType::Sqlite, &options, None, |sql, _| {
|
||||||
|
let sqlite = sqlite.clone();
|
||||||
|
async move { crate::db::sqlite::execute_query(&sqlite, &sql).await.map_err(|error| error.to_string()) }
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(exported, 2);
|
||||||
|
// 导出的 NULL 与空串必须已经可区分:空串 `""`、NULL 裸写 `\N`
|
||||||
|
assert_eq!(
|
||||||
|
String::from_utf8(exported_bytes.clone()).unwrap(),
|
||||||
|
"\u{FEFF}\"id\",\"name\",\"note\"\n\"1\",\"\",\\N\n\"2\",\"Ada\",\"x\""
|
||||||
|
);
|
||||||
|
|
||||||
|
let csv_path = directory.path().join("contacts.csv");
|
||||||
|
let mut csv_file = std::fs::File::create(&csv_path).unwrap();
|
||||||
|
csv_file.write_all(&exported_bytes).unwrap();
|
||||||
|
csv_file.flush().unwrap();
|
||||||
|
drop(csv_file);
|
||||||
|
|
||||||
|
// 目标环境:同名 DDL,`name` 同样是 NOT NULL DEFAULT ''
|
||||||
|
let target_path = directory.path().join("target.db");
|
||||||
|
let target = crate::db::sqlite::connect_path_create_if_missing(target_path.to_str().unwrap()).await.unwrap();
|
||||||
|
crate::db::sqlite::execute_query(&target, DDL).await.unwrap();
|
||||||
|
|
||||||
|
let storage = crate::persistence::test_storage::open(&directory.path().join("storage.db")).await.unwrap();
|
||||||
|
let state = AppState::new(storage);
|
||||||
|
let connection_id = "csv-round-trip";
|
||||||
|
let pool_key = format!("{connection_id}:session:import");
|
||||||
|
state
|
||||||
|
.update_connection_pools(|connections| {
|
||||||
|
connections.insert(pool_key.clone(), crate::connection::PoolKind::Sqlite(target.clone()));
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": connection_id,
|
||||||
|
"name": "CSV round-trip",
|
||||||
|
"db_type": "sqlite",
|
||||||
|
"host": "",
|
||||||
|
"port": 0,
|
||||||
|
"username": "",
|
||||||
|
"password": "",
|
||||||
|
"database": target_path.to_string_lossy()
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
state.configs.write().await.insert(connection_id.to_string(), config);
|
||||||
|
|
||||||
|
let request = TableImportRequest {
|
||||||
|
import_id: "csv-round-trip".to_string(),
|
||||||
|
connection_id: connection_id.to_string(),
|
||||||
|
database: String::new(),
|
||||||
|
schema: String::new(),
|
||||||
|
table: "contacts".to_string(),
|
||||||
|
file_path: csv_path.to_string_lossy().into_owned(),
|
||||||
|
source_ref: None,
|
||||||
|
source_format: Some(TableImportSourceFormat::Csv),
|
||||||
|
parse_options: TableImportParseOptions::default(),
|
||||||
|
mappings: ["id", "name", "note"]
|
||||||
|
.into_iter()
|
||||||
|
.map(|column| TableImportColumnMapping {
|
||||||
|
source_column: column.to_string(),
|
||||||
|
target_column: column.to_string(),
|
||||||
|
target_data_type: None,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
mode: TableImportMode::Append,
|
||||||
|
create_table: false,
|
||||||
|
batch_size: 2,
|
||||||
|
date_time_format: None,
|
||||||
|
prepared_source: None,
|
||||||
|
retain_source: false,
|
||||||
|
conflict_policy: None,
|
||||||
|
skip_duplicate_rows: false,
|
||||||
|
};
|
||||||
|
let summary = import_table_file_core(
|
||||||
|
&state,
|
||||||
|
&request,
|
||||||
|
&DatabaseType::Sqlite,
|
||||||
|
&pool_key,
|
||||||
|
|_| Box::pin(async { false }),
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("NOT NULL DEFAULT '' 的列必须能原样导入空字符串");
|
||||||
|
|
||||||
|
assert_eq!(summary.rows_imported, 2);
|
||||||
|
assert_eq!(
|
||||||
|
crate::db::sqlite::execute_query(&target, "SELECT id, name, note FROM contacts ORDER BY id")
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.rows,
|
||||||
|
vec![
|
||||||
|
vec![serde_json::json!(1), serde_json::json!(""), serde_json::Value::Null],
|
||||||
|
vec![serde_json::json!(2), serde_json::json!("Ada"), serde_json::json!("x")],
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
// 反向验证:把 NULL 字面量关掉、恢复「空字段即 NULL」的旧配置再导一次同一个文件,
|
||||||
|
// 必须重现用户报的「不允许为 null」,否则这条用例并没有真正覆盖该缺陷。
|
||||||
|
let legacy_request = TableImportRequest {
|
||||||
|
import_id: "csv-round-trip-legacy".to_string(),
|
||||||
|
parse_options: TableImportParseOptions {
|
||||||
|
null_literal: Some(String::new()),
|
||||||
|
..TableImportParseOptions::default()
|
||||||
|
},
|
||||||
|
..request
|
||||||
|
};
|
||||||
|
let error = import_table_file_core(
|
||||||
|
&state,
|
||||||
|
&legacy_request,
|
||||||
|
&DatabaseType::Sqlite,
|
||||||
|
&pool_key,
|
||||||
|
|_| Box::pin(async { false }),
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("旧配置会把空字符串写成 NULL,必须触发 NOT NULL 约束错误");
|
||||||
|
assert!(error.to_lowercase().contains("not null"), "unexpected error: {error}");
|
||||||
|
}
|
||||||
|
|
||||||
/// The refusal has to land before `File::create`, otherwise the user is left
|
/// The refusal has to land before `File::create`, otherwise the user is left
|
||||||
/// holding a header-only CSV that looks like an object with no records.
|
/// holding a header-only CSV that looks like an object with no records.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -377,6 +575,7 @@ mod tests {
|
|||||||
page_size: None,
|
page_size: None,
|
||||||
timeout_secs: None,
|
timeout_secs: None,
|
||||||
csv_quote_mode: CsvQuoteMode::default(),
|
csv_quote_mode: CsvQuoteMode::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ use std::time::{Duration, Instant};
|
|||||||
|
|
||||||
use crate::connection::{AppState, PoolKind};
|
use crate::connection::{AppState, PoolKind};
|
||||||
use crate::csv_export::{
|
use crate::csv_export::{
|
||||||
format_query_result_csv_with_quote_mode, format_tsv, push_query_result_csv_row_with_quote_mode, push_tsv_row,
|
csv_null_literal, format_query_result_csv_with_options, format_tsv, push_query_result_csv_row_with_options,
|
||||||
CsvQuoteMode,
|
push_tsv_row, CsvQuoteMode,
|
||||||
};
|
};
|
||||||
pub use crate::database_export::ExportStatus;
|
pub use crate::database_export::ExportStatus;
|
||||||
use crate::database_export::{
|
use crate::database_export::{
|
||||||
@@ -88,6 +88,10 @@ pub struct QueryResultExportRequest {
|
|||||||
pub insert_mode: SqlInsertMode,
|
pub insert_mode: SqlInsertMode,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub csv_quote_mode: CsvQuoteMode,
|
pub csv_quote_mode: CsvQuoteMode,
|
||||||
|
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||||
|
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。
|
||||||
|
#[serde(default = "dbx_formats::csv_export::default_csv_null_literal")]
|
||||||
|
pub null_literal: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub include_sql_sheet: bool,
|
pub include_sql_sheet: bool,
|
||||||
pub page_size: usize,
|
pub page_size: usize,
|
||||||
@@ -543,9 +547,14 @@ fn effective_row_limit(request: &QueryResultExportRequest) -> Option<usize> {
|
|||||||
request.row_limit
|
request.row_limit
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_text_export_header(format: &str, columns: &[String], csv_quote_mode: CsvQuoteMode) -> String {
|
fn format_text_export_header(
|
||||||
|
format: &str,
|
||||||
|
columns: &[String],
|
||||||
|
csv_quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
|
) -> String {
|
||||||
let content = if format == "csv" {
|
let content = if format == "csv" {
|
||||||
format_query_result_csv_with_quote_mode(columns, &[], csv_quote_mode)
|
format_query_result_csv_with_options(columns, &[], csv_quote_mode, null_literal)
|
||||||
} else {
|
} else {
|
||||||
format_tsv(columns, &[])
|
format_tsv(columns, &[])
|
||||||
};
|
};
|
||||||
@@ -558,13 +567,14 @@ fn write_text_export_row<W: Write>(
|
|||||||
row: &[Value],
|
row: &[Value],
|
||||||
buffer: &mut String,
|
buffer: &mut String,
|
||||||
csv_quote_mode: CsvQuoteMode,
|
csv_quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
buffer.clear();
|
buffer.clear();
|
||||||
buffer.push('\n');
|
buffer.push('\n');
|
||||||
if format == "csv" {
|
if format == "csv" {
|
||||||
push_query_result_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
push_query_result_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||||
} else {
|
} else {
|
||||||
push_tsv_row(buffer, row);
|
push_tsv_row(buffer, row, null_literal);
|
||||||
}
|
}
|
||||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
}
|
}
|
||||||
@@ -575,14 +585,15 @@ fn write_text_export_rows<W: Write>(
|
|||||||
rows: &[Vec<Value>],
|
rows: &[Vec<Value>],
|
||||||
buffer: &mut String,
|
buffer: &mut String,
|
||||||
csv_quote_mode: CsvQuoteMode,
|
csv_quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
buffer.clear();
|
buffer.clear();
|
||||||
for row in rows {
|
for row in rows {
|
||||||
buffer.push('\n');
|
buffer.push('\n');
|
||||||
if format == "csv" {
|
if format == "csv" {
|
||||||
push_query_result_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
push_query_result_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||||
} else {
|
} else {
|
||||||
push_tsv_row(buffer, row);
|
push_tsv_row(buffer, row, null_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
@@ -1093,7 +1104,12 @@ async fn export_query_result_core_inner(
|
|||||||
if format == "csv" || format == "txt" {
|
if format == "csv" || format == "txt" {
|
||||||
if let Some(file) = text_file.as_mut() {
|
if let Some(file) = text_file.as_mut() {
|
||||||
if !wrote_text_header {
|
if !wrote_text_header {
|
||||||
let header = format_text_export_header(&format, &columns, request.csv_quote_mode);
|
let header = format_text_export_header(
|
||||||
|
&format,
|
||||||
|
&columns,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
if row_count > 0 {
|
if row_count > 0 {
|
||||||
write_text_export_rows(
|
write_text_export_rows(
|
||||||
@@ -1102,6 +1118,7 @@ async fn export_query_result_core_inner(
|
|||||||
formatted_rows.as_ref(),
|
formatted_rows.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
wrote_text_header = true;
|
wrote_text_header = true;
|
||||||
@@ -1112,6 +1129,7 @@ async fn export_query_result_core_inner(
|
|||||||
formatted_rows.as_ref(),
|
formatted_rows.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1184,7 +1202,12 @@ async fn export_query_result_core_inner(
|
|||||||
|
|
||||||
if format == "csv" || format == "txt" {
|
if format == "csv" || format == "txt" {
|
||||||
if !wrote_text_header {
|
if !wrote_text_header {
|
||||||
let header = format_text_export_header(&format, &columns, request.csv_quote_mode);
|
let header = format_text_export_header(
|
||||||
|
&format,
|
||||||
|
&columns,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
if let Some(file) = text_file.as_mut() {
|
if let Some(file) = text_file.as_mut() {
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
}
|
}
|
||||||
@@ -1301,7 +1324,12 @@ async fn try_export_postgres_query_result_stream(
|
|||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
let header = format_text_export_header(
|
||||||
|
format,
|
||||||
|
&columns,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
} else if json_writer.is_none() {
|
} else if json_writer.is_none() {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
@@ -1329,6 +1357,7 @@ async fn try_export_postgres_query_result_stream(
|
|||||||
formatted.as_ref(),
|
formatted.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
} else if let Some(writer) = json_writer.as_mut() {
|
} else if let Some(writer) = json_writer.as_mut() {
|
||||||
writer.write_row(&columns, formatted.as_ref())?;
|
writer.write_row(&columns, formatted.as_ref())?;
|
||||||
@@ -1547,7 +1576,12 @@ async fn try_export_mysql_query_result_stream(
|
|||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
let header = format_text_export_header(
|
||||||
|
format,
|
||||||
|
&columns,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
} else if json_writer.is_none() {
|
} else if json_writer.is_none() {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
@@ -1575,6 +1609,7 @@ async fn try_export_mysql_query_result_stream(
|
|||||||
formatted.as_ref(),
|
formatted.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
} else if let Some(writer) = json_writer.as_mut() {
|
} else if let Some(writer) = json_writer.as_mut() {
|
||||||
writer.write_row(&columns, formatted.as_ref())?;
|
writer.write_row(&columns, formatted.as_ref())?;
|
||||||
@@ -1772,7 +1807,12 @@ async fn try_export_clickhouse_query_result_stream(
|
|||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
let header = format_text_export_header(
|
||||||
|
format,
|
||||||
|
&columns,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
} else if json_writer.is_none() {
|
} else if json_writer.is_none() {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
@@ -1800,6 +1840,7 @@ async fn try_export_clickhouse_query_result_stream(
|
|||||||
formatted.as_ref(),
|
formatted.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
} else if let Some(writer) = json_writer.as_mut() {
|
} else if let Some(writer) = json_writer.as_mut() {
|
||||||
writer.write_row(&columns, formatted.as_ref())?;
|
writer.write_row(&columns, formatted.as_ref())?;
|
||||||
@@ -1966,7 +2007,12 @@ async fn try_export_sqlserver_query_result_stream(
|
|||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.set_columns(columns.clone(), &temporal_column_types, &[], request)?;
|
writer.set_columns(columns.clone(), &temporal_column_types, &[], request)?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
let header = format_text_export_header(
|
||||||
|
format,
|
||||||
|
&columns,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
} else if json_writer.is_none() {
|
} else if json_writer.is_none() {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
@@ -1990,6 +2036,7 @@ async fn try_export_sqlserver_query_result_stream(
|
|||||||
formatted.as_ref(),
|
formatted.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
} else if let Some(writer) = json_writer.as_mut() {
|
} else if let Some(writer) = json_writer.as_mut() {
|
||||||
writer.write_row(&columns, formatted.as_ref())?;
|
writer.write_row(&columns, formatted.as_ref())?;
|
||||||
@@ -2191,6 +2238,7 @@ mod tests {
|
|||||||
identifier_quote: None,
|
identifier_quote: None,
|
||||||
insert_mode: SqlInsertMode::Batch,
|
insert_mode: SqlInsertMode::Batch,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
exclude_primary_keys: false,
|
exclude_primary_keys: false,
|
||||||
primary_keys: Vec::new(),
|
primary_keys: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -2281,6 +2329,7 @@ mod tests {
|
|||||||
identifier_quote: None,
|
identifier_quote: None,
|
||||||
insert_mode: Default::default(),
|
insert_mode: Default::default(),
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
exclude_primary_keys: true,
|
exclude_primary_keys: true,
|
||||||
primary_keys: vec!["id".to_string()],
|
primary_keys: vec!["id".to_string()],
|
||||||
};
|
};
|
||||||
@@ -2357,6 +2406,7 @@ mod tests {
|
|||||||
execution_id: None,
|
execution_id: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -2560,7 +2610,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn txt_export_header_keeps_columns_for_empty_results() {
|
fn txt_export_header_keeps_columns_for_empty_results() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
format_text_export_header("txt", &["id".to_string(), "note".to_string()], CsvQuoteMode::All),
|
format_text_export_header("txt", &["id".to_string(), "note".to_string()], CsvQuoteMode::All, None),
|
||||||
"id\tnote"
|
"id\tnote"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -2602,7 +2652,7 @@ mod tests {
|
|||||||
let mut output = Vec::new();
|
let mut output = Vec::new();
|
||||||
let mut buffer = String::new();
|
let mut buffer = String::new();
|
||||||
|
|
||||||
write_text_export_row(&mut output, "csv", &row, &mut buffer, CsvQuoteMode::All).expect("write csv row");
|
write_text_export_row(&mut output, "csv", &row, &mut buffer, CsvQuoteMode::All, None).expect("write csv row");
|
||||||
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\"");
|
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\"");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ use crate::csv_export::format_csv;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::csv_export::format_tsv_rows;
|
use crate::csv_export::format_tsv_rows;
|
||||||
use crate::csv_export::{
|
use crate::csv_export::{
|
||||||
format_csv_with_quote_mode, format_tsv, push_table_csv_row, push_table_csv_row_with_quote_mode, push_tsv_row,
|
csv_null_literal, format_csv_with_options, format_tsv, push_table_csv_row, push_table_csv_row_with_options,
|
||||||
CsvQuoteMode,
|
push_tsv_row, CsvQuoteMode,
|
||||||
};
|
};
|
||||||
use crate::data_grid_sql::extra_is_auto_generated;
|
use crate::data_grid_sql::extra_is_auto_generated;
|
||||||
pub use crate::database_export::ExportStatus;
|
pub use crate::database_export::ExportStatus;
|
||||||
@@ -87,6 +87,10 @@ pub struct TableExportRequest {
|
|||||||
pub insert_dialect: SqlInsertDialect,
|
pub insert_dialect: SqlInsertDialect,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub csv_quote_mode: CsvQuoteMode,
|
pub csv_quote_mode: CsvQuoteMode,
|
||||||
|
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||||
|
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。
|
||||||
|
#[serde(default = "dbx_formats::csv_export::default_csv_null_literal")]
|
||||||
|
pub null_literal: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub columns: Option<Vec<String>>,
|
pub columns: Option<Vec<String>>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
@@ -164,13 +168,14 @@ fn write_table_text_row<W: Write>(
|
|||||||
row: &[Value],
|
row: &[Value],
|
||||||
buffer: &mut String,
|
buffer: &mut String,
|
||||||
csv_quote_mode: CsvQuoteMode,
|
csv_quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
buffer.clear();
|
buffer.clear();
|
||||||
buffer.push('\n');
|
buffer.push('\n');
|
||||||
if csv {
|
if csv {
|
||||||
push_table_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
push_table_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||||
} else {
|
} else {
|
||||||
push_tsv_row(buffer, row);
|
push_tsv_row(buffer, row, null_literal);
|
||||||
}
|
}
|
||||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
}
|
}
|
||||||
@@ -181,14 +186,15 @@ fn write_table_text_rows<W: Write>(
|
|||||||
rows: &[Vec<Value>],
|
rows: &[Vec<Value>],
|
||||||
buffer: &mut String,
|
buffer: &mut String,
|
||||||
csv_quote_mode: CsvQuoteMode,
|
csv_quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
buffer.clear();
|
buffer.clear();
|
||||||
for row in rows {
|
for row in rows {
|
||||||
buffer.push('\n');
|
buffer.push('\n');
|
||||||
if csv {
|
if csv {
|
||||||
push_table_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
push_table_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||||
} else {
|
} else {
|
||||||
push_tsv_row(buffer, row);
|
push_tsv_row(buffer, row, null_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
@@ -1128,7 +1134,12 @@ async fn try_export_native_table_stream(
|
|||||||
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
||||||
);
|
);
|
||||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||||
let header = format_csv_with_quote_mode(col_names, &[], request.csv_quote_mode);
|
let header = format_csv_with_options(
|
||||||
|
col_names,
|
||||||
|
&[],
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||||
let mut row_buffer = String::new();
|
let mut row_buffer = String::new();
|
||||||
@@ -1147,7 +1158,14 @@ async fn try_export_native_table_stream(
|
|||||||
column_types,
|
column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer, request.csv_quote_mode)?;
|
write_table_text_row(
|
||||||
|
&mut file,
|
||||||
|
true,
|
||||||
|
formatted.as_ref(),
|
||||||
|
&mut row_buffer,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
)?;
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
if rows_exported.is_multiple_of(progress_interval) {
|
if rows_exported.is_multiple_of(progress_interval) {
|
||||||
on_progress(TableExportProgress {
|
on_progress(TableExportProgress {
|
||||||
@@ -1197,6 +1215,7 @@ async fn try_export_native_table_stream(
|
|||||||
formatted.as_ref(),
|
formatted.as_ref(),
|
||||||
&mut row_buffer,
|
&mut row_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
if rows_exported.is_multiple_of(progress_interval) {
|
if rows_exported.is_multiple_of(progress_interval) {
|
||||||
@@ -1780,8 +1799,12 @@ async fn export_table_data_core_inner(
|
|||||||
|
|
||||||
if is_first_batch {
|
if is_first_batch {
|
||||||
// First batch: write header + rows via format_csv
|
// First batch: write header + rows via format_csv
|
||||||
let csv_content =
|
let csv_content = format_csv_with_options(
|
||||||
format_csv_with_quote_mode(&col_names, formatted_rows.as_ref(), request.csv_quote_mode);
|
&col_names,
|
||||||
|
formatted_rows.as_ref(),
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||||
is_first_batch = false;
|
is_first_batch = false;
|
||||||
} else {
|
} else {
|
||||||
@@ -1792,6 +1815,7 @@ async fn export_table_data_core_inner(
|
|||||||
formatted_rows.as_ref(),
|
formatted_rows.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1880,6 +1904,7 @@ async fn export_table_data_core_inner(
|
|||||||
formatted_rows.as_ref(),
|
formatted_rows.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
is_first_batch = false;
|
is_first_batch = false;
|
||||||
} else {
|
} else {
|
||||||
@@ -1889,6 +1914,7 @@ async fn export_table_data_core_inner(
|
|||||||
formatted_rows.as_ref(),
|
formatted_rows.as_ref(),
|
||||||
&mut text_buffer,
|
&mut text_buffer,
|
||||||
request.csv_quote_mode,
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2476,6 +2502,7 @@ mod tests {
|
|||||||
row_limit,
|
row_limit,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -2631,6 +2658,7 @@ mod tests {
|
|||||||
insert_mode: Default::default(),
|
insert_mode: Default::default(),
|
||||||
insert_dialect: Default::default(),
|
insert_dialect: Default::default(),
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
exclude_primary_keys: false,
|
exclude_primary_keys: false,
|
||||||
columns: None,
|
columns: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -2730,7 +2758,7 @@ mod tests {
|
|||||||
let mut output = Vec::new();
|
let mut output = Vec::new();
|
||||||
let mut buffer = String::new();
|
let mut buffer = String::new();
|
||||||
|
|
||||||
write_table_text_row(&mut output, true, &row, &mut buffer, CsvQuoteMode::All).expect("write csv row");
|
write_table_text_row(&mut output, true, &row, &mut buffer, CsvQuoteMode::All, None).expect("write csv row");
|
||||||
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\"");
|
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\"");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2813,6 +2841,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -2876,6 +2905,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -2917,6 +2947,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -2953,6 +2984,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -2997,6 +3029,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -3053,6 +3086,7 @@ mod tests {
|
|||||||
row_limit: Some(1000),
|
row_limit: Some(1000),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -3103,6 +3137,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -3181,6 +3216,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -3241,6 +3277,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
@@ -3303,6 +3340,7 @@ mod tests {
|
|||||||
row_limit: None,
|
row_limit: None,
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: CsvQuoteMode::All,
|
csv_quote_mode: CsvQuoteMode::All,
|
||||||
|
null_literal: String::new(),
|
||||||
numeric_column_right_align: false,
|
numeric_column_right_align: false,
|
||||||
column_comments: None,
|
column_comments: None,
|
||||||
auto_filter: None,
|
auto_filter: None,
|
||||||
|
|||||||
@@ -208,6 +208,10 @@ pub struct TableImportParseOptions {
|
|||||||
pub last_data_row: Option<usize>,
|
pub last_data_row: Option<usize>,
|
||||||
pub trim_values: Option<bool>,
|
pub trim_values: Option<bool>,
|
||||||
pub empty_string_as_null: Option<bool>,
|
pub empty_string_as_null: Option<bool>,
|
||||||
|
/// 分隔文本里代表 NULL 的字面量。缺省表示使用与导出端一致的默认字面量(`\N`);
|
||||||
|
/// 显式配置空串表示关闭字面量,退回「空字段即 NULL」的旧行为。
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub null_literal: Option<String>,
|
||||||
pub sheet_name: Option<String>,
|
pub sheet_name: Option<String>,
|
||||||
pub sheet_index: Option<usize>,
|
pub sheet_index: Option<usize>,
|
||||||
pub json_shape: Option<TableImportJsonShape>,
|
pub json_shape: Option<TableImportJsonShape>,
|
||||||
@@ -227,6 +231,7 @@ impl Default for TableImportParseOptions {
|
|||||||
last_data_row: None,
|
last_data_row: None,
|
||||||
trim_values: Some(false),
|
trim_values: Some(false),
|
||||||
empty_string_as_null: Some(true),
|
empty_string_as_null: Some(true),
|
||||||
|
null_literal: None,
|
||||||
sheet_name: None,
|
sheet_name: None,
|
||||||
sheet_index: None,
|
sheet_index: None,
|
||||||
json_shape: Some(TableImportJsonShape::Auto),
|
json_shape: Some(TableImportJsonShape::Auto),
|
||||||
@@ -651,14 +656,26 @@ fn unique_import_headers(headers: impl IntoIterator<Item = String>) -> Vec<Strin
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct DelimitedParseConfig {
|
pub struct DelimitedParseConfig {
|
||||||
pub delimiter: u8,
|
pub delimiter: u8,
|
||||||
pub trim_values: bool,
|
pub trim_values: bool,
|
||||||
pub empty_string_as_null: bool,
|
pub empty_string_as_null: bool,
|
||||||
|
/// 命中的字段按 NULL 处理;`None` 表示不按字面量识别 NULL。
|
||||||
|
pub null_literal: Option<String>,
|
||||||
pub row_range: ImportRowRange,
|
pub row_range: ImportRowRange,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 分隔文本里 NULL 字面量的默认值:与导出端共用 [`dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL`],
|
||||||
|
/// 因此 DBX 自己导出的 CSV 默认即可无损导入。
|
||||||
|
fn effective_delimited_null_literal(raw: Option<&str>) -> Option<String> {
|
||||||
|
match raw {
|
||||||
|
None => Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string()),
|
||||||
|
// 显式空串 = 关闭字面量
|
||||||
|
Some(raw) => dbx_formats::csv_export::csv_null_literal(raw).map(str::to_string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub struct ImportRowRange {
|
pub struct ImportRowRange {
|
||||||
pub title_row: Option<usize>,
|
pub title_row: Option<usize>,
|
||||||
@@ -707,10 +724,15 @@ pub fn effective_delimited_config(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let null_literal = effective_delimited_null_literal(options.null_literal.as_deref());
|
||||||
|
|
||||||
Ok(DelimitedParseConfig {
|
Ok(DelimitedParseConfig {
|
||||||
delimiter,
|
delimiter,
|
||||||
trim_values: options.trim_values.unwrap_or(false),
|
trim_values: options.trim_values.unwrap_or(false),
|
||||||
empty_string_as_null: options.empty_string_as_null.unwrap_or(true),
|
// 配了 NULL 字面量时空字段一律是空串:否则字面量刚把 NULL 和空串分开,
|
||||||
|
// 这里又会把空串重新当成 NULL。
|
||||||
|
empty_string_as_null: null_literal.is_none() && options.empty_string_as_null.unwrap_or(true),
|
||||||
|
null_literal,
|
||||||
row_range: effective_import_row_range(options)?,
|
row_range: effective_import_row_range(options)?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -734,9 +756,13 @@ fn unwrap_csv_force_text(value: &str) -> &str {
|
|||||||
inner
|
inner
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn csv_value_with_config(value: &str, config: DelimitedParseConfig) -> serde_json::Value {
|
pub fn csv_value_with_config(value: &str, config: &DelimitedParseConfig) -> serde_json::Value {
|
||||||
let value = if config.trim_values { value.trim() } else { value };
|
let value = if config.trim_values { value.trim() } else { value };
|
||||||
let value = unwrap_csv_force_text(value);
|
let value = unwrap_csv_force_text(value);
|
||||||
|
// 字面量优先:命中即为 NULL,空字段则原样保留为空字符串。
|
||||||
|
if config.null_literal.as_deref() == Some(value) {
|
||||||
|
return serde_json::Value::Null;
|
||||||
|
}
|
||||||
if config.empty_string_as_null && value.is_empty() {
|
if config.empty_string_as_null && value.is_empty() {
|
||||||
serde_json::Value::Null
|
serde_json::Value::Null
|
||||||
} else {
|
} else {
|
||||||
@@ -747,10 +773,11 @@ pub fn csv_value_with_config(value: &str, config: DelimitedParseConfig) -> serde
|
|||||||
pub fn csv_value(value: &str) -> serde_json::Value {
|
pub fn csv_value(value: &str) -> serde_json::Value {
|
||||||
csv_value_with_config(
|
csv_value_with_config(
|
||||||
value,
|
value,
|
||||||
DelimitedParseConfig {
|
&DelimitedParseConfig {
|
||||||
delimiter: b',',
|
delimiter: b',',
|
||||||
trim_values: false,
|
trim_values: false,
|
||||||
empty_string_as_null: true,
|
empty_string_as_null: true,
|
||||||
|
null_literal: None,
|
||||||
row_range: ImportRowRange { title_row: Some(1), data_start_row: 2, last_data_row: None },
|
row_range: ImportRowRange { title_row: Some(1), data_start_row: 2, last_data_row: None },
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -1089,7 +1116,7 @@ fn open_delimited_csv_reader_with_progress(
|
|||||||
|
|
||||||
pub fn parse_delimited_reader<R: std::io::Read>(
|
pub fn parse_delimited_reader<R: std::io::Read>(
|
||||||
reader: R,
|
reader: R,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
preview_limit: usize,
|
preview_limit: usize,
|
||||||
) -> Result<ParsedImportFile, String> {
|
) -> Result<ParsedImportFile, String> {
|
||||||
parse_decoded_delimited_reader(reader, config, preview_limit, TableImportTextEncoding::Utf8)
|
parse_decoded_delimited_reader(reader, config, preview_limit, TableImportTextEncoding::Utf8)
|
||||||
@@ -1097,7 +1124,7 @@ pub fn parse_delimited_reader<R: std::io::Read>(
|
|||||||
|
|
||||||
fn parse_decoded_delimited_reader<R: IoRead>(
|
fn parse_decoded_delimited_reader<R: IoRead>(
|
||||||
reader: R,
|
reader: R,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
preview_limit: usize,
|
preview_limit: usize,
|
||||||
effective_encoding: TableImportTextEncoding,
|
effective_encoding: TableImportTextEncoding,
|
||||||
) -> Result<ParsedImportFile, String> {
|
) -> Result<ParsedImportFile, String> {
|
||||||
@@ -1114,7 +1141,12 @@ pub fn parse_delimited_bytes_with_options(
|
|||||||
) -> Result<ParsedImportFile, String> {
|
) -> Result<ParsedImportFile, String> {
|
||||||
let (encoding, bom_len) = resolve_text_encoding_from_bytes(bytes, options.encoding)?;
|
let (encoding, bom_len) = resolve_text_encoding_from_bytes(bytes, options.encoding)?;
|
||||||
let reader = StrictTranscodingReader::new(std::io::Cursor::new(&bytes[bom_len..]), encoding)?;
|
let reader = StrictTranscodingReader::new(std::io::Cursor::new(&bytes[bom_len..]), encoding)?;
|
||||||
parse_decoded_delimited_reader(reader, effective_delimited_config(source_format, options)?, preview_limit, encoding)
|
parse_decoded_delimited_reader(
|
||||||
|
reader,
|
||||||
|
&effective_delimited_config(source_format, options)?,
|
||||||
|
preview_limit,
|
||||||
|
encoding,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_delimited_file_with_options(
|
pub fn parse_delimited_file_with_options(
|
||||||
@@ -1132,7 +1164,7 @@ pub fn parse_delimited_file_with_options(
|
|||||||
explicit_options.encoding = Some(encoding);
|
explicit_options.encoding = Some(encoding);
|
||||||
let (reader, config, encoding) =
|
let (reader, config, encoding) =
|
||||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||||
return parse_csv_reader(reader, config, preview_limit, encoding);
|
return parse_csv_reader(reader, &config, preview_limit, encoding);
|
||||||
}
|
}
|
||||||
|
|
||||||
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
||||||
@@ -1140,7 +1172,7 @@ pub fn parse_delimited_file_with_options(
|
|||||||
explicit_options.encoding = Some(encoding);
|
explicit_options.encoding = Some(encoding);
|
||||||
let (reader, config, encoding) =
|
let (reader, config, encoding) =
|
||||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||||
match parse_csv_reader(reader, config, preview_limit, encoding) {
|
match parse_csv_reader(reader, &config, preview_limit, encoding) {
|
||||||
Ok(parsed) => return Ok(parsed),
|
Ok(parsed) => return Ok(parsed),
|
||||||
Err(error) if error.starts_with("Invalid byte sequence for ") => continue,
|
Err(error) if error.starts_with("Invalid byte sequence for ") => continue,
|
||||||
Err(error) => return Err(error),
|
Err(error) => return Err(error),
|
||||||
@@ -1150,12 +1182,12 @@ pub fn parse_delimited_file_with_options(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
||||||
parse_csv_reader(reader, config, preview_limit, encoding)
|
parse_csv_reader(reader, &config, preview_limit, encoding)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_csv_reader<R: IoRead>(
|
fn parse_csv_reader<R: IoRead>(
|
||||||
mut reader: csv::Reader<R>,
|
mut reader: csv::Reader<R>,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
preview_limit: usize,
|
preview_limit: usize,
|
||||||
effective_encoding: TableImportTextEncoding,
|
effective_encoding: TableImportTextEncoding,
|
||||||
) -> Result<ParsedImportFile, String> {
|
) -> Result<ParsedImportFile, String> {
|
||||||
@@ -1164,7 +1196,7 @@ fn parse_csv_reader<R: IoRead>(
|
|||||||
|
|
||||||
fn parse_csv_reader_bounded<R: IoRead>(
|
fn parse_csv_reader_bounded<R: IoRead>(
|
||||||
mut reader: csv::Reader<R>,
|
mut reader: csv::Reader<R>,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
preview_limit: usize,
|
preview_limit: usize,
|
||||||
effective_encoding: TableImportTextEncoding,
|
effective_encoding: TableImportTextEncoding,
|
||||||
) -> Result<ParsedImportFile, String> {
|
) -> Result<ParsedImportFile, String> {
|
||||||
@@ -1173,7 +1205,7 @@ fn parse_csv_reader_bounded<R: IoRead>(
|
|||||||
|
|
||||||
fn parse_csv_reader_inner<R: IoRead>(
|
fn parse_csv_reader_inner<R: IoRead>(
|
||||||
reader: &mut csv::Reader<R>,
|
reader: &mut csv::Reader<R>,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
preview_limit: usize,
|
preview_limit: usize,
|
||||||
effective_encoding: TableImportTextEncoding,
|
effective_encoding: TableImportTextEncoding,
|
||||||
count_all_rows: bool,
|
count_all_rows: bool,
|
||||||
@@ -1236,7 +1268,7 @@ fn parse_delimited_preview_file_with_options(
|
|||||||
explicit_options.encoding = Some(encoding);
|
explicit_options.encoding = Some(encoding);
|
||||||
let (reader, config, encoding) =
|
let (reader, config, encoding) =
|
||||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||||
return parse_csv_reader_bounded(reader, config, preview_limit, encoding);
|
return parse_csv_reader_bounded(reader, &config, preview_limit, encoding);
|
||||||
}
|
}
|
||||||
|
|
||||||
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
||||||
@@ -1244,7 +1276,7 @@ fn parse_delimited_preview_file_with_options(
|
|||||||
explicit_options.encoding = Some(encoding);
|
explicit_options.encoding = Some(encoding);
|
||||||
let (reader, config, encoding) =
|
let (reader, config, encoding) =
|
||||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||||
match parse_csv_reader_bounded(reader, config, preview_limit, encoding) {
|
match parse_csv_reader_bounded(reader, &config, preview_limit, encoding) {
|
||||||
Ok(parsed) => return Ok(parsed),
|
Ok(parsed) => return Ok(parsed),
|
||||||
Err(error) if error.starts_with("Invalid byte sequence for ") => continue,
|
Err(error) if error.starts_with("Invalid byte sequence for ") => continue,
|
||||||
Err(error) => return Err(error),
|
Err(error) => return Err(error),
|
||||||
@@ -1254,7 +1286,7 @@ fn parse_delimited_preview_file_with_options(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
||||||
parse_csv_reader_bounded(reader, config, preview_limit, encoding)
|
parse_csv_reader_bounded(reader, &config, preview_limit, encoding)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
|
pub fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||||
@@ -6291,7 +6323,7 @@ fn import_cancelled_progress(
|
|||||||
fn delimited_record_to_row(
|
fn delimited_record_to_row(
|
||||||
record: &csv::StringRecord,
|
record: &csv::StringRecord,
|
||||||
columns_len: usize,
|
columns_len: usize,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
) -> Vec<serde_json::Value> {
|
) -> Vec<serde_json::Value> {
|
||||||
(0..columns_len)
|
(0..columns_len)
|
||||||
.map(|index| {
|
.map(|index| {
|
||||||
@@ -6302,7 +6334,7 @@ fn delimited_record_to_row(
|
|||||||
|
|
||||||
fn delimited_columns_and_first_record<R: std::io::Read>(
|
fn delimited_columns_and_first_record<R: std::io::Read>(
|
||||||
reader: &mut csv::Reader<R>,
|
reader: &mut csv::Reader<R>,
|
||||||
config: DelimitedParseConfig,
|
config: &DelimitedParseConfig,
|
||||||
) -> Result<(Vec<String>, Option<csv::StringRecord>), String> {
|
) -> Result<(Vec<String>, Option<csv::StringRecord>), String> {
|
||||||
let mut columns = Vec::new();
|
let mut columns = Vec::new();
|
||||||
for (index, record) in reader.records().enumerate() {
|
for (index, record) in reader.records().enumerate() {
|
||||||
@@ -6351,7 +6383,7 @@ fn stream_delimited_rows_to_channel(
|
|||||||
// Keep CSV parsing off the async executor while the bounded channel prevents unbounded
|
// Keep CSV parsing off the async executor while the bounded channel prevents unbounded
|
||||||
// accumulation when the database consumer is under load.
|
// accumulation when the database consumer is under load.
|
||||||
let (mut reader, config, _) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
let (mut reader, config, _) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
||||||
let (columns, first_record) = delimited_columns_and_first_record(&mut reader, config)?;
|
let (columns, first_record) = delimited_columns_and_first_record(&mut reader, &config)?;
|
||||||
sender
|
sender
|
||||||
.blocking_send(Ok(DelimitedStreamMessage::Header(columns.clone())))
|
.blocking_send(Ok(DelimitedStreamMessage::Header(columns.clone())))
|
||||||
.map_err(|_| "Delimited import consumer closed before the stream started".to_string())?;
|
.map_err(|_| "Delimited import consumer closed before the stream started".to_string())?;
|
||||||
@@ -6374,7 +6406,7 @@ fn stream_delimited_rows_to_channel(
|
|||||||
if config.row_range.last_data_row.is_some_and(|last| source_row_number > last) {
|
if config.row_range.last_data_row.is_some_and(|last| source_row_number > last) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
pending_rows.push(delimited_record_to_row(&record, columns.len(), config));
|
pending_rows.push(delimited_record_to_row(&record, columns.len(), &config));
|
||||||
if pending_rows.len() >= batch_size {
|
if pending_rows.len() >= batch_size {
|
||||||
sender
|
sender
|
||||||
.blocking_send(Ok(DelimitedStreamMessage::Rows {
|
.blocking_send(Ok(DelimitedStreamMessage::Rows {
|
||||||
@@ -9372,12 +9404,93 @@ mod tests {
|
|||||||
parsed.rows[1],
|
parsed.rows[1],
|
||||||
vec![
|
vec![
|
||||||
serde_json::Value::String("2".to_string()),
|
serde_json::Value::String("2".to_string()),
|
||||||
serde_json::Value::Null,
|
// 默认 NULL 字面量生效后,空字段保留为空字符串,只有字面量才是 NULL
|
||||||
|
serde_json::Value::String(String::new()),
|
||||||
serde_json::Value::String("false".to_string()),
|
serde_json::Value::String("false".to_string()),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn effective_delimited_config_shares_the_export_null_literal_by_default() {
|
||||||
|
let config =
|
||||||
|
effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap();
|
||||||
|
assert_eq!(config.null_literal.as_deref(), Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL));
|
||||||
|
assert!(!config.empty_string_as_null, "NULL 字面量生效时不再把空字段当作 NULL");
|
||||||
|
|
||||||
|
// 显式空串 = 关闭字面量,退回「空字段即 NULL」的旧行为
|
||||||
|
let legacy =
|
||||||
|
TableImportParseOptions { null_literal: Some(String::new()), ..TableImportParseOptions::default() };
|
||||||
|
let config = effective_delimited_config(TableImportSourceFormat::Csv, &legacy).unwrap();
|
||||||
|
assert_eq!(config.null_literal, None);
|
||||||
|
assert!(config.empty_string_as_null);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn csv_null_literal_separates_null_from_an_empty_string() {
|
||||||
|
// `\N` 是 NULL;`""` 与裸空字段都是空字符串
|
||||||
|
let parsed = parse_csv_bytes(b"id,name\n1,\\N\n2,\"\"\n3,\n", 10).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(parsed.total_rows, 3);
|
||||||
|
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||||
|
assert_eq!(parsed.rows[1][1], serde_json::json!(""));
|
||||||
|
assert_eq!(parsed.rows[2][1], serde_json::json!(""));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tsv_export_with_null_literal_round_trips_through_the_importer() {
|
||||||
|
// 导出端(push_tsv_row + 默认字面量)写出的 TSV,按 TSV 规则导入:
|
||||||
|
// NULL 仍是 NULL、空字符串仍是空字符串
|
||||||
|
let mut text = String::from("id\tnote\n");
|
||||||
|
dbx_formats::csv_export::push_tsv_row(
|
||||||
|
&mut text,
|
||||||
|
&[serde_json::json!(1), serde_json::Value::Null],
|
||||||
|
Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL),
|
||||||
|
);
|
||||||
|
text.push('\n');
|
||||||
|
dbx_formats::csv_export::push_tsv_row(
|
||||||
|
&mut text,
|
||||||
|
&[serde_json::json!(2), serde_json::json!("")],
|
||||||
|
Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL),
|
||||||
|
);
|
||||||
|
|
||||||
|
let parsed = parse_delimited_bytes_with_options(
|
||||||
|
text.as_bytes(),
|
||||||
|
TableImportSourceFormat::Tsv,
|
||||||
|
&TableImportParseOptions::default(),
|
||||||
|
10,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||||
|
assert_eq!(parsed.rows[1][1], serde_json::json!(""));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn csv_null_literal_can_be_replaced_or_disabled() {
|
||||||
|
let custom =
|
||||||
|
TableImportParseOptions { null_literal: Some("(null)".to_string()), ..TableImportParseOptions::default() };
|
||||||
|
let parsed =
|
||||||
|
parse_delimited_bytes_with_options(b"id,name\n1,(null)\n2,\n", TableImportSourceFormat::Csv, &custom, 10)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||||
|
assert_eq!(parsed.rows[1][1], serde_json::json!(""));
|
||||||
|
|
||||||
|
let legacy =
|
||||||
|
TableImportParseOptions { null_literal: Some(String::new()), ..TableImportParseOptions::default() };
|
||||||
|
let parsed =
|
||||||
|
parse_delimited_bytes_with_options(b"id,name\n1,\n2,Ada\n", TableImportSourceFormat::Csv, &legacy, 10)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||||
|
assert_eq!(parsed.rows[1][1], serde_json::json!("Ada"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn csv_null_literal_is_not_reported_as_a_missing_value() {
|
||||||
|
// 字面量只按整字段匹配:夹带空格的 `\N ` 不是 NULL,避免误伤真实数据
|
||||||
|
let parsed = parse_csv_bytes(b"id,name\n1, \\N \n", 10).unwrap();
|
||||||
|
assert_eq!(parsed.rows[0][1], serde_json::json!(" \\N "));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn duplicate_csv_headers_map_to_distinct_source_indexes() {
|
fn duplicate_csv_headers_map_to_distinct_source_indexes() {
|
||||||
let parsed = parse_csv_bytes(b"name,name\nAda,Lovelace\n", 10).unwrap();
|
let parsed = parse_csv_bytes(b"name,name\nAda,Lovelace\n", 10).unwrap();
|
||||||
@@ -9627,7 +9740,7 @@ mod tests {
|
|||||||
let config =
|
let config =
|
||||||
effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap();
|
effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap();
|
||||||
|
|
||||||
let preview = parse_csv_reader_bounded(reader, config, 1, TableImportTextEncoding::Utf8).unwrap();
|
let preview = parse_csv_reader_bounded(reader, &config, 1, TableImportTextEncoding::Utf8).unwrap();
|
||||||
|
|
||||||
assert_eq!(preview.columns, vec!["id", "name"]);
|
assert_eq!(preview.columns, vec!["id", "name"]);
|
||||||
assert_eq!(preview.rows, vec![vec![serde_json::json!("1"), serde_json::json!("Ada")]]);
|
assert_eq!(preview.rows, vec![vec![serde_json::json!("1"), serde_json::json!("Ada")]]);
|
||||||
@@ -9754,7 +9867,9 @@ mod tests {
|
|||||||
assert_eq!(parsed.columns, vec!["column_1", "column_2"]);
|
assert_eq!(parsed.columns, vec!["column_1", "column_2"]);
|
||||||
assert_eq!(parsed.total_rows, 2);
|
assert_eq!(parsed.total_rows, 2);
|
||||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("Ada")]);
|
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("Ada")]);
|
||||||
assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::Value::Null]);
|
// 默认 NULL 字面量生效时,显式打开的「空字符串作为 NULL」不再接管空字段:
|
||||||
|
// 否则字面量刚把 NULL 和空串分开,这里又会把空串重新写成 NULL。
|
||||||
|
assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::json!("")]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_onc
|
|||||||
execution_id: Some(format!("live-clickhouse-query-export-{suffix}")),
|
execution_id: Some(format!("live-clickhouse-query-export-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
|
|||||||
@@ -477,6 +477,7 @@ async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplic
|
|||||||
execution_id: Some(format!("live-mysql-query-export-{suffix}")),
|
execution_id: Some(format!("live-mysql-query-export-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -572,6 +573,7 @@ async fn live_mysql_csv_temporal_export_round_trip_preserves_dbx_force_text_valu
|
|||||||
execution_id: Some(format!("live-mysql-issue-8803-{suffix}")),
|
execution_id: Some(format!("live-mysql-issue-8803-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -691,6 +693,7 @@ async fn live_mysql_xlsx_export_can_outlive_query_timeout_while_rows_keep_arrivi
|
|||||||
execution_id: Some(format!("live-mysql-query-export-timeout-{suffix}")),
|
execution_id: Some(format!("live-mysql-query-export-timeout-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
|
|||||||
@@ -150,6 +150,7 @@ async fn live_postgres_query_result_export_uses_single_streamed_query() {
|
|||||||
execution_id: Some(format!("live-postgres-query-export-{suffix}")),
|
execution_id: Some(format!("live-postgres-query-export-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -238,6 +239,7 @@ async fn live_postgres_query_result_xlsx_preserves_temporal_cell_types() {
|
|||||||
execution_id: Some(format!("live-postgres-xlsx-temporal-{suffix}")),
|
execution_id: Some(format!("live-postgres-xlsx-temporal-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -315,6 +317,7 @@ async fn live_postgres_numeric_xlsx_ignores_fractional_trailing_zeros() {
|
|||||||
execution_id: Some(format!("live-postgres-xlsx-numeric-{suffix}")),
|
execution_id: Some(format!("live-postgres-xlsx-numeric-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -397,6 +400,7 @@ async fn live_postgres_truncated_batch_result_export_replays_safe_temp_setup() {
|
|||||||
execution_id: Some(format!("temp-export-csv-{short_suffix}")),
|
execution_id: Some(format!("temp-export-csv-{short_suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -481,6 +485,7 @@ async fn live_postgres_xlsx_export_can_outlive_query_timeout_while_rows_keep_arr
|
|||||||
execution_id: Some(format!("live-postgres-query-export-timeout-{suffix}")),
|
execution_id: Some(format!("live-postgres-query-export-timeout-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
@@ -556,6 +561,7 @@ async fn live_postgres_stream_still_times_out_without_progress_and_recovers() {
|
|||||||
execution_id: Some(format!("live-postgres-query-export-stall-{suffix}")),
|
execution_id: Some(format!("live-postgres-query-export-stall-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
|
|||||||
@@ -1215,6 +1215,7 @@ async fn live_sqlserver_query_result_export_streams_cte_query_to_csv() {
|
|||||||
execution_id: Some(format!("live-sqlserver-export-{suffix}")),
|
execution_id: Some(format!("live-sqlserver-export-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
|
|||||||
@@ -129,6 +129,7 @@ async fn live_sqlserver_xlsx_export_can_outlive_query_timeout_while_rows_keep_ar
|
|||||||
execution_id: Some(format!("live-sqlserver-xlsx-{suffix}")),
|
execution_id: Some(format!("live-sqlserver-xlsx-{suffix}")),
|
||||||
date_time_format: None,
|
date_time_format: None,
|
||||||
csv_quote_mode: Default::default(),
|
csv_quote_mode: Default::default(),
|
||||||
|
null_literal: String::new(),
|
||||||
export_table_name: None,
|
export_table_name: None,
|
||||||
export_column_types: None,
|
export_column_types: None,
|
||||||
selected_columns: None,
|
selected_columns: None,
|
||||||
|
|||||||
@@ -11,6 +11,27 @@ pub enum CsvQuoteMode {
|
|||||||
Necessary,
|
Necessary,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// CSV 里 NULL 的默认字面量(`\N`,MySQL `SELECT ... INTO OUTFILE` 惯例)。
|
||||||
|
///
|
||||||
|
/// 默认值必须同时被导出与导入两侧引用:导出写出该字面量、导入按该字面量还原 NULL,
|
||||||
|
/// 空字符串则始终写成 `""`,这样「空字符串」与「NULL」在 CSV 中不再互相冒充。
|
||||||
|
pub const DEFAULT_CSV_NULL_LITERAL: &str = "\\N";
|
||||||
|
|
||||||
|
/// 把导出/导入配置里的原始字面量解析成实际生效的值:空串表示「不写 NULL 标记」,
|
||||||
|
/// 即沿用历史行为(NULL 写成空字段,无法与空字符串区分)。
|
||||||
|
pub fn csv_null_literal(raw: &str) -> Option<&str> {
|
||||||
|
if raw.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 导出/导入配置里 NULL 字面量的 serde 默认值:未显式配置时用 [`DEFAULT_CSV_NULL_LITERAL`]。
|
||||||
|
pub fn default_csv_null_literal() -> String {
|
||||||
|
DEFAULT_CSV_NULL_LITERAL.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
/// CSV 转义直写目标 buffer:包引号 + 内部 `"` 翻倍。值不含 `"` 时整段拷贝,
|
/// CSV 转义直写目标 buffer:包引号 + 内部 `"` 翻倍。值不含 `"` 时整段拷贝,
|
||||||
/// 不做 replace 分配(逐批流式导出对每个单元格调用,是导出热路径)。
|
/// 不做 replace 分配(逐批流式导出对每个单元格调用,是导出热路径)。
|
||||||
fn push_csv_escaped_content(out: &mut String, value: &str) {
|
fn push_csv_escaped_content(out: &mut String, value: &str) {
|
||||||
@@ -67,7 +88,26 @@ pub fn push_csv_text_value(out: &mut String, value: &Value) {
|
|||||||
out.push('"');
|
out.push('"');
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_csv_value_with_quote_mode(out: &mut String, value: &Value, quote_mode: CsvQuoteMode, quote_null: bool) {
|
fn push_csv_value_with_quote_mode(
|
||||||
|
out: &mut String,
|
||||||
|
value: &Value,
|
||||||
|
quote_mode: CsvQuoteMode,
|
||||||
|
quote_null: bool,
|
||||||
|
null_literal: Option<&str>,
|
||||||
|
) {
|
||||||
|
if value.is_null() {
|
||||||
|
// 配置了 NULL 字面量时,NULL 写成该字面量;空字符串仍然写成 `""`。
|
||||||
|
// 两者不再互相冒充,导出→导入才能无损往返。
|
||||||
|
//
|
||||||
|
// 字面量必须尽量裸写:PostgreSQL 的 COPY ... FORMAT csv 从不把带引号的值当成 NULL
|
||||||
|
// (`"\N"` 是普通字符串,整数列直接报错),ClickHouse 的 CSV 读取更是在引号内遇到
|
||||||
|
// `\N` 就解析失败;只有裸 `\N` 才是这两者与 MySQL 共同识别的 NULL。字面量自身含
|
||||||
|
// 分隔符/引号/换行时才加引号(此时它本来也不再是标准 NULL 标记)。
|
||||||
|
if let Some(literal) = null_literal {
|
||||||
|
push_csv_field(out, literal, CsvQuoteMode::Necessary);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
if quote_mode == CsvQuoteMode::All {
|
if quote_mode == CsvQuoteMode::All {
|
||||||
if value.is_null() && !quote_null {
|
if value.is_null() && !quote_null {
|
||||||
return;
|
return;
|
||||||
@@ -87,13 +127,6 @@ fn push_csv_value_with_quote_mode(out: &mut String, value: &Value, quote_mode: C
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_csv_value(out: &mut String, value: &Value) {
|
|
||||||
if value.is_null() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
push_csv_text_value(out, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// TSV 转义直写:仅含特殊字符时包引号(语义与原 escape_tsv 一致)。
|
/// TSV 转义直写:仅含特殊字符时包引号(语义与原 escape_tsv 一致)。
|
||||||
fn push_tsv_escaped(out: &mut String, value: &str) {
|
fn push_tsv_escaped(out: &mut String, value: &str) {
|
||||||
if value.contains('\t') || value.contains('\n') || value.contains('\r') || value.contains('"') {
|
if value.contains('\t') || value.contains('\n') || value.contains('\r') || value.contains('"') {
|
||||||
@@ -103,7 +136,14 @@ fn push_tsv_escaped(out: &mut String, value: &str) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_tsv_value(out: &mut String, value: &Value) {
|
fn push_tsv_value(out: &mut String, value: &Value, null_literal: Option<&str>) {
|
||||||
|
if value.is_null() {
|
||||||
|
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返。
|
||||||
|
if let Some(literal) = null_literal {
|
||||||
|
push_tsv_escaped(out, literal);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
match value {
|
match value {
|
||||||
Value::Null => {}
|
Value::Null => {}
|
||||||
Value::String(v) => push_tsv_escaped(out, v),
|
Value::String(v) => push_tsv_escaped(out, v),
|
||||||
@@ -115,12 +155,12 @@ fn push_tsv_value(out: &mut String, value: &Value) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn push_tsv_row(out: &mut String, row: &[Value]) {
|
pub fn push_tsv_row(out: &mut String, row: &[Value], null_literal: Option<&str>) {
|
||||||
for (cell_index, cell) in row.iter().enumerate() {
|
for (cell_index, cell) in row.iter().enumerate() {
|
||||||
if cell_index > 0 {
|
if cell_index > 0 {
|
||||||
out.push('\t');
|
out.push('\t');
|
||||||
}
|
}
|
||||||
push_tsv_value(out, cell);
|
push_tsv_value(out, cell, null_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,12 +186,6 @@ fn escape_csv_with_quote_mode(value: &str, quote_mode: CsvQuoteMode) -> String {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_csv_value(value: &Value) -> String {
|
|
||||||
let mut out = String::new();
|
|
||||||
push_csv_value(&mut out, value);
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg_attr(not(test), allow(dead_code))]
|
#[cfg_attr(not(test), allow(dead_code))]
|
||||||
pub fn escape_tsv(value: &str) -> String {
|
pub fn escape_tsv(value: &str) -> String {
|
||||||
let mut out = String::with_capacity(value.len());
|
let mut out = String::with_capacity(value.len());
|
||||||
@@ -159,19 +193,19 @@ pub fn escape_tsv(value: &str) -> String {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>], null_literal: Option<&str>) {
|
||||||
for (row_index, row) in rows.iter().enumerate() {
|
for (row_index, row) in rows.iter().enumerate() {
|
||||||
if row_index > 0 {
|
if row_index > 0 {
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
}
|
}
|
||||||
push_tsv_row(out, row);
|
push_tsv_row(out, row, null_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg_attr(not(test), allow(dead_code))]
|
#[cfg_attr(not(test), allow(dead_code))]
|
||||||
pub fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
pub fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
||||||
let mut out = String::with_capacity(estimated_rows_capacity(rows));
|
let mut out = String::with_capacity(estimated_rows_capacity(rows));
|
||||||
push_tsv_rows(&mut out, rows);
|
push_tsv_rows(&mut out, rows, None);
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -186,7 +220,7 @@ pub fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
|||||||
push_tsv_escaped(&mut out, column);
|
push_tsv_escaped(&mut out, column);
|
||||||
}
|
}
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
push_tsv_rows(&mut out, rows);
|
push_tsv_rows(&mut out, rows, None);
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,11 +232,21 @@ pub fn push_query_result_csv_row(out: &mut String, row: &[Value]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) {
|
pub fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) {
|
||||||
|
push_query_result_csv_row_with_options(out, row, quote_mode, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 与 [`push_query_result_csv_row_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||||
|
pub fn push_query_result_csv_row_with_options(
|
||||||
|
out: &mut String,
|
||||||
|
row: &[Value],
|
||||||
|
quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
|
) {
|
||||||
for (cell_index, cell) in row.iter().enumerate() {
|
for (cell_index, cell) in row.iter().enumerate() {
|
||||||
if cell_index > 0 {
|
if cell_index > 0 {
|
||||||
out.push(',');
|
out.push(',');
|
||||||
}
|
}
|
||||||
push_csv_value_with_quote_mode(out, cell, quote_mode, false);
|
push_csv_value_with_quote_mode(out, cell, quote_mode, false, null_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,11 +255,21 @@ pub fn push_table_csv_row(out: &mut String, row: &[Value]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn push_table_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) {
|
pub fn push_table_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) {
|
||||||
|
push_table_csv_row_with_options(out, row, quote_mode, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 与 [`push_table_csv_row_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||||
|
pub fn push_table_csv_row_with_options(
|
||||||
|
out: &mut String,
|
||||||
|
row: &[Value],
|
||||||
|
quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
|
) {
|
||||||
for (cell_index, cell) in row.iter().enumerate() {
|
for (cell_index, cell) in row.iter().enumerate() {
|
||||||
if cell_index > 0 {
|
if cell_index > 0 {
|
||||||
out.push(',');
|
out.push(',');
|
||||||
}
|
}
|
||||||
push_csv_value_with_quote_mode(out, cell, quote_mode, true);
|
push_csv_value_with_quote_mode(out, cell, quote_mode, true, null_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,6 +308,19 @@ pub fn format_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec<Value>], quote_mode: CsvQuoteMode) -> String {
|
pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec<Value>], quote_mode: CsvQuoteMode) -> String {
|
||||||
|
format_csv_with_options(columns, rows, quote_mode, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 与 [`format_csv_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||||
|
///
|
||||||
|
/// 表数据导出的首批行走这里,后续分批走 [`push_table_csv_row_with_options`]:
|
||||||
|
/// 两处必须收到同一个字面量,否则同一个文件里 NULL 会前后不一致。
|
||||||
|
pub fn format_csv_with_options(
|
||||||
|
columns: &[String],
|
||||||
|
rows: &[Vec<Value>],
|
||||||
|
quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
|
) -> String {
|
||||||
let mut out = String::with_capacity(
|
let mut out = String::with_capacity(
|
||||||
estimated_rows_capacity(rows).saturating_add(columns.len().saturating_mul(12)).min(ROWS_CAPACITY_ESTIMATE_MAX),
|
estimated_rows_capacity(rows).saturating_add(columns.len().saturating_mul(12)).min(ROWS_CAPACITY_ESTIMATE_MAX),
|
||||||
);
|
);
|
||||||
@@ -268,7 +335,7 @@ pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec<Value>], quote
|
|||||||
if row_index > 0 {
|
if row_index > 0 {
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
}
|
}
|
||||||
push_query_result_csv_row_with_quote_mode(&mut out, row, quote_mode);
|
push_query_result_csv_row_with_options(&mut out, row, quote_mode, null_literal);
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
@@ -282,7 +349,17 @@ pub fn format_query_result_csv_with_quote_mode(
|
|||||||
rows: &[Vec<Value>],
|
rows: &[Vec<Value>],
|
||||||
quote_mode: CsvQuoteMode,
|
quote_mode: CsvQuoteMode,
|
||||||
) -> String {
|
) -> String {
|
||||||
format_csv_with_quote_mode(columns, rows, quote_mode)
|
format_csv_with_options(columns, rows, quote_mode, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 与 [`format_query_result_csv_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||||
|
pub fn format_query_result_csv_with_options(
|
||||||
|
columns: &[String],
|
||||||
|
rows: &[Vec<Value>],
|
||||||
|
quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
|
) -> String {
|
||||||
|
format_csv_with_options(columns, rows, quote_mode, null_literal)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn write_csv_text_row(
|
pub fn write_csv_text_row(
|
||||||
@@ -305,20 +382,26 @@ pub fn write_csv_value_row(
|
|||||||
writer: &mut impl Write,
|
writer: &mut impl Write,
|
||||||
values: impl IntoIterator<Item = Value>,
|
values: impl IntoIterator<Item = Value>,
|
||||||
quote_mode: CsvQuoteMode,
|
quote_mode: CsvQuoteMode,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
write_csv_value_row_with_options(writer, values, quote_mode, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 与 [`write_csv_value_row`] 相同,但允许把 NULL 写成指定字面量。
|
||||||
|
pub fn write_csv_value_row_with_options(
|
||||||
|
writer: &mut impl Write,
|
||||||
|
values: impl IntoIterator<Item = Value>,
|
||||||
|
quote_mode: CsvQuoteMode,
|
||||||
|
null_literal: Option<&str>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let mut first = true;
|
let mut first = true;
|
||||||
|
let mut formatted = String::new();
|
||||||
for value in values {
|
for value in values {
|
||||||
if !first {
|
if !first {
|
||||||
writer.write_all(b",").map_err(|err| err.to_string())?;
|
writer.write_all(b",").map_err(|err| err.to_string())?;
|
||||||
}
|
}
|
||||||
first = false;
|
first = false;
|
||||||
let formatted = if quote_mode == CsvQuoteMode::All {
|
formatted.clear();
|
||||||
format_csv_value(&value)
|
push_csv_value_with_quote_mode(&mut formatted, &value, quote_mode, false, null_literal);
|
||||||
} else {
|
|
||||||
let mut formatted = String::new();
|
|
||||||
push_csv_value_with_quote_mode(&mut formatted, &value, quote_mode, false);
|
|
||||||
formatted
|
|
||||||
};
|
|
||||||
writer.write_all(formatted.as_bytes()).map_err(|err| err.to_string())?;
|
writer.write_all(formatted.as_bytes()).map_err(|err| err.to_string())?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -327,8 +410,8 @@ pub fn write_csv_value_row(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
format_csv, format_csv_with_quote_mode, format_query_result_csv, format_query_result_csv_rows, format_tsv,
|
format_csv, format_csv_with_options, format_csv_with_quote_mode, format_query_result_csv,
|
||||||
CsvQuoteMode,
|
format_query_result_csv_rows, format_tsv, CsvQuoteMode, DEFAULT_CSV_NULL_LITERAL,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
@@ -434,6 +517,104 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_null_literal_is_the_mysql_outfile_convention() {
|
||||||
|
assert_eq!(DEFAULT_CSV_NULL_LITERAL, "\\N");
|
||||||
|
// 空串表示「不写 NULL 标记」,即沿用历史行为
|
||||||
|
assert_eq!(super::csv_null_literal(""), None);
|
||||||
|
assert_eq!(super::csv_null_literal(DEFAULT_CSV_NULL_LITERAL), Some("\\N"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tsv_null_literal_separates_null_from_an_empty_string() {
|
||||||
|
let row = vec![Value::Null, json!("")];
|
||||||
|
|
||||||
|
// 未配置字面量(旧行为):NULL 与空串都写空字段,二者在文件里无法区分
|
||||||
|
let mut out = String::new();
|
||||||
|
super::push_tsv_row(&mut out, &row, None);
|
||||||
|
assert_eq!(out, "\t");
|
||||||
|
|
||||||
|
// 配置字面量后:NULL 写成字面量,空字符串仍是空字段
|
||||||
|
let mut out = String::new();
|
||||||
|
super::push_tsv_row(&mut out, &row, Some(DEFAULT_CSV_NULL_LITERAL));
|
||||||
|
assert_eq!(out, "\\N\t");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn null_literal_stops_null_from_impersonating_an_empty_string() {
|
||||||
|
let columns = vec!["a".to_string(), "b".to_string()];
|
||||||
|
let row = vec![Value::Null, json!("")];
|
||||||
|
|
||||||
|
// 未配置字面量(旧行为):NULL 写成裸空字段,空串写成 `""`,
|
||||||
|
// 文件里看似可区分,但导入端拿不到引号信息,两者最终都会变成 NULL
|
||||||
|
assert_eq!(
|
||||||
|
format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, None),
|
||||||
|
"\"a\",\"b\"\n,\"\""
|
||||||
|
);
|
||||||
|
|
||||||
|
// 配置字面量后:NULL 写成裸字面量,空串写成 `""`;即使 quote mode = All 也不给
|
||||||
|
// 字面量加引号,否则 PostgreSQL/ClickHouse 的 CSV 读取不认它是 NULL
|
||||||
|
assert_eq!(
|
||||||
|
format_csv_with_options(
|
||||||
|
&columns,
|
||||||
|
std::slice::from_ref(&row),
|
||||||
|
CsvQuoteMode::All,
|
||||||
|
Some(DEFAULT_CSV_NULL_LITERAL)
|
||||||
|
),
|
||||||
|
"\"a\",\"b\"\n\\N,\"\""
|
||||||
|
);
|
||||||
|
// Necessary 模式:空串裸写为空字段,但 NULL 有独立字面量,仍然可区分
|
||||||
|
assert_eq!(
|
||||||
|
format_csv_with_options(
|
||||||
|
&columns,
|
||||||
|
std::slice::from_ref(&row),
|
||||||
|
CsvQuoteMode::Necessary,
|
||||||
|
Some(DEFAULT_CSV_NULL_LITERAL)
|
||||||
|
),
|
||||||
|
"a,b\n\\N,"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn null_literal_is_quoted_only_when_it_needs_quoting() {
|
||||||
|
let columns = vec!["a".to_string()];
|
||||||
|
let row = vec![Value::Null];
|
||||||
|
|
||||||
|
// 标准字面量裸写(PostgreSQL/ClickHouse/MySQL 共同识别的 NULL 写法)
|
||||||
|
assert_eq!(
|
||||||
|
format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, Some("\\N")),
|
||||||
|
"\"a\"\n\\N"
|
||||||
|
);
|
||||||
|
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
|
||||||
|
assert_eq!(
|
||||||
|
format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, Some("a,b")),
|
||||||
|
"\"a\"\n\"a,b\""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn null_literal_applies_to_streaming_row_writers_too() {
|
||||||
|
let row = vec![Value::Null, json!("NULL"), json!("")];
|
||||||
|
|
||||||
|
let mut table = String::new();
|
||||||
|
super::push_table_csv_row_with_options(&mut table, &row, CsvQuoteMode::All, Some(DEFAULT_CSV_NULL_LITERAL));
|
||||||
|
assert_eq!(table, "\\N,\"NULL\",\"\"");
|
||||||
|
|
||||||
|
let mut query = String::new();
|
||||||
|
super::push_query_result_csv_row_with_options(
|
||||||
|
&mut query,
|
||||||
|
&row,
|
||||||
|
CsvQuoteMode::All,
|
||||||
|
Some(DEFAULT_CSV_NULL_LITERAL),
|
||||||
|
);
|
||||||
|
assert_eq!(query, "\\N,\"NULL\",\"\"");
|
||||||
|
|
||||||
|
let mut written = Vec::new();
|
||||||
|
super::write_csv_value_row_with_options(&mut written, row, CsvQuoteMode::All, Some(DEFAULT_CSV_NULL_LITERAL))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(String::from_utf8(written).unwrap(), "\\N,\"NULL\",\"\"");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reusable_row_buffers_match_batch_formatters() {
|
fn reusable_row_buffers_match_batch_formatters() {
|
||||||
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
|
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
|
||||||
@@ -447,7 +628,7 @@ mod tests {
|
|||||||
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
|
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
|
||||||
|
|
||||||
let mut tsv = String::new();
|
let mut tsv = String::new();
|
||||||
super::push_tsv_row(&mut tsv, &row);
|
super::push_tsv_row(&mut tsv, &row, None);
|
||||||
assert_eq!(tsv, super::format_tsv_rows(&[row]));
|
assert_eq!(tsv, super::format_tsv_rows(&[row]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 128 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 175 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 145 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 156 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 137 KiB |
+21
-3
@@ -2,7 +2,9 @@ import { strict as assert } from "node:assert";
|
|||||||
import { test } from "vitest";
|
import { test } from "vitest";
|
||||||
import { formatCsv } from "../../apps/desktop/src/lib/export/exportFormats.ts";
|
import { formatCsv } from "../../apps/desktop/src/lib/export/exportFormats.ts";
|
||||||
|
|
||||||
test("formatCsv writes database null as an empty cell and preserves literal NULL", () => {
|
// NULL 必须写成独立字面量、空字符串才写成空字段,否则导出再导入时 '' 会被还原成
|
||||||
|
// NULL,写回 NOT NULL DEFAULT '' 的列就会报「不允许为 null」。
|
||||||
|
test("formatCsv writes database null as the shared NULL literal and keeps empty strings empty", () => {
|
||||||
assert.equal(
|
assert.equal(
|
||||||
formatCsv(
|
formatCsv(
|
||||||
["id", "note"],
|
["id", "note"],
|
||||||
@@ -12,7 +14,22 @@ test("formatCsv writes database null as an empty cell and preserves literal NULL
|
|||||||
[3, "NULL"],
|
[3, "NULL"],
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
'"id","note"\n"1",\n"2",""\n"3","NULL"',
|
'"id","note"\n"1",\\N\n"2",""\n"3","NULL"',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("formatCsv keeps the legacy empty-cell NULL when the literal is explicitly disabled", () => {
|
||||||
|
assert.equal(
|
||||||
|
formatCsv(
|
||||||
|
["id", "note"],
|
||||||
|
[
|
||||||
|
[1, null],
|
||||||
|
[2, ""],
|
||||||
|
],
|
||||||
|
"all",
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
'"id","note"\n"1",\n"2",""',
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -23,9 +40,10 @@ test("formatCsv only quotes fields with CSV special characters in necessary mode
|
|||||||
[
|
[
|
||||||
[2085252644, "延庆县", "plain"],
|
[2085252644, "延庆县", "plain"],
|
||||||
[2085252645, "门头沟区", 'line 1\n"line 2"'],
|
[2085252645, "门头沟区", 'line 1\n"line 2"'],
|
||||||
|
[2085252646, "昌平区", null],
|
||||||
],
|
],
|
||||||
"necessary",
|
"necessary",
|
||||||
),
|
),
|
||||||
'id,"district,name",note\n2085252644,延庆县,plain\n2085252645,门头沟区,"line 1\n""line 2"""',
|
'id,"district,name",note\n2085252644,延庆县,plain\n2085252645,门头沟区,"line 1\n""line 2"""\n2085252646,昌平区,\\N',
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -2,7 +2,8 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use crate::commands::connection::AppState;
|
use crate::commands::connection::AppState;
|
||||||
use dbx_core::csv_export::{
|
use dbx_core::csv_export::{
|
||||||
export_table_data_csv_core, format_query_result_csv_with_quote_mode, CsvQuoteMode, TableCsvExportOptions,
|
csv_null_literal, default_csv_null_literal, export_table_data_csv_core, format_query_result_csv_with_options,
|
||||||
|
CsvQuoteMode, TableCsvExportOptions,
|
||||||
};
|
};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -16,12 +17,21 @@ pub struct QueryResultCsvExportRequest {
|
|||||||
pub rows: Vec<Vec<Value>>,
|
pub rows: Vec<Vec<Value>>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub csv_quote_mode: CsvQuoteMode,
|
pub csv_quote_mode: CsvQuoteMode,
|
||||||
|
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||||
|
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。
|
||||||
|
#[serde(default = "default_csv_null_literal")]
|
||||||
|
pub null_literal: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
pub async fn export_query_result_csv(request: QueryResultCsvExportRequest) -> Result<(), String> {
|
pub async fn export_query_result_csv(request: QueryResultCsvExportRequest) -> Result<(), String> {
|
||||||
tauri::async_runtime::spawn_blocking(move || {
|
tauri::async_runtime::spawn_blocking(move || {
|
||||||
let csv = format_query_result_csv_with_quote_mode(&request.columns, &request.rows, request.csv_quote_mode);
|
let csv = format_query_result_csv_with_options(
|
||||||
|
&request.columns,
|
||||||
|
&request.rows,
|
||||||
|
request.csv_quote_mode,
|
||||||
|
csv_null_literal(&request.null_literal),
|
||||||
|
);
|
||||||
std::fs::write(&request.file_path, format!("\u{FEFF}{csv}")).map_err(|err| err.to_string())
|
std::fs::write(&request.file_path, format!("\u{FEFF}{csv}")).map_err(|err| err.to_string())
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
|||||||
Reference in new issue
Block a user