feat(import): add SQL script import with dialect-aware parsing
This commit is contained in:
1 parent
ab4e747bf9
commit
dfd4d3b7ff
15 files changed
+809
-12
No files matched your search
@@ -8,7 +8,7 @@ import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { AlertTriangle, ArrowLeft, ArrowRight, Check, CheckCircle2, FileJson, FileSpreadsheet, FileText, FileUp, Loader2, RefreshCw, Square, Upload, X } from "@lucide/vue";
|
||||
import { AlertTriangle, ArrowLeft, ArrowRight, Check, CheckCircle2, FileCode, FileJson, FileSpreadsheet, FileText, FileUp, Loader2, RefreshCw, Square, Upload, X } from "@lucide/vue";
|
||||
import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import { useSettingsStore } from "@/stores/settingsStore";
|
||||
import { useToast } from "@/composables/useToast";
|
||||
@@ -109,6 +109,7 @@ const formatOptions: Array<{ value: api.TableImportSourceFormat; icon: any; labe
|
||||
{ value: "delimited", icon: FileText, labelKey: "tableImport.formatDelimited", descriptionKey: "tableImport.formatDelimitedDescription" },
|
||||
{ value: "json", icon: FileJson, labelKey: "tableImport.formatJson", descriptionKey: "tableImport.formatJsonDescription" },
|
||||
{ value: "excel", icon: FileSpreadsheet, labelKey: "tableImport.formatExcel", descriptionKey: "tableImport.formatExcelDescription" },
|
||||
{ value: "sql", icon: FileCode, labelKey: "tableImport.formatSql", descriptionKey: "tableImport.formatSqlDescription" },
|
||||
];
|
||||
|
||||
const encodingOptions: Array<{ value: api.TableImportTextEncoding; labelKey: string }> = [
|
||||
@@ -275,6 +276,7 @@ function detectFormat(name: string): api.TableImportSourceFormat {
|
||||
if (lower.endsWith(".txt")) return "delimited";
|
||||
if (lower.endsWith(".json")) return "json";
|
||||
if (lower.endsWith(".xls") || lower.endsWith(".xlsx") || lower.endsWith(".xlsm")) return "excel";
|
||||
if (lower.endsWith(".sql")) return "sql";
|
||||
return "csv";
|
||||
}
|
||||
|
||||
@@ -317,12 +319,14 @@ function taskParseOptions(format: api.TableImportSourceFormat, sheetName = ""):
|
||||
emptyStringAsNull: emptyStringAsNull.value,
|
||||
sheetName,
|
||||
jsonShape: jsonShape.value,
|
||||
databaseType: structureDatabaseType.value,
|
||||
});
|
||||
}
|
||||
|
||||
function importParseOptions(format: api.TableImportSourceFormat, currentPreview: api.TableImportPreview, sheetName = ""): api.TableImportParseOptions {
|
||||
const options = taskParseOptions(format, sheetName);
|
||||
if (isDelimitedFormat(format) && currentPreview.totalRowsExact !== false && options.encoding === "auto" && currentPreview.effectiveEncoding) {
|
||||
// SQL 脚本与分隔文本一样依赖文本编码检测,导入时沿用预览阶段确定的编码避免二次检测结果不一致
|
||||
if ((isDelimitedFormat(format) || format === "sql") && currentPreview.totalRowsExact !== false && options.encoding === "auto" && currentPreview.effectiveEncoding) {
|
||||
options.encoding = currentPreview.effectiveEncoding;
|
||||
}
|
||||
return options;
|
||||
@@ -615,10 +619,11 @@ async function selectFile() {
|
||||
const selected = await open({
|
||||
multiple: targetMode.value === "create",
|
||||
filters: [
|
||||
{ name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls"] },
|
||||
{ name: "Data files", extensions: ["csv", "tsv", "txt", "json", "xlsx", "xlsm", "xls", "sql"] },
|
||||
{ name: "Text", extensions: ["csv", "tsv", "txt"] },
|
||||
{ name: "JSON", extensions: ["json"] },
|
||||
{ name: "Excel", extensions: ["xlsx", "xlsm", "xls"] },
|
||||
{ name: "SQL", extensions: ["sql"] },
|
||||
],
|
||||
});
|
||||
if (!selected) return;
|
||||
@@ -939,7 +944,8 @@ async function reloadBatchPreviewsForEncoding() {
|
||||
errorMessage.value = "";
|
||||
try {
|
||||
for (const task of batchTasks.value) {
|
||||
if (!isDelimitedFormat(task.format)) continue;
|
||||
// SQL 脚本同样是文本源,编码变化时需要重新预览
|
||||
if (!isDelimitedFormat(task.format) && task.format !== "sql") continue;
|
||||
const reusableSource = task.preview.sourceRef ? task.preview.filePath : task.source;
|
||||
const nextPreview = await api.previewTableImportFile(reusableSource, {
|
||||
sourceRef: task.preview.sourceRef || null,
|
||||
@@ -1018,7 +1024,7 @@ watch(rawProgressPercent, (percent) => {
|
||||
|
||||
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1">
|
||||
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
|
||||
<input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
|
||||
<input ref="fileInput" type="file" accept=".csv,.tsv,.txt,.json,.xlsx,.xlsm,.xls,.sql" :multiple="targetMode === 'create'" class="hidden" @change="handleFileInputChange" />
|
||||
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
|
||||
<span class="shrink-0 text-xs text-muted-foreground">{{ t("tableImport.target") }}</span>
|
||||
<span class="min-w-0 truncate text-sm font-medium">
|
||||
@@ -1092,7 +1098,7 @@ watch(rawProgressPercent, (percent) => {
|
||||
{{ t("tableImport.selectFile") }}
|
||||
</Button>
|
||||
</div>
|
||||
<div class="grid grid-cols-5 gap-2">
|
||||
<div class="grid grid-cols-6 gap-2">
|
||||
<button v-for="format in formatOptions" :key="format.value" type="button" class="min-h-20 rounded-md border px-3 py-2 text-left" :class="sourceFormat === format.value ? 'border-primary bg-primary/5' : 'hover:bg-muted/30'" @click="sourceFormat = format.value">
|
||||
<component :is="format.icon" class="mb-2 h-4 w-4 text-muted-foreground" />
|
||||
<div class="text-xs font-medium">{{ t(format.labelKey) }}</div>
|
||||
@@ -1194,6 +1200,25 @@ watch(rawProgressPercent, (percent) => {
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div v-else-if="sourceFormat === 'sql'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
|
||||
<div class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.encoding") }}</Label>
|
||||
<Select :model-value="textEncoding" @update:model-value="(value: any) => (textEncoding = value)">
|
||||
<SelectTrigger class="h-8 text-xs">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem v-for="option in encodingOptions" :key="option.value" :value="option.value">
|
||||
{{ t(option.labelKey) }}
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div v-if="textEncoding === 'auto' && preview?.effectiveEncoding" class="truncate text-[11px] text-muted-foreground" :title="t('tableImport.encodingDetected', { encoding: encodingLabel(preview.effectiveEncoding) })">
|
||||
{{ t("tableImport.encodingDetected", { encoding: encodingLabel(preview.effectiveEncoding) }) }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="sourceFormat === 'json'" class="grid grid-cols-2 gap-3 rounded-md border p-3">
|
||||
<div class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.jsonShape") }}</Label>
|
||||
|
||||
@@ -5338,6 +5338,8 @@ export default {
|
||||
formatJsonDescription: "Object or array rows",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "INSERT statements in SQL scripts",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Encoding",
|
||||
|
||||
@@ -5096,6 +5096,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "Object or array rows",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Sentencias INSERT en scripts SQL",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codificación",
|
||||
|
||||
@@ -5094,6 +5094,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "Object or array rows",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Istruzioni INSERT negli script SQL",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codifica",
|
||||
|
||||
@@ -5123,6 +5123,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "Object or array rows",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL スクリプトの INSERT ステートメント",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "文字コード",
|
||||
|
||||
@@ -4763,6 +4763,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "객체 또는 배열 행",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL 스크립트의 INSERT 문",
|
||||
previewRows: "미리보기 행",
|
||||
delimiter: "구분자",
|
||||
encoding: "인코딩",
|
||||
|
||||
@@ -5096,6 +5096,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "Object or array rows",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS, XLSX, XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "Instruções INSERT em scripts SQL",
|
||||
previewRows: "Preview rows",
|
||||
delimiter: "Delimiter",
|
||||
encoding: "Codificação",
|
||||
|
||||
@@ -5323,6 +5323,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "对象行或数组行",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS、XLSX、XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL 脚本中的 INSERT 语句",
|
||||
previewRows: "预览行数",
|
||||
delimiter: "分隔符",
|
||||
encoding: "文件编码",
|
||||
|
||||
@@ -4420,6 +4420,8 @@ export default withEnglishFallback({
|
||||
formatJsonDescription: "物件列或陣列列",
|
||||
formatExcel: "Excel",
|
||||
formatExcelDescription: "XLS、XLSX、XLSM",
|
||||
formatSql: "SQL",
|
||||
formatSqlDescription: "SQL 指令檔中的 INSERT 語句",
|
||||
previewRows: "預覽列數",
|
||||
delimiter: "分隔符",
|
||||
encoding: "檔案編碼",
|
||||
|
||||
@@ -115,6 +115,26 @@ describe("tableImport", () => {
|
||||
expect(buildTableImportParseOptions({ ...baseSettings, format: "csv", sheetName: "Second" }).sheetName).toBeNull();
|
||||
});
|
||||
|
||||
it("passes the text encoding for SQL script sources like delimited text", () => {
|
||||
const settings = {
|
||||
delimiter: ",",
|
||||
textEncoding: "gbk" as const,
|
||||
titleRow: 1,
|
||||
dataStartRow: 2,
|
||||
lastDataRow: 0,
|
||||
trimValues: false,
|
||||
emptyStringAsNull: true,
|
||||
jsonShape: "auto" as const,
|
||||
databaseType: "mysql" as const,
|
||||
};
|
||||
|
||||
expect(buildTableImportParseOptions({ ...settings, format: "sql" }).encoding).toBe("gbk");
|
||||
expect(buildTableImportParseOptions({ ...settings, format: "sql" }).sqlDialect).toBe("mysql");
|
||||
expect(buildTableImportParseOptions({ ...settings, format: "delimited" }).encoding).toBe("gbk");
|
||||
expect(buildTableImportParseOptions({ ...settings, format: "delimited" }).sqlDialect).toBeNull();
|
||||
expect(buildTableImportParseOptions({ ...settings, format: "excel" }).encoding).toBeNull();
|
||||
});
|
||||
|
||||
it("suggests create-table data types from preview rows", () => {
|
||||
expect(
|
||||
suggestImportTargetDataTypes(
|
||||
|
||||
@@ -4477,7 +4477,7 @@ export async function sortTablesByFkDependency(options: SortTablesByFkOptions):
|
||||
export type TableImportMode = "append" | "truncate";
|
||||
export type TableImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type TableImportPhase = "preparing" | "detectingEncoding" | "reading" | "writing" | "finalizing" | "done";
|
||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel";
|
||||
export type TableImportSourceFormat = "csv" | "tsv" | "delimited" | "json" | "excel" | "sql";
|
||||
export type TableImportJsonShape = "auto" | "objects" | "arrays";
|
||||
export type TableImportTextEncoding = "auto" | "utf8" | "gbk" | "utf16Le" | "utf16Be";
|
||||
|
||||
@@ -4499,6 +4499,7 @@ export interface TableImportParseOptions {
|
||||
sheetName?: string | null;
|
||||
sheetIndex?: number | null;
|
||||
jsonShape?: TableImportJsonShape | null;
|
||||
sqlDialect?: DatabaseType | null;
|
||||
}
|
||||
|
||||
export interface TableImportPreviewRequest {
|
||||
|
||||
@@ -86,6 +86,7 @@ export interface TableImportParseSettings {
|
||||
emptyStringAsNull: boolean;
|
||||
sheetName?: string;
|
||||
jsonShape: TableImportJsonShape;
|
||||
databaseType?: DatabaseType | null;
|
||||
}
|
||||
|
||||
export function defaultTableImportEmptyStringAsNull(format: TableImportSourceFormat): boolean {
|
||||
@@ -94,9 +95,11 @@ export function defaultTableImportEmptyStringAsNull(format: TableImportSourceFor
|
||||
|
||||
export function buildTableImportParseOptions(settings: TableImportParseSettings): TableImportParseOptions {
|
||||
const isDelimited = settings.format === "csv" || settings.format === "tsv" || settings.format === "delimited";
|
||||
// SQL 脚本与分隔文本一样是纯文本源,需要传递编码供后端解码(支持 GBK 等常见转储编码)
|
||||
const isTextSource = isDelimited || settings.format === "sql";
|
||||
return {
|
||||
delimiter: settings.format === "tsv" ? "\\t" : settings.format === "csv" ? "," : settings.delimiter,
|
||||
encoding: isDelimited ? settings.textEncoding : null,
|
||||
encoding: isTextSource ? settings.textEncoding : null,
|
||||
titleRow: settings.titleRow,
|
||||
dataStartRow: settings.dataStartRow,
|
||||
lastDataRow: settings.lastDataRow,
|
||||
@@ -104,6 +107,8 @@ export function buildTableImportParseOptions(settings: TableImportParseSettings)
|
||||
emptyStringAsNull: settings.emptyStringAsNull,
|
||||
sheetName: settings.format === "excel" ? settings.sheetName || null : null,
|
||||
jsonShape: settings.format === "json" ? settings.jsonShape : null,
|
||||
// SQL 脚本需要按源方言解析字符串转义与标识符大小写(取目标连接的数据库类型)
|
||||
sqlDialect: settings.format === "sql" ? (settings.databaseType ?? null) : null,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user