fix(export): split XLSX sheets and stream worksheet XML

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-07-31 19:45:37 +08:00
1 parent ecac4800c4
commit c43e8455c4
16 files changed
+1006 -250

No files matched your search

-7
View File
@@ -38,12 +38,6 @@ const WINDOWS_JRE_REMOVE_ERROR = [
// Every backend message changed away from hardcoded Chinese, paired with the // Every backend message changed away from hardcoded Chinese, paired with the
// key and params it must resolve to. // key and params it must resolve to.
const CASES: { name: string; message: string; key: string; params?: Record<string, string> }[] = [ const CASES: { name: string; message: string; key: string; params?: Record<string, string> }[] = [
{
name: "XLSX row limit",
message: "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result.",
key: "exportProgress.xlsxRowLimit",
params: { limit: "1,048,575" },
},
{ {
name: "streaming export unsupported", name: "streaming export unsupported",
message: "Streaming export is unsupported for this query. Simplify it or use a supported driver.", message: "Streaming export is unsupported for this query. Simplify it or use a supported driver.",
@@ -172,7 +166,6 @@ describe("backend error wording is pinned to the Rust sources", () => {
const rust = (path: string) => readFileSync(new URL(`../../../../../${path}`, import.meta.url), "utf8"); const rust = (path: string) => readFileSync(new URL(`../../../../../${path}`, import.meta.url), "utf8");
test.each([ test.each([
["crates/dbx-core/src/query_result_export.rs", "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result."],
["crates/dbx-core/src/query_result_export.rs", "Streaming export is unsupported for this query. Simplify it or use a supported driver."], ["crates/dbx-core/src/query_result_export.rs", "Streaming export is unsupported for this query. Simplify it or use a supported driver."],
["crates/dbx-core/src/query_result_export.rs", "Streaming export needs a result-set session, but this driver returned no session_id."], ["crates/dbx-core/src/query_result_export.rs", "Streaming export needs a result-set session, but this driver returned no session_id."],
["crates/dbx-core/src/agent_service.rs", "Failed to remove the old JRE directory: "], ["crates/dbx-core/src/agent_service.rs", "Failed to remove the old JRE directory: "],
-2
View File
@@ -64,7 +64,6 @@ const patterns: [RegExp, string][] = [
[/^Unsupported SOCKS bound address type: (\d+)$/, "settings.tunnelsSocksUnsupportedAddrType"], [/^Unsupported SOCKS bound address type: (\d+)$/, "settings.tunnelsSocksUnsupportedAddrType"],
// Query result export limits (crates/dbx-core/src/query_result_export.rs) // Query result export limits (crates/dbx-core/src/query_result_export.rs)
[/^XLSX supports at most ([\d,]+) data rows\. Use CSV export for the full result\.$/, "exportProgress.xlsxRowLimit"],
[/^Streaming export is unsupported for this query\. Simplify it or use a supported driver\.$/, "exportProgress.streamingUnsupported"], [/^Streaming export is unsupported for this query\. Simplify it or use a supported driver\.$/, "exportProgress.streamingUnsupported"],
[/^Streaming export needs a result-set session, but this driver returned no session_id\.$/, "exportProgress.agentSessionMissing"], [/^Streaming export needs a result-set session, but this driver returned no session_id\.$/, "exportProgress.agentSessionMissing"],
@@ -105,7 +104,6 @@ const paramNames: Record<string, string | string[]> = {
"settings.tunnelsSocksUnsupportedAuth": "method", "settings.tunnelsSocksUnsupportedAuth": "method",
"settings.tunnelsSocksConnectRejected": "code", "settings.tunnelsSocksConnectRejected": "code",
"settings.tunnelsSocksUnsupportedAddrType": "type", "settings.tunnelsSocksUnsupportedAddrType": "type",
"exportProgress.xlsxRowLimit": "limit",
"driverStore.jreDirRemoveFailedWindows": ["path", "error"], "driverStore.jreDirRemoveFailedWindows": ["path", "error"],
"driverStore.jreDirRemoveFailed": ["path", "error"], "driverStore.jreDirRemoveFailed": ["path", "error"],
"driverStore.jreInUseByDrivers": ["jre", "drivers"], "driverStore.jreInUseByDrivers": ["jre", "drivers"],
-1
View File
@@ -1447,7 +1447,6 @@ export default {
truncatedHint: "Results were truncated after loading {count} rows. Use the footer pagination to browse loaded data; exporting the full result reruns the database query.", truncatedHint: "Results were truncated after loading {count} rows. Use the footer pagination to browse loaded data; exporting the full result reruns the database query.",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX supports at most {limit} data rows. Use CSV export for the full result.",
streamingUnsupported: "Streaming export is unsupported for this query. Simplify it or use a supported driver.", streamingUnsupported: "Streaming export is unsupported for this query. Simplify it or use a supported driver.",
agentSessionMissing: "Streaming export needs a result-set session, but this driver returned no session_id.", agentSessionMissing: "Streaming export needs a result-set session, but this driver returned no session_id.",
title: "Exporting Table Data", title: "Exporting Table Data",
-1
View File
@@ -1389,7 +1389,6 @@ export default withEnglishFallback({
xlsxHeaderComment: "Encabezado usando comentarios", xlsxHeaderComment: "Encabezado usando comentarios",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX admite como máximo {limit} filas de datos. Use la exportación CSV para obtener el resultado completo.",
streamingUnsupported: "La exportación en streaming no es compatible con esta consulta. Simplifíquela o use un controlador compatible.", streamingUnsupported: "La exportación en streaming no es compatible con esta consulta. Simplifíquela o use un controlador compatible.",
agentSessionMissing: "La exportación en streaming requiere una sesión de conjunto de resultados, pero este controlador no devolvió session_id.", agentSessionMissing: "La exportación en streaming requiere una sesión de conjunto de resultados, pero este controlador no devolvió session_id.",
title: "Exportando datos de la tabla", title: "Exportando datos de la tabla",
-1
View File
@@ -1387,7 +1387,6 @@ export default withEnglishFallback({
xlsxHeaderComment: "Intestazione con commento", xlsxHeaderComment: "Intestazione con commento",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX supporta al massimo {limit} righe di dati. Usa l'esportazione CSV per il risultato completo.",
streamingUnsupported: "L'esportazione in streaming non è supportata per questa query. Semplificala o usa un driver supportato.", streamingUnsupported: "L'esportazione in streaming non è supportata per questa query. Semplificala o usa un driver supportato.",
agentSessionMissing: "L'esportazione in streaming richiede una sessione del set di risultati, ma questo driver non ha restituito session_id.", agentSessionMissing: "L'esportazione in streaming richiede una sessione del set di risultati, ma questo driver non ha restituito session_id.",
title: "Esportazione Dati Tabella", title: "Esportazione Dati Tabella",
-1
View File
@@ -1414,7 +1414,6 @@ export default withEnglishFallback({
emptyStringValue: "空文字", emptyStringValue: "空文字",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX は最大 {limit} 行のデータに対応しています。完全な結果を得るには CSV エクスポートを使用してください。",
streamingUnsupported: "このクエリはストリーミングエクスポートに対応していません。クエリを簡略化するか、対応しているドライバーを使用してください。", streamingUnsupported: "このクエリはストリーミングエクスポートに対応していません。クエリを簡略化するか、対応しているドライバーを使用してください。",
agentSessionMissing: "ストリーミングエクスポートには結果セットのセッションが必要ですが、このドライバーは session_id を返しませんでした。", agentSessionMissing: "ストリーミングエクスポートには結果セットのセッションが必要ですが、このドライバーは session_id を返しませんでした。",
title: "テーブルデータをエクスポート中", title: "テーブルデータをエクスポート中",
-1
View File
@@ -1425,7 +1425,6 @@ export default withEnglishFallback({
truncatedHint: "{count}행 로드 후 결과가 잘렸습니다. 로드된 데이터를 탐색하려면 푸터 페이지 매김을 사용하세요. 전체 결과를 내보내면 데이터베이스 쿼리를 다시 실행합니다.", truncatedHint: "{count}행 로드 후 결과가 잘렸습니다. 로드된 데이터를 탐색하려면 푸터 페이지 매김을 사용하세요. 전체 결과를 내보내면 데이터베이스 쿼리를 다시 실행합니다.",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX는 최대 {limit}개의 데이터 행을 지원합니다. 전체 결과는 CSV 내보내기를 사용하세요.",
streamingUnsupported: "이 쿼리는 스트리밍 내보내기를 지원하지 않습니다. 쿼리를 단순화하거나 지원되는 드라이버를 사용하세요.", streamingUnsupported: "이 쿼리는 스트리밍 내보내기를 지원하지 않습니다. 쿼리를 단순화하거나 지원되는 드라이버를 사용하세요.",
agentSessionMissing: "스트리밍 내보내기에는 결과 집합 세션이 필요하지, 이 드라이버는 session_id를 반환하지 않았습니다.", agentSessionMissing: "스트리밍 내보내기에는 결과 집합 세션이 필요하지, 이 드라이버는 session_id를 반환하지 않았습니다.",
title: "테이블 데이터 내보내는 중", title: "테이블 데이터 내보내는 중",
-1
View File
@@ -1389,7 +1389,6 @@ export default withEnglishFallback({
xlsxHeaderComment: "Cabeçalho usa comentário", xlsxHeaderComment: "Cabeçalho usa comentário",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "O XLSX suporta no máximo {limit} linhas de dados. Use a exportação CSV para obter o resultado completo.",
streamingUnsupported: "A exportação em streaming não é compatível com esta consulta. Simplifique-a ou use um driver compatível.", streamingUnsupported: "A exportação em streaming não é compatível com esta consulta. Simplifique-a ou use um driver compatível.",
agentSessionMissing: "A exportação em streaming requer uma sessão de conjunto de resultados, mas este driver não retornou session_id.", agentSessionMissing: "A exportação em streaming requer uma sessão de conjunto de resultados, mas este driver não retornou session_id.",
title: "Exportando Dados da Tabela", title: "Exportando Dados da Tabela",
-1
View File
@@ -1448,7 +1448,6 @@ export default withEnglishFallback({
truncatedHint: "结果已截断,已加载前 {count} 行。可通过底部分页浏览已加载数据;导出完整结果时会重新查询数据库。", truncatedHint: "结果已截断,已加载前 {count} 行。可通过底部分页浏览已加载数据;导出完整结果时会重新查询数据库。",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX 最多支持 {limit} 行数据,请改用 CSV 导出完整结果。",
streamingUnsupported: "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。", streamingUnsupported: "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。",
agentSessionMissing: "流式导出需要结果集会话,但当前驱动未返回 session_id。", agentSessionMissing: "流式导出需要结果集会话,但当前驱动未返回 session_id。",
title: "导出表数据", title: "导出表数据",
-1
View File
@@ -1388,7 +1388,6 @@ export default withEnglishFallback({
xlsxHeaderComment: "表頭使用註解", xlsxHeaderComment: "表頭使用註解",
}, },
exportProgress: { exportProgress: {
xlsxRowLimit: "XLSX 最多支援 {limit} 列資料,請改用 CSV 匯出完整結果。",
streamingUnsupported: "目前查詢暫不支援串流匯出,請簡化查詢或使用支援的驅動程式。", streamingUnsupported: "目前查詢暫不支援串流匯出,請簡化查詢或使用支援的驅動程式。",
agentSessionMissing: "串流匯出需要結果集工作階段,但目前驅動程式未回傳 session_id。", agentSessionMissing: "串流匯出需要結果集工作階段,但目前驅動程式未回傳 session_id。",
title: "匯出資料表資料", title: "匯出資料表資料",
+61 -19
View File
@@ -11,6 +11,13 @@ export interface XlsxWorksheetData {
numericColumnRightAlign?: boolean; numericColumnRightAlign?: boolean;
} }
interface XlsxWorksheetSegment {
worksheet: XlsxWorksheetData;
sheetName?: string;
rowStart: number;
rowEnd: number;
}
type ZipEntry = { type ZipEntry = {
path: string; path: string;
data: Uint8Array; data: Uint8Array;
@@ -20,6 +27,7 @@ type ZipEntry = {
const encoder = new TextEncoder(); const encoder = new TextEncoder();
const CRC_TABLE = buildCrcTable(); const CRC_TABLE = buildCrcTable();
const XLSX_MAX_DATA_ROWS = 1_048_575;
function buildCrcTable(): number[] { function buildCrcTable(): number[] {
const table: number[] = []; const table: number[] = [];
@@ -84,15 +92,20 @@ function normalizeSheetName(value?: string): string {
return name.slice(0, 31); return name.slice(0, 31);
} }
function normalizeUniqueSheetNames(sheets: readonly XlsxWorksheetData[]): string[] { function appendSheetSuffix(base: string, suffix: number): string {
const suffixText = ` (${suffix})`;
const maxBaseLength = Math.max(0, 31 - [...suffixText].length);
return `${[...base].slice(0, maxBaseLength).join("")}${suffixText}`;
}
function normalizeUniqueSheetNames(sheets: readonly { sheetName?: string }[]): string[] {
const names: string[] = []; const names: string[] = [];
sheets.forEach((sheet, index) => { sheets.forEach((sheet, index) => {
const base = normalizeSheetName(sheet.sheetName || `Sheet${index + 1}`); const base = normalizeSheetName(sheet.sheetName || `Sheet${index + 1}`);
let candidate = base; let candidate = base;
let suffix = 2; let suffix = 2;
while (names.includes(candidate)) { while (names.includes(candidate)) {
const suffixText = ` (${suffix})`; candidate = appendSheetSuffix(base, suffix);
candidate = `${base.slice(0, 31 - suffixText.length)}${suffixText}`;
suffix += 1; suffix += 1;
} }
names.push(candidate); names.push(candidate);
@@ -100,10 +113,10 @@ function normalizeUniqueSheetNames(sheets: readonly XlsxWorksheetData[]): string
return names; return names;
} }
function estimateColumnWidths(columns: readonly string[], rows: readonly (readonly XlsxCellValue[])[], columnComments?: readonly (string | null)[]): number[] { function estimateColumnWidths(columns: readonly string[], rows: readonly (readonly XlsxCellValue[])[], rowStart: number, rowEnd: number, columnComments?: readonly (string | null)[]): number[] {
return columns.map((column, colIndex) => { return columns.map((column, colIndex) => {
const headerText = columnComments?.[colIndex] || column; const headerText = columnComments?.[colIndex] || column;
const values = rows.slice(0, 100).map((row) => row[colIndex]); const values = Array.from({ length: Math.min(100, rowEnd - rowStart) }, (_, index) => rows[rowStart + index]?.[colIndex]);
const maxLen = [headerText, ...values.map((value) => (value == null ? "" : String(value)))].map((value) => Math.min(value.length, 60)).reduce((max, length) => Math.max(max, length), 8); const maxLen = [headerText, ...values.map((value) => (value == null ? "" : String(value)))].map((value) => Math.min(value.length, 60)).reduce((max, length) => Math.max(max, length), 8);
return Math.max(10, Math.min(60, maxLen + 2)); return Math.max(10, Math.min(60, maxLen + 2));
}); });
@@ -143,22 +156,22 @@ function cellXml(value: XlsxCellValue, rowIndex: number, colIndex: number, style
return `<c r="${ref}" t="inlineStr"${styleAttr}><is><t>${escapeXml(String(value))}</t></is></c>`; return `<c r="${ref}" t="inlineStr"${styleAttr}><is><t>${escapeXml(String(value))}</t></is></c>`;
} }
function worksheetXml(data: XlsxWorksheetData): string { function worksheetXml(segment: XlsxWorksheetSegment): string {
const data = segment.worksheet;
const columns = data.columns; const columns = data.columns;
const rows = data.rows; const rows = data.rows;
const totalRows = rows.length + 1; const totalRows = segment.rowEnd - segment.rowStart + 1;
const range = sheetRange(columns.length, totalRows); const range = sheetRange(columns.length, totalRows);
const widths = estimateColumnWidths(columns, rows, data.columnComments); const widths = estimateColumnWidths(columns, rows, segment.rowStart, segment.rowEnd, data.columnComments);
const rightAlignEnabled = data.numericColumnRightAlign !== false; const rightAlignEnabled = data.numericColumnRightAlign !== false;
const colsXml = widths.map((width, index) => `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"/>`).join(""); const colsXml = widths.map((width, index) => `<col min="${index + 1}" max="${index + 1}" width="${width}" customWidth="1"/>`).join("");
const headerXml = `<row r="1">${columns.map((column, index) => cellXml(data.columnComments?.[index] || column, 0, index, 1)).join("")}</row>`; const headerXml = `<row r="1">${columns.map((column, index) => cellXml(data.columnComments?.[index] || column, 0, index, 1)).join("")}</row>`;
const bodyXml = rows const bodyXml = Array.from({ length: segment.rowEnd - segment.rowStart }, (_, rowIndex) => {
.map((row, rowIndex) => { const row = rows[segment.rowStart + rowIndex]!;
const excelRowIndex = rowIndex + 2; const excelRowIndex = rowIndex + 2;
const cells = columns.map((_, colIndex) => cellXml(row[colIndex], excelRowIndex - 1, colIndex, numericColumnStyle(data.columnTypes?.[colIndex], rightAlignEnabled), data.columnTypes?.[colIndex])).join(""); const cells = columns.map((_, colIndex) => cellXml(row[colIndex], excelRowIndex - 1, colIndex, numericColumnStyle(data.columnTypes?.[colIndex], rightAlignEnabled), data.columnTypes?.[colIndex])).join("");
return `<row r="${excelRowIndex}">${cells}</row>`; return `<row r="${excelRowIndex}">${cells}</row>`;
}) }).join("");
.join("");
return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?> return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"> <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
@@ -171,6 +184,30 @@ function worksheetXml(data: XlsxWorksheetData): string {
</worksheet>`; </worksheet>`;
} }
function splitWorksheetsForMaxRows(sheets: readonly XlsxWorksheetData[], maxDataRowsPerSheet: number): XlsxWorksheetSegment[] {
const maxRows = Number.isFinite(maxDataRowsPerSheet) ? Math.max(1, Math.floor(maxDataRowsPerSheet)) : XLSX_MAX_DATA_ROWS;
const segments: XlsxWorksheetSegment[] = [];
for (const worksheet of sheets) {
if (worksheet.rows.length <= maxRows) {
segments.push({ worksheet, sheetName: worksheet.sheetName, rowStart: 0, rowEnd: worksheet.rows.length });
continue;
}
const baseName = normalizeSheetName(worksheet.sheetName);
for (let rowStart = 0, chunkIndex = 0; rowStart < worksheet.rows.length; rowStart += maxRows, chunkIndex += 1) {
segments.push({
worksheet,
sheetName: chunkIndex === 0 ? baseName : appendSheetSuffix(baseName, chunkIndex + 1),
rowStart,
rowEnd: Math.min(rowStart + maxRows, worksheet.rows.length),
});
}
}
return segments;
}
function contentTypesXml(sheetCount = 1): string { function contentTypesXml(sheetCount = 1): string {
const worksheetOverrides = Array.from({ length: sheetCount }, (_, index) => ` <Override PartName="/xl/worksheets/sheet${index + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`).join("\n"); const worksheetOverrides = Array.from({ length: sheetCount }, (_, index) => ` <Override PartName="/xl/worksheets/sheet${index + 1}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>`).join("\n");
return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?> return `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
@@ -278,14 +315,19 @@ export function buildXlsxWorkbook(data: XlsxWorksheetData): Uint8Array {
} }
export function buildXlsxWorkbookMulti(sheets: readonly XlsxWorksheetData[]): Uint8Array { export function buildXlsxWorkbookMulti(sheets: readonly XlsxWorksheetData[]): Uint8Array {
return buildXlsxWorkbookMultiWithMaxRows(sheets, XLSX_MAX_DATA_ROWS);
}
export function buildXlsxWorkbookMultiWithMaxRows(sheets: readonly XlsxWorksheetData[], maxDataRowsPerSheet: number): Uint8Array {
if (sheets.length === 0) throw new Error("At least one worksheet is required"); if (sheets.length === 0) throw new Error("At least one worksheet is required");
const sheetNames = normalizeUniqueSheetNames(sheets); const segments = splitWorksheetsForMaxRows(sheets, maxDataRowsPerSheet);
const sheetNames = normalizeUniqueSheetNames(segments);
return createZip([ return createZip([
{ path: "[Content_Types].xml", content: contentTypesXml(sheets.length) }, { path: "[Content_Types].xml", content: contentTypesXml(segments.length) },
{ path: "_rels/.rels", content: rootRelsXml() }, { path: "_rels/.rels", content: rootRelsXml() },
{ path: "xl/workbook.xml", content: workbookXml(sheetNames) }, { path: "xl/workbook.xml", content: workbookXml(sheetNames) },
{ path: "xl/_rels/workbook.xml.rels", content: workbookRelsXml(sheets.length) }, { path: "xl/_rels/workbook.xml.rels", content: workbookRelsXml(segments.length) },
{ path: "xl/styles.xml", content: stylesXml() }, { path: "xl/styles.xml", content: stylesXml() },
...sheets.map((sheet, index) => ({ path: `xl/worksheets/sheet${index + 1}.xml`, content: worksheetXml(sheet) })), ...segments.map((segment, index) => ({ path: `xl/worksheets/sheet${index + 1}.xml`, content: worksheetXml(segment) })),
]); ]);
} }
+2 -1
View File
@@ -984,6 +984,7 @@ pub async fn run_codex_agent(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::codex_process_env_with_system_proxy;
#[cfg(not(windows))] #[cfg(not(windows))]
use super::shell_quote; use super::shell_quote;
use super::{ use super::{
@@ -996,7 +997,7 @@ mod tests {
DEFAULT_CODEX_MODELS, DEFAULT_CODEX_MODELS,
}; };
#[cfg(not(windows))] #[cfg(not(windows))]
use super::{codex_process_env, codex_process_env_with_system_proxy, common_executable_dirs, merged_path_with_dir}; use super::{codex_process_env, common_executable_dirs, merged_path_with_dir};
#[cfg(windows)] #[cfg(windows)]
use super::{ use super::{
direct_program_path, first_windows_program_path, program_path_candidates, resolve_codex_command, direct_program_path, first_windows_program_path, program_path_candidates, resolve_codex_command,
+28 -92
View File
@@ -35,8 +35,6 @@ use sqlparser::parser::Parser;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
const AGENT_UNBOUNDED_ROW_LIMIT: usize = i32::MAX as usize; const AGENT_UNBOUNDED_ROW_LIMIT: usize = i32::MAX as usize;
pub const XLSX_MAX_DATA_ROWS: usize = 1_048_575;
const XLSX_ROW_LIMIT_ERROR: &str = "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result.";
const STREAMING_PAGINATION_UNSUPPORTED_ERROR: &str = const STREAMING_PAGINATION_UNSUPPORTED_ERROR: &str =
"Streaming export is unsupported for this query. Simplify it or use a supported driver."; "Streaming export is unsupported for this query. Simplify it or use a supported driver.";
const AGENT_SESSION_MISSING_ERROR: &str = const AGENT_SESSION_MISSING_ERROR: &str =
@@ -241,11 +239,7 @@ fn progress(
status: ExportStatus, status: ExportStatus,
error_message: Option<String>, error_message: Option<String>,
) -> TableExportProgress { ) -> TableExportProgress {
let total_rows = request.total_rows.map(|total| { let total_rows = request.total_rows.map(|total| request.row_limit.map_or(total, |limit| total.min(limit as u64)));
let format = request.format.to_lowercase();
let limit = effective_row_limit(&format, request);
limit.map_or(total, |limit| total.min(limit as u64))
});
TableExportProgress { TableExportProgress {
export_id: request.export_id.clone(), export_id: request.export_id.clone(),
table_name: String::new(), table_name: String::new(),
@@ -387,16 +381,8 @@ impl SqlInsertWriter {
} }
} }
fn effective_row_limit(format: &str, request: &QueryResultExportRequest) -> Option<usize> { fn effective_row_limit(request: &QueryResultExportRequest) -> Option<usize> {
if format == "xlsx" { request.row_limit
Some(request.row_limit.map_or(XLSX_MAX_DATA_ROWS, |limit| limit.min(XLSX_MAX_DATA_ROWS)))
} else {
request.row_limit
}
}
fn xlsx_hard_limit_active(format: &str, request: &QueryResultExportRequest) -> bool {
format == "xlsx" && request.row_limit.is_none_or(|limit| limit > XLSX_MAX_DATA_ROWS)
} }
fn format_text_export_header(format: &str, columns: &[String]) -> String { fn format_text_export_header(format: &str, columns: &[String]) -> String {
@@ -619,18 +605,8 @@ async fn export_query_result_core_inner(
} }
let page_size = request.page_size.max(1); let page_size = request.page_size.max(1);
let effective_row_limit = effective_row_limit(&format, request); let effective_row_limit = effective_row_limit(request);
let xlsx_hard_limit_active = xlsx_hard_limit_active(&format, request); let agent_max_rows = effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT).max(1);
if xlsx_hard_limit_active && request.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64) {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
let agent_max_rows = if xlsx_hard_limit_active {
XLSX_MAX_DATA_ROWS + 1
} else {
effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT)
}
.max(1);
on_progress(progress(request, 0, ExportStatus::Running, None)); on_progress(progress(request, 0, ExportStatus::Running, None));
@@ -695,11 +671,6 @@ async fn export_query_result_core_inner(
break; break;
} }
let this_page = remaining.map_or(page_size, |rem| rem.min(page_size)).max(1); let this_page = remaining.map_or(page_size, |rem| rem.min(page_size)).max(1);
let fetch_limit = if xlsx_hard_limit_active && remaining.is_some_and(|rem| rem <= page_size) {
this_page.saturating_add(1)
} else {
this_page
};
let (sql_to_execute, plan_limit, use_agent_result_session) = if let Some(plan) = keyset_plan.as_ref() { let (sql_to_execute, plan_limit, use_agent_result_session) = if let Some(plan) = keyset_plan.as_ref() {
( (
@@ -710,9 +681,9 @@ async fn export_query_result_core_inner(
&request.database_type, &request.database_type,
&plan.primary_keys, &plan.primary_keys,
&plan.last_pk_values, &plan.last_pk_values,
fetch_limit, this_page,
), ),
fetch_limit, this_page,
false, false,
) )
} else { } else {
@@ -720,7 +691,7 @@ async fn export_query_result_core_inner(
sql: request.sql.clone(), sql: request.sql.clone(),
query_base_sql: request.query_base_sql.clone(), query_base_sql: request.query_base_sql.clone(),
database_type: Some(request.database_type), database_type: Some(request.database_type),
pagination: QueryPagination { limit: fetch_limit, offset, session_id: session_id.clone() }, pagination: QueryPagination { limit: this_page, offset, session_id: session_id.clone() },
use_agent_cursor: request.use_agent_cursor, use_agent_cursor: request.use_agent_cursor,
first_page_uses_actual_sql: true, first_page_uses_actual_sql: true,
}); });
@@ -801,12 +772,6 @@ async fn export_query_result_core_inner(
} }
} }
let fetched_row_count = result.rows.len(); let fetched_row_count = result.rows.len();
if xlsx_hard_limit_active {
let remaining_rows = XLSX_MAX_DATA_ROWS.saturating_sub(rows_exported as usize);
if fetched_row_count > remaining_rows {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
}
if result.rows.len() > this_page { if result.rows.len() > this_page {
result.rows.truncate(this_page); result.rows.truncate(this_page);
} }
@@ -964,10 +929,8 @@ async fn try_export_postgres_query_result_stream(
state.touch_pool_activity(&pool_key).await; state.touch_pool_activity(&pool_key).await;
let _activity_touch = state.pool_activity_touch(&pool_key); let _activity_touch = state.pool_activity_touch(&pool_key);
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); let row_limit = effective_row_limit(request);
let row_limit = effective_row_limit(format, request); let stream_row_limit = row_limit;
let stream_row_limit =
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
let progress_row_interval = request.page_size.max(1) as u64; let progress_row_interval = request.page_size.max(1) as u64;
let mut columns: Vec<String> = Vec::new(); let mut columns: Vec<String> = Vec::new();
let mut temporal_column_types: Vec<String> = Vec::new(); let mut temporal_column_types: Vec<String> = Vec::new();
@@ -1020,9 +983,6 @@ async fn try_export_postgres_query_result_stream(
} }
} }
crate::db::postgres::PostgresQueryStreamItem::Row(row) => { crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
&row, &row,
&temporal_column_types, &temporal_column_types,
@@ -1161,10 +1121,8 @@ async fn try_export_mysql_query_result_stream(
crate::query_execution_sql::check_read_only(&request.sql, &name, database_type)?; crate::query_execution_sql::check_read_only(&request.sql, &name, database_type)?;
} }
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); let row_limit = effective_row_limit(request);
let row_limit = effective_row_limit(format, request); let stream_row_limit = row_limit;
let stream_row_limit =
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
let progress_row_interval = request.page_size.max(1) as u64; let progress_row_interval = request.page_size.max(1) as u64;
let mut columns: Vec<String> = Vec::new(); let mut columns: Vec<String> = Vec::new();
let mut temporal_column_types: Vec<String> = Vec::new(); let mut temporal_column_types: Vec<String> = Vec::new();
@@ -1266,9 +1224,6 @@ async fn try_export_mysql_query_result_stream(
} }
} }
crate::db::mysql::MySqlQueryStreamItem::Row(row) => { crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
&row, &row,
&temporal_column_types, &temporal_column_types,
@@ -1427,10 +1382,8 @@ async fn try_export_clickhouse_query_result_stream(
state.touch_pool_activity(&pool_key).await; state.touch_pool_activity(&pool_key).await;
let _activity_touch = state.pool_activity_touch(&pool_key); let _activity_touch = state.pool_activity_touch(&pool_key);
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); let row_limit = effective_row_limit(request);
let row_limit = effective_row_limit(format, request); let stream_row_limit = row_limit;
let stream_row_limit =
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
let progress_row_interval = request.page_size.max(1) as u64; let progress_row_interval = request.page_size.max(1) as u64;
let mut columns: Vec<String> = Vec::new(); let mut columns: Vec<String> = Vec::new();
let mut temporal_column_types: Vec<String> = Vec::new(); let mut temporal_column_types: Vec<String> = Vec::new();
@@ -1484,9 +1437,6 @@ async fn try_export_clickhouse_query_result_stream(
} }
} }
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => { crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
&row, &row,
&temporal_column_types, &temporal_column_types,
@@ -1600,10 +1550,8 @@ async fn try_export_sqlserver_query_result_stream(
state.running_queries.set_pool_key(execution_id, pool_key); state.running_queries.set_pool_key(execution_id, pool_key);
} }
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request); let row_limit = effective_row_limit(request);
let row_limit = effective_row_limit(format, request); let stream_row_limit = row_limit;
let stream_row_limit =
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
let mut columns: Vec<String> = Vec::new(); let mut columns: Vec<String> = Vec::new();
let mut temporal_column_types: Vec<String> = Vec::new(); let mut temporal_column_types: Vec<String> = Vec::new();
let mut rows_exported = 0_u64; let mut rows_exported = 0_u64;
@@ -1667,9 +1615,6 @@ async fn try_export_sqlserver_query_result_stream(
} }
} }
crate::db::sqlserver::SqlServerStreamItem::Row(row) => { crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
row, row,
&temporal_column_types, &temporal_column_types,
@@ -1865,12 +1810,12 @@ mod tests {
#[test] #[test]
fn csv_unlimited_export_has_no_effective_row_limit() { fn csv_unlimited_export_has_no_effective_row_limit() {
assert_eq!(effective_row_limit("csv", &request("csv", None, None)), None); assert_eq!(effective_row_limit(&request("csv", None, None)), None);
} }
#[test] #[test]
fn txt_unlimited_export_has_no_effective_row_limit() { fn txt_unlimited_export_has_no_effective_row_limit() {
assert_eq!(effective_row_limit("txt", &request("txt", None, None)), None); assert_eq!(effective_row_limit(&request("txt", None, None)), None);
} }
#[test] #[test]
@@ -1879,31 +1824,22 @@ mod tests {
} }
#[test] #[test]
fn xlsx_unlimited_export_uses_excel_hard_limit() { fn xlsx_no_row_limit_has_no_query_layer_cap() {
assert_eq!(effective_row_limit("xlsx", &request("xlsx", None, None)), Some(XLSX_MAX_DATA_ROWS)); // Without the old hard limit, XLSX uses the writer's internal splitting.
assert_eq!(effective_row_limit(&request("xlsx", None, None)), None);
} }
#[test] #[test]
fn xlsx_row_limit_caps_to_excel_hard_limit() { fn xlsx_user_row_limit_still_respected() {
assert_eq!( assert_eq!(effective_row_limit(&request("xlsx", Some(500), None)), Some(500));
effective_row_limit("xlsx", &request("xlsx", Some(XLSX_MAX_DATA_ROWS + 10), None)),
Some(XLSX_MAX_DATA_ROWS)
);
} }
#[test] #[test]
fn xlsx_known_total_above_hard_limit_errors_before_export() { fn xlsx_total_rows_above_sheet_limit_no_longer_errors() {
let req = request("xlsx", None, Some(XLSX_MAX_DATA_ROWS as u64 + 1)); // total_rows > 1M no longer triggers a pre-check error; the writer splits.
assert!(xlsx_hard_limit_active("xlsx", &req)); let req = request("xlsx", None, Some(2_000_000));
assert!(req.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64)); assert!(effective_row_limit(&req).is_none());
} // The function that used to check this (xlsx_hard_limit_active) no longer exists.
#[test]
fn sql_insert_export_has_no_xlsx_row_cap() {
// SQL format should not be limited by XLSX_MAX_DATA_ROWS.
let req = request("sql", None, None);
assert!(!xlsx_hard_limit_active("sql", &req));
assert_eq!(effective_row_limit("sql", &req), None);
} }
#[test] #[test]
File diff suppressed because it is too large. Load diff
+5 -5
View File
@@ -28,7 +28,7 @@ const dialogMock = vi.hoisted(() => ({ save: vi.fn() }));
const toastMock = vi.hoisted(() => vi.fn()); const toastMock = vi.hoisted(() => vi.fn());
const translateMock = vi.hoisted(() => const translateMock = vi.hoisted(() =>
vi.fn((key: string, params?: Record<string, unknown>) => { vi.fn((key: string, params?: Record<string, unknown>) => {
if (key === "exportProgress.xlsxRowLimit") return `XLSX 最多支持 ${params?.limit} 行数据,请使用 CSV 导出完整结果。`; if (key === "exportProgress.streamingUnsupported") return "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。";
if (key === "grid.exportFailed") return `导出失败:${params?.message}`; if (key === "grid.exportFailed") return `导出失败:${params?.message}`;
return key; return key;
}), }),
@@ -397,10 +397,10 @@ test("full query result CSV export streams through the backend without loading a
assert.equal(exportProgressState.value.filePath, apiMock.startQueryResultExport.mock.calls[0][0].filePath); assert.equal(exportProgressState.value.filePath, apiMock.startQueryResultExport.mock.calls[0][0].filePath);
}); });
test("streaming query result export translates terminal backend errors before the toast", async () => { test("streaming query result export translates streaming unsupported error before the toast", async () => {
const rawMessage = "XLSX supports at most 1,048,575 data rows. Use CSV export for the full result."; const rawMessage = "Streaming export is unsupported for this query. Simplify it or use a supported driver.";
apiMock.startQueryResultExport.mockImplementationOnce(async (request, onProgress) => { apiMock.startQueryResultExport.mockImplementationOnce(async (request, onProgress) => {
onProgress({ exportId: request.exportId, tableName: "", rowsExported: 0, totalRows: 1_048_576, status: "Error", errorMessage: rawMessage }); onProgress({ exportId: request.exportId, tableName: "", rowsExported: 0, totalRows: 0, status: "Error", errorMessage: rawMessage });
throw new Error(rawMessage); throw new Error(rawMessage);
}); });
const { composable, exportProgressState } = buildExportHarness(); const { composable, exportProgressState } = buildExportHarness();
@@ -408,7 +408,7 @@ test("streaming query result export translates terminal backend errors before th
await composable.exportXlsx(); await composable.exportXlsx();
assert.equal(exportProgressState.value.errorMessage, rawMessage); assert.equal(exportProgressState.value.errorMessage, rawMessage);
assert.deepEqual(toastMock.mock.calls.at(-1), ["导出失败:XLSX 最多支持 1,048,575 行数据,请使用 CSV 导出完整结果。", 5000]); assert.deepEqual(toastMock.mock.calls.at(-1), ["导出失败:当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。", 5000]);
}); });
test("complete local query result XLSX export does not re-execute the query", async () => { test("complete local query result XLSX export does not re-execute the query", async () => {
+72 -19
View File
@@ -1,8 +1,24 @@
import { strict as assert } from "node:assert"; import { strict as assert } from "node:assert";
import { test } from "vitest"; import { test } from "vitest";
import { buildXlsxWorkbook, buildXlsxWorkbookMulti } from "../../apps/desktop/src/lib/export/xlsxExport.ts"; import { buildXlsxWorkbook, buildXlsxWorkbookMulti, buildXlsxWorkbookMultiWithMaxRows } from "../../apps/desktop/src/lib/export/xlsxExport.ts";
import { buildXlsxSqlWorksheet } from "../../apps/desktop/src/lib/export/xlsxSqlSheet.ts"; import { buildXlsxSqlWorksheet } from "../../apps/desktop/src/lib/export/xlsxSqlSheet.ts";
function readStoredZipEntry(workbook: Uint8Array, entryPath: string): string {
const view = new DataView(workbook.buffer, workbook.byteOffset, workbook.byteLength);
let offset = 0;
while (offset + 30 <= workbook.length && view.getUint32(offset, true) === 0x04034b50) {
const compressedSize = view.getUint32(offset + 18, true);
const fileNameLength = view.getUint16(offset + 26, true);
const extraLength = view.getUint16(offset + 28, true);
const fileNameStart = offset + 30;
const dataStart = fileNameStart + fileNameLength + extraLength;
const fileName = new TextDecoder().decode(workbook.subarray(fileNameStart, fileNameStart + fileNameLength));
if (fileName === entryPath) return new TextDecoder().decode(workbook.subarray(dataStart, dataStart + compressedSize));
offset = dataStart + compressedSize;
}
throw new Error(`Missing ZIP entry: ${entryPath}`);
}
test("builds an xlsx workbook zip with worksheet data", () => { test("builds an xlsx workbook zip with worksheet data", () => {
const workbook = buildXlsxWorkbook({ const workbook = buildXlsxWorkbook({
sheetName: "Users", sheetName: "Users",
@@ -64,6 +80,60 @@ test("builds a result workbook with a separate SQL worksheet", () => {
assert.match(text, /SELECT id, name FROM users WHERE active = true/); assert.match(text, /SELECT id, name FROM users WHERE active = true/);
}); });
test("web in-memory XLSX export splits oversized worksheets", () => {
const workbook = buildXlsxWorkbookMultiWithMaxRows(
[
{
sheetName: "Result",
columns: ["id", "name"],
rows: [
[1, "row_1"],
[2, "row_2"],
[3, "row_3"],
[4, "row_4"],
[5, "row_5"],
],
},
],
2,
);
const workbookXml = readStoredZipEntry(workbook, "xl/workbook.xml");
assert.match(workbookXml, /name="Result"/);
assert.match(workbookXml, /name="Result \(2\)"/);
assert.match(workbookXml, /name="Result \(3\)"/);
const sheet1 = readStoredZipEntry(workbook, "xl/worksheets/sheet1.xml");
const sheet2 = readStoredZipEntry(workbook, "xl/worksheets/sheet2.xml");
const sheet3 = readStoredZipEntry(workbook, "xl/worksheets/sheet3.xml");
assert.match(sheet1, /row_1/);
assert.match(sheet1, /row_2/);
assert.doesNotMatch(sheet1, /row_3/);
assert.match(sheet2, /row_3/);
assert.match(sheet2, /row_4/);
assert.doesNotMatch(sheet2, /row_5/);
assert.match(sheet3, /row_5/);
assert.equal(sheet1.match(/<row r="/g)?.length, 3);
assert.equal(sheet2.match(/<row r="/g)?.length, 3);
assert.equal(sheet3.match(/<row r="/g)?.length, 2);
});
test("web in-memory XLSX export handles a realistic large result", () => {
const rowCount = 20_001;
const rows = Array.from({ length: rowCount }, (_, index) => [index + 1, `user_${index + 1}`, `user_${index + 1}@example.com`, index % 2 === 0, `note-${String(index + 1).padStart(6, "0")}-${"x".repeat(48)}`]);
const workbook = buildXlsxWorkbookMultiWithMaxRows([{ sheetName: "Users", columns: ["id", "name", "email", "active", "notes"], rows }], 10_000);
const workbookXml = readStoredZipEntry(workbook, "xl/workbook.xml");
assert.equal(workbookXml.match(/<sheet /g)?.length, 3);
assert.match(workbookXml, /name="Users \(3\)"/);
assert.match(readStoredZipEntry(workbook, "xl/worksheets/sheet1.xml"), /user_10000@example\.com/);
assert.match(readStoredZipEntry(workbook, "xl/worksheets/sheet2.xml"), /user_20000@example\.com/);
const finalSheet = readStoredZipEntry(workbook, "xl/worksheets/sheet3.xml");
assert.match(finalSheet, /user_20001@example\.com/);
assert.equal(finalSheet.match(/<row r="/g)?.length, 2);
assert.ok(workbook.byteLength > 5_000_000);
});
test("maps multiple result statements and splits SQL at the Excel cell limit", () => { test("maps multiple result statements and splits SQL at the Excel cell limit", () => {
const bmpPrefix = "x".repeat(32_766); const bmpPrefix = "x".repeat(32_766);
const longSql = `${bmpPrefix}😀tail`; const longSql = `${bmpPrefix}😀tail`;
@@ -130,24 +200,7 @@ test("numeric right-align style is applied consistently across cross-database nu
// Ensures the front-end XLSX classifier covers the same cross-database // Ensures the front-end XLSX classifier covers the same cross-database
// numeric types as the Rust classifier and the grid (ClickHouse wide // numeric types as the Rust classifier and the grid (ClickHouse wide
// integers, Oracle/Dameng binary floats, SQL Server internal names, etc.). // integers, Oracle/Dameng binary floats, SQL Server internal names, etc.).
const columnTypes = [ const columnTypes = ["Int16", "Int32", "Int64", "Int128", "UInt256", "Decimal128(18, 2)", "Float16", "BINARY_FLOAT", "BINARY_DOUBLE", "decimaln", "numericn", "intn", "floatn", "moneyn", "smallmoneyn", "varchar(50)"];
"Int16",
"Int32",
"Int64",
"Int128",
"UInt256",
"Decimal128(18, 2)",
"Float16",
"BINARY_FLOAT",
"BINARY_DOUBLE",
"decimaln",
"numericn",
"intn",
"floatn",
"moneyn",
"smallmoneyn",
"varchar(50)",
];
const workbook = buildXlsxWorkbook({ const workbook = buildXlsxWorkbook({
sheetName: "CrossDb", sheetName: "CrossDb",
columns: columnTypes.map((t) => t.toLowerCase()), columns: columnTypes.map((t) => t.toLowerCase()),