From ce8cb2b509aa67062c91693612392805c0fcddbd Mon Sep 17 00:00:00 2001 From: zipg Date: Thu, 9 Jul 2026 15:04:34 +0800 Subject: [PATCH] fix(oracle): preserve DATE time in display and export --- agents/drivers/oracle-go/main.go | 46 +++++-- agents/drivers/oracle-go/main_test.go | 7 +- apps/desktop/src/components/grid/DataGrid.vue | 16 ++- .../src/composables/useDataGridExport.ts | 5 +- .../dataGrid/dataGridCellCoercion.spec.ts | 34 ++++++ .../src/lib/dataGrid/dataGridCellCoercion.ts | 36 ++++++ apps/desktop/src/lib/dataGrid/dataGridSql.ts | 1 + .../queryStore.database-open.spec.ts | 2 + apps/desktop/src/stores/queryStore.ts | 6 + crates/dbx-core/Cargo.toml | 7 ++ crates/dbx-core/src/data_grid_sql.rs | 101 +++++++++++++++- crates/dbx-core/src/database_export.rs | 114 ++++++++++++++++++ .../dbx-core/src/db/duckdb_worker_process.rs | 2 +- crates/dbx-core/src/transfer.rs | 104 ++++++++++++++++ .../dbx-core/tests/duckdb_worker_process.rs | 64 ++++------ .../dbx-core/tests/live_postgres_tsvector.rs | 1 + .../support/duckdb_worker_file_lock_owner.rs | 16 +++ 17 files changed, 504 insertions(+), 58 deletions(-) create mode 100644 apps/desktop/src/lib/__tests__/dataGrid/dataGridCellCoercion.spec.ts create mode 100644 crates/dbx-core/tests/support/duckdb_worker_file_lock_owner.rs diff --git a/agents/drivers/oracle-go/main.go b/agents/drivers/oracle-go/main.go index ae14bccf99..880d480800 100644 --- a/agents/drivers/oracle-go/main.go +++ b/agents/drivers/oracle-go/main.go @@ -174,6 +174,7 @@ type queryOptions struct { type queryResult struct { Columns []string `json:"columns"` + ColumnTypes []string `json:"column_types"` Rows [][]any `json:"rows"` AffectedRows int64 `json:"affected_rows"` ExecutionTimeMS int64 `json:"execution_time_ms"` @@ -186,6 +187,9 @@ func (r queryResult) MarshalJSON() ([]byte, error) { if value.Columns == nil { value.Columns = []string{} } + if value.ColumnTypes == nil { + value.ColumnTypes = []string{} + } if value.Rows == nil { value.Rows = [][]any{} } @@ -194,6 +198,7 @@ func (r queryResult) MarshalJSON() ([]byte, error) { type queryPageResult struct { Columns []string `json:"columns"` + ColumnTypes []string `json:"column_types"` Rows [][]any `json:"rows"` AffectedRows int64 `json:"affected_rows"` ExecutionTimeMS int64 `json:"execution_time_ms"` @@ -208,6 +213,9 @@ func (r queryPageResult) MarshalJSON() ([]byte, error) { if value.Columns == nil { value.Columns = []string{} } + if value.ColumnTypes == nil { + value.ColumnTypes = []string{} + } if value.Rows == nil { value.Rows = [][]any{} } @@ -215,10 +223,11 @@ func (r queryPageResult) MarshalJSON() ([]byte, error) { } type querySession struct { - rows *sql.Rows - columns []string - pending []any - remaining int + rows *sql.Rows + columns []string + columnTypes []string + pending []any + remaining int } type oracleColumnMeta struct { @@ -1618,6 +1627,7 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes result, err := s.executeQuery(opts) return queryPageResult{ Columns: result.Columns, + ColumnTypes: result.ColumnTypes, Rows: result.Rows, AffectedRows: result.AffectedRows, ExecutionTimeMS: result.ExecutionTimeMS, @@ -1635,11 +1645,12 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes rows.Close() return queryPageResult{}, err } + columnTypes := columnTypeNames(rows) maxRows := opts.MaxRows if maxRows <= 0 { maxRows = defaultMaxRows } - session := &querySession{rows: rows, columns: columns, remaining: maxRows} + session := &querySession{rows: rows, columns: columns, columnTypes: columnTypes, remaining: maxRows} result, err := readQuerySessionPage(session, pageSize) result.ExecutionTimeMS = time.Since(start).Milliseconds() if err != nil { @@ -1658,7 +1669,7 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes func (s *server) fetchQueryPage(sessionID string, pageSize int) (queryPageResult, error) { session := s.sessions[sessionID] if session == nil { - return queryPageResult{Columns: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil + return queryPageResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil } result, err := readQuerySessionPage(session, pageSize) if err != nil { @@ -1700,11 +1711,12 @@ func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResul rows.Close() return queryPageResult{}, err } + columnTypes := columnTypeNames(rows) maxRows := opts.MaxRows if maxRows <= 0 { maxRows = defaultMaxRows } - session := &querySession{rows: rows, columns: columns, remaining: maxRows} + session := &querySession{rows: rows, columns: columns, columnTypes: columnTypes, remaining: maxRows} result, err := readQuerySessionPage(session, pageSize) result.ExecutionTimeMS = time.Since(start).Milliseconds() if err != nil { @@ -1723,7 +1735,7 @@ func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResul func (s *server) fetchTableReadPage(sessionID string, pageSize int) (queryPageResult, error) { session := s.tableReadSessions[sessionID] if session == nil { - return queryPageResult{Columns: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil + return queryPageResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil } result, err := readQuerySessionPage(session, pageSize) if err != nil { @@ -1778,7 +1790,7 @@ func readQuerySessionPage(session *querySession, pageSize int) (queryPageResult, if pageSize <= 0 { pageSize = defaultMaxRows } - result := queryPageResult{Columns: session.columns, Rows: [][]any{}, SessionID: nil, HasMore: false} + result := queryPageResult{Columns: session.columns, ColumnTypes: session.columnTypes, Rows: [][]any{}, SessionID: nil, HasMore: false} for len(result.Rows) < pageSize && session.remaining > 0 { if session.pending != nil { result.Rows = append(result.Rows, session.pending) @@ -1838,7 +1850,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) { return queryResult{}, err } affected, _ := execResult.RowsAffected() - return queryResult{Columns: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil + return queryResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil } func (s *server) executeSelect(sqlText string, maxRows int) (queryResult, error) { @@ -1851,7 +1863,7 @@ func (s *server) executeSelect(sqlText string, maxRows int) (queryResult, error) if err != nil { return queryResult{}, err } - result := queryResult{Columns: columns, Rows: [][]any{}} + result := queryResult{Columns: columns, ColumnTypes: columnTypeNames(rows), Rows: [][]any{}} for rows.Next() { if len(result.Rows) >= maxRows { result.Truncated = true @@ -1881,6 +1893,18 @@ func scanRow(rows *sql.Rows, columnCount int) ([]any, error) { return values, nil } +func columnTypeNames(rows *sql.Rows) []string { + types, err := rows.ColumnTypes() + if err != nil { + return []string{} + } + result := make([]string, 0, len(types)) + for _, columnType := range types { + result = append(result, columnType.DatabaseTypeName()) + } + return result +} + func (s *server) queryRowsWithXMLTypeRewriteIfNeeded(sqlText string) (*sql.Rows, error) { rows, err := s.queryRows(sqlText, nil) if err != nil { diff --git a/agents/drivers/oracle-go/main_test.go b/agents/drivers/oracle-go/main_test.go index b152d0c4ae..8a20b0abe5 100644 --- a/agents/drivers/oracle-go/main_test.go +++ b/agents/drivers/oracle-go/main_test.go @@ -79,7 +79,7 @@ func TestMissingTableReadSessionMethodsReturnEmptyOrFalse(t *testing.T) { if err := json.Unmarshal(data, &page); err != nil { t.Fatal(err) } - if len(page.Columns) != 0 || len(page.Rows) != 0 || page.HasMore || page.SessionID != nil { + if len(page.Columns) != 0 || len(page.ColumnTypes) != 0 || len(page.Rows) != 0 || page.HasMore || page.SessionID != nil { t.Fatalf("missing table read session should return empty page, got %+v", page) } @@ -101,9 +101,12 @@ func TestEmptyResultSlicesMarshalAsArrays(t *testing.T) { t.Fatal(err) } text := string(data) - if strings.Contains(text, `"columns":null`) || strings.Contains(text, `"rows":null`) { + if strings.Contains(text, `"columns":null`) || strings.Contains(text, `"column_types":null`) || strings.Contains(text, `"rows":null`) { t.Fatalf("query result should marshal nil slices as arrays: %s", text) } + if !strings.Contains(text, `"column_types":[]`) { + t.Fatalf("query result should marshal empty column types array: %s", text) + } data, err = json.Marshal(indexInfo{}) if err != nil { diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index 287c29d07f..b65537b9b0 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -4024,6 +4024,11 @@ function tableColumnForGridColumn(columnIndex: number): ColumnInfo | undefined { return props.tableMeta?.columns.find((column) => column.name.toLowerCase() === columnName.toLowerCase()); } +function resultColumnInfoForGridColumn(columnIndex: number): Pick | undefined { + const dataType = props.result.column_types?.[columnIndex]?.trim(); + return dataType ? { data_type: dataType } : undefined; +} + function temporalEditorKindForColumn(columnIndex: number): TemporalCellEditorKind | undefined { return temporalCellEditorKind(tableColumnForGridColumn(columnIndex)?.data_type, props.databaseType); } @@ -5351,7 +5356,8 @@ function formatCell(value: CellValue, columnIndex?: number): string { const formatter = columnIndex === undefined ? undefined : resolvedColumnFormatters.value[columnIndex]; const columnName = columnIndex === undefined ? undefined : props.result.columns[columnIndex]; const columnInfo = columnIndex === undefined ? undefined : tableColumnForGridColumn(columnIndex); - const arrayDisplay = formatter ? undefined : dataGridCellDisplayText({ value, databaseType: props.databaseType, columnInfo }); + const displayColumnInfo = columnInfo ?? (columnIndex === undefined ? undefined : resultColumnInfoForGridColumn(columnIndex)); + const arrayDisplay = formatter ? undefined : dataGridCellDisplayText({ value, databaseType: props.databaseType, columnInfo: displayColumnInfo }); if (arrayDisplay !== undefined) return arrayDisplay; const binaryDisplay = formatter ? null : binaryCellDisplayText(value, columnInfo?.data_type ?? (columnName ? columnTypeMap.value.get(columnName) : undefined)); if (binaryDisplay) return binaryDisplay; @@ -5392,7 +5398,13 @@ function draftCellPlaceholder(item: RowItem | undefined, columnIndex: number): s return item?.isDraft && item.data[columnIndex] === null ? t("grid.quickEntryDraftPlaceholder") : null; } -watch(() => [props.result.columns.join("\u0000"), columnFormatterSignatures.value.join("\u0000")], clearCellFormatCache); +function columnTypeCacheSignature(): string { + const resultTypes = props.result.column_types?.join("\u0000") ?? ""; + const tableTypes = props.tableMeta?.columns?.map((column) => `${column.name}:${column.data_type}`).join("\u0000") ?? ""; + return `${resultTypes}\u0001${tableTypes}`; +} + +watch(() => [props.result.columns.join("\u0000"), columnFormatterSignatures.value.join("\u0000"), columnTypeCacheSignature()], clearCellFormatCache); function quoteIdent(name: string): string { return quoteTableIdentifier(props.databaseType, name); diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index 226c2db993..95b03e3056 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -237,6 +237,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { tableName: tableMeta.value?.tableName ?? null, copyInsertTargetLabel: copyInsertTargetLabel?.value ?? null, columns: columns.value, + columnTypes: columnTypes.value ?? null, sourceColumns: sourceColumns.value ?? null, excludePrimaryKeys, insertMode, @@ -311,6 +312,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { databaseType: databaseType.value, tableMeta: tableMeta.value, columns: columns.value, + columnTypes: columnTypes.value, sourceColumns: sourceColumns.value, rows: rows.map((item) => item.data), excludePrimaryKeys, @@ -1075,9 +1077,10 @@ export function useDataGridExport(options: UseDataGridExportOptions) { } { const exportColumns = tableMeta.value ? effectiveColumns(sourceColumns.value, result.columns) : result.columns; const columnIndexes = exportColumns.map((column, index) => ({ column, index })).filter((item): item is { column: string; index: number } => !!item.column); + const exportColumnTypes = columnTypes.value?.length === result.columns.length ? columnTypes.value : undefined; return { columns: columnIndexes.map((item) => item.column), - columnTypes: tableMeta.value ? columnIndexes.map((item) => columnTypes.value?.[item.index]) : undefined, + columnTypes: exportColumnTypes ? columnIndexes.map((item) => exportColumnTypes[item.index]) : undefined, rows: result.rows.map((row) => columnIndexes.map((item) => row[item.index] ?? null)), }; } diff --git a/apps/desktop/src/lib/__tests__/dataGrid/dataGridCellCoercion.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/dataGridCellCoercion.spec.ts new file mode 100644 index 0000000000..42832db5c7 --- /dev/null +++ b/apps/desktop/src/lib/__tests__/dataGrid/dataGridCellCoercion.spec.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { dataGridCellDisplayText } from "@/lib/dataGrid/dataGridCellCoercion"; + +describe("dataGridCellDisplayText", () => { + it("formats Oracle DATE values without RFC3339 separators", () => { + expect( + dataGridCellDisplayText({ + value: "2022-08-25T09:58:43Z", + databaseType: "oracle", + columnInfo: { data_type: "DATE" }, + }), + ).toBe("2022-08-25 09:58:43"); + }); + + it("formats midnight Oracle DATE values as a date", () => { + expect( + dataGridCellDisplayText({ + value: "2022-08-25T00:00:00Z", + databaseType: "oracle", + columnInfo: { data_type: "DATE" }, + }), + ).toBe("2022-08-25"); + }); + + it("does not format non-date Oracle strings", () => { + expect( + dataGridCellDisplayText({ + value: "2022-08-25T09:58:43Z", + databaseType: "oracle", + columnInfo: { data_type: "VARCHAR2(64)" }, + }), + ).toBeUndefined(); + }); +}); diff --git a/apps/desktop/src/lib/dataGrid/dataGridCellCoercion.ts b/apps/desktop/src/lib/dataGrid/dataGridCellCoercion.ts index 35663b40b1..84b2e21486 100644 --- a/apps/desktop/src/lib/dataGrid/dataGridCellCoercion.ts +++ b/apps/desktop/src/lib/dataGrid/dataGridCellCoercion.ts @@ -45,6 +45,9 @@ export function dataGridCellDisplayText(options: { value: GridCellValue; databas if (Array.isArray(options.value) && options.databaseType === "postgres" && isPostgresArrayColumn(options.columnInfo, options.value)) { return formatPostgresArrayText(options.value); } + if (typeof options.value === "string" && isOracleDateColumn(options.databaseType, options.columnInfo)) { + return formatOracleDateDisplayText(options.value); + } return undefined; } @@ -113,6 +116,39 @@ function normalizeDataType(dataType: string | undefined): string { return (dataType ?? "").trim().toLowerCase(); } +function isOracleDateColumn(databaseType: DatabaseType | undefined, columnInfo: Pick | undefined): boolean { + if (databaseType !== "oracle" && databaseType !== "oceanbase-oracle") return false; + const base = normalizeDataType(columnInfo?.data_type).split(/[()\s\t\n]/)[0] ?? ""; + return base === "date"; +} + +function formatOracleDateDisplayText(value: string): string | undefined { + const parts = parseOracleDateLikeText(value); + if (!parts) return undefined; + if (parts.time === "00:00:00" && !parts.fraction) return parts.date; + return `${parts.date} ${parts.time}${parts.fraction ?? ""}`; +} + +function parseOracleDateLikeText(value: string): { date: string; time: string; fraction?: string } | undefined { + if (!/^\d{4}-\d{2}-\d{2}/.test(value)) return undefined; + const date = value.slice(0, 10); + if (value.length === 10) return { date, time: "00:00:00" }; + const separator = value[10]; + if (separator !== "T" && separator !== " ") return undefined; + if (!/^\d{2}:\d{2}:\d{2}/.test(value.slice(11))) return undefined; + const time = value.slice(11, 19); + let rest = value.slice(19); + let fraction: string | undefined; + if (rest.startsWith(".")) { + const match = rest.match(/^(\.\d{1,9})(.*)$/); + if (!match) return undefined; + fraction = match[1]; + rest = match[2]; + } + if (rest && !/^z$/i.test(rest) && !/^[+-]\d{2}:\d{2}$/.test(rest)) return undefined; + return { date, time, fraction }; +} + function isExactDecimalDataType(dataType: string): boolean { return /\b(?:decimal|numeric|number|dec|money|smallmoney|bigdecimal|bignumeric|big_numeric|fixed)\b/.test(dataType); } diff --git a/apps/desktop/src/lib/dataGrid/dataGridSql.ts b/apps/desktop/src/lib/dataGrid/dataGridSql.ts index a4938cea44..1cc9a5089a 100644 --- a/apps/desktop/src/lib/dataGrid/dataGridSql.ts +++ b/apps/desktop/src/lib/dataGrid/dataGridSql.ts @@ -45,6 +45,7 @@ export interface DataGridCopyInsertStatementOptions { databaseType?: DatabaseType; tableMeta?: DataGridTableMeta; columns: string[]; + columnTypes?: Array; sourceColumns?: Array; rows: GridCellValue[][]; excludePrimaryKeys?: boolean; diff --git a/apps/desktop/src/stores/__tests__/queryStore.database-open.spec.ts b/apps/desktop/src/stores/__tests__/queryStore.database-open.spec.ts index d0665ba45f..7ee2ad4595 100644 --- a/apps/desktop/src/stores/__tests__/queryStore.database-open.spec.ts +++ b/apps/desktop/src/stores/__tests__/queryStore.database-open.spec.ts @@ -156,6 +156,7 @@ describe("queryStore database open state", () => { vi.unstubAllGlobals(); const storage = installLocalStorage(); storage.set("dbx-editor-settings", JSON.stringify({ openTabsRestoreMode: "none" })); + storage.set("dbx-app-state:open_tabs", JSON.stringify({ tabs: JSON.parse(persistedTabs), activeTabId: "tab-1" })); storage.set(OPEN_TABS_STORAGE_KEY, persistedTabs); storage.set(ACTIVE_TAB_STORAGE_KEY, "tab-1"); setActivePinia(createPinia()); @@ -170,6 +171,7 @@ describe("queryStore database open state", () => { expect(store.activeTabId).toBeNull(); expect(storage.get(OPEN_TABS_STORAGE_KEY)).toBeUndefined(); expect(storage.get(ACTIVE_TAB_STORAGE_KEY)).toBeUndefined(); + expect(JSON.parse(storage.get("dbx-app-state:open_tabs") ?? "{}")).toEqual({ tabs: [], activeTabId: null }); }); it("restores only pinned tabs when launch restore mode is pinned", async () => { diff --git a/apps/desktop/src/stores/queryStore.ts b/apps/desktop/src/stores/queryStore.ts index 1abcce9c34..09adc08a3c 100644 --- a/apps/desktop/src/stores/queryStore.ts +++ b/apps/desktop/src/stores/queryStore.ts @@ -681,6 +681,12 @@ export const useQueryStore = defineStore("query", () => { if (saved?.tabs && Array.isArray(saved.tabs)) { const restored = restoreSavedTabsFromPayload(saved); applyRestoredOpenTabs(restored); + if (useSettingsStore().editorSettings.openTabsRestoreMode === "none") { + // Restore is explicitly disabled, so stale saved payloads should not + // reappear if the user later changes the setting. + clearLegacySavedTabs(); + await saveTabs(tabs.value, activeTabId.value).catch(() => undefined); + } isOpenTabsLoaded.value = true; return; } diff --git a/crates/dbx-core/Cargo.toml b/crates/dbx-core/Cargo.toml index ef6de41fd6..93aeb2fe1e 100644 --- a/crates/dbx-core/Cargo.toml +++ b/crates/dbx-core/Cargo.toml @@ -24,6 +24,13 @@ required-features = ["duckdb-bundled"] test = false bench = false +[[bin]] +name = "duckdb-worker-file-lock-owner" +path = "tests/support/duckdb_worker_file_lock_owner.rs" +required-features = ["duckdb-bundled"] +test = false +bench = false + [features] default = ["duckdb-bundled", "mq-admin", "sqlite-sqlcipher"] duckdb-bundled = ["duckdb/bundled"] diff --git a/crates/dbx-core/src/data_grid_sql.rs b/crates/dbx-core/src/data_grid_sql.rs index 291fc87b82..63134abc1e 100644 --- a/crates/dbx-core/src/data_grid_sql.rs +++ b/crates/dbx-core/src/data_grid_sql.rs @@ -93,6 +93,8 @@ pub struct DataGridCopyInsertStatementOptions { pub table_meta: Option, pub columns: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] + pub column_types: Option>>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub source_columns: Option>>, #[serde(default)] pub rows: Vec>, @@ -372,10 +374,19 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen insert_columns .iter() .map(|(column, index)| { - format_grid_sql_literal( + format_grid_copy_insert_sql_literal( row.get(*index).unwrap_or(&Value::Null), options.database_type, - column_info_for(column_info, column), + copy_column_info( + column_info, + column, + options + .column_types + .as_deref() + .and_then(|types| types.get(*index)) + .and_then(|value| value.as_deref()), + ) + .as_ref(), ) }) .collect::>() @@ -1020,6 +1031,24 @@ fn effective_copy_columns(source_columns: Option<&[Option]>, columns: &[ } } +fn copy_column_info( + column_info: &[DataGridColumnInfo], + column: &str, + fallback_type: Option<&str>, +) -> Option { + if let Some(info) = column_info_for(column_info, column) { + return Some(info.clone()); + } + fallback_type.map(|data_type| DataGridColumnInfo { + name: column.to_string(), + data_type: data_type.to_string(), + is_nullable: true, + is_primary_key: false, + column_default: None, + extra: None, + }) +} + fn effective_column(options: &DataGridSaveStatementOptions, index: usize) -> Option<&str> { match &options.source_columns { Some(source_columns) if source_columns.len() == options.columns.len() => source_columns.get(index)?.as_deref(), @@ -1046,6 +1075,23 @@ pub fn normalize_data_grid_save_error(database_type: Option, error error.to_string() } +fn format_grid_copy_insert_sql_literal( + value: &Value, + database_type: Option, + column_info: Option<&DataGridColumnInfo>, +) -> String { + if is_oracle_temporal_literal_database(database_type) { + if let Some(text) = value.as_str() { + if let Some(literal) = + format_oracle_temporal_literal(text, column_info.map(|column| column.data_type.as_str())) + { + return literal; + } + } + } + format_grid_sql_literal(value, database_type, column_info) +} + pub fn format_grid_sql_literal( value: &Value, database_type: Option, @@ -1169,9 +1215,12 @@ fn is_oracle_temporal_literal_database(database_type: Option) -> b fn format_oracle_temporal_literal(text: &str, data_type: Option<&str>) -> Option { let kind = oracle_temporal_column_kind(data_type?)?; let parts = regex_like_oracle_temporal(text)?; - let fraction = parts.fraction.unwrap_or_default(); + let fraction = parts.fraction.as_deref().unwrap_or_default(); let datetime = format!("{} {}{}", parts.date, parts.time, fraction); match kind { + OracleTemporalKind::Date if oracle_temporal_parts_are_midnight(&parts) => { + Some(format!("DATE '{}'", parts.date)) + } OracleTemporalKind::Date => Some(format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time)), OracleTemporalKind::Timestamp => { let mask = oracle_timestamp_format_mask(datetime.contains('.')); @@ -1189,6 +1238,15 @@ fn format_oracle_temporal_literal(text: &str, data_type: Option<&str>) -> Option } } +fn oracle_temporal_parts_are_midnight(parts: &Rfc3339Parts) -> bool { + parts.time == "00:00:00" + && parts + .fraction + .as_deref() + .map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0')) + .unwrap_or(true) +} + fn oracle_temporal_column_kind(data_type: &str) -> Option { let lower = data_type.trim().to_ascii_lowercase(); let base = lower.split(['(', ' ']).next().unwrap_or(""); @@ -2103,6 +2161,7 @@ mod tests { columns: None, }), columns: vec!["id".to_string(), "login_name".to_string(), "display_name".to_string()], + column_types: None, source_columns: None, rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]], exclude_primary_keys: true, @@ -2126,6 +2185,7 @@ mod tests { columns: None, }), columns: vec!["id".to_string(), "login_name".to_string(), "display_name".to_string()], + column_types: None, source_columns: None, rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]], exclude_primary_keys: false, @@ -2151,6 +2211,7 @@ mod tests { columns: None, }), columns: vec!["ID".to_string(), "NAME".to_string()], + column_types: None, source_columns: None, rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]], exclude_primary_keys: false, @@ -2179,6 +2240,7 @@ mod tests { database_type: Some(DatabaseType::Mysql), table_meta: Some(table_meta.clone()), columns: columns.clone(), + column_types: None, source_columns: None, rows: rows.clone(), exclude_primary_keys: false, @@ -2241,6 +2303,7 @@ mod tests { ]), }), columns: vec!["id".to_string(), "title".to_string(), "search_vector".to_string()], + column_types: None, source_columns: None, rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]], exclude_primary_keys: false, @@ -2253,6 +2316,29 @@ mod tests { ); } + #[test] + fn oracle_copy_insert_uses_result_column_types_for_date_literals() { + let statement = build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions { + database_type: Some(DatabaseType::Oracle), + table_meta: None, + columns: vec!["ID".to_string(), "CREATED_ON".to_string(), "RAW_TEXT".to_string()], + column_types: Some(vec![ + Some("NUMBER".to_string()), + Some("DATE".to_string()), + Some("VARCHAR2".to_string()), + ]), + source_columns: None, + rows: vec![vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")]], + exclude_primary_keys: false, + insert_mode: DataGridCopyInsertMode::Merged, + }); + + assert_eq!( + statement.as_deref(), + Some("INSERT INTO table_name (\"ID\", \"CREATED_ON\", \"RAW_TEXT\") VALUES (1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z');") + ); + } + #[test] fn builds_filter_conditions() { assert_eq!( @@ -2601,6 +2687,14 @@ mod tests { format_grid_sql_literal(&json!("2022-08-25 09:58:43"), Some(DatabaseType::Oracle), Some(&date)), "TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS')" ); + assert_eq!( + format_grid_sql_literal(&json!("2022-08-25"), Some(DatabaseType::Oracle), Some(&date)), + "DATE '2022-08-25'" + ); + assert_eq!( + format_grid_sql_literal(&json!("2022-08-25T00:00:00Z"), Some(DatabaseType::Oracle), Some(&date)), + "DATE '2022-08-25'" + ); } #[test] @@ -3032,6 +3126,7 @@ mod tests { database_type: Some(DatabaseType::Doris), table_meta: Some(table_meta.clone()), columns: vec!["id".to_string(), "status".to_string()], + column_types: None, source_columns: None, rows: vec![vec![json!(2), json!("new")]], exclude_primary_keys: false, diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index 8028a659a5..fdab47ff4c 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -206,6 +206,9 @@ fn format_export_sql_literal_typed( return format_ch_array_sql_literal(arr); } } + if let Some(literal) = format_oracle_export_date_literal(value, database_type, column_type) { + return literal; + } if let Some(literal) = format_export_temporal_literal(value, database_type, column_type) { return literal; } @@ -295,6 +298,43 @@ fn is_mysql_compatible_export_literal_target(database_type: Option ) } +fn format_oracle_export_date_literal( + value: &Value, + database_type: Option, + column_type: Option<&str>, +) -> Option { + if !matches!(database_type, Some(DatabaseType::Oracle | DatabaseType::OceanbaseOracle)) { + return None; + } + if export_temporal_column_kind(database_type, column_type?)? != ExportTemporalKind::DateTime { + return None; + } + let lower = column_type?.trim().trim_matches('"').to_ascii_lowercase(); + let base = lower.split(['(', ' ', '\t', '\n']).next().unwrap_or(""); + if base != "date" { + return None; + } + let parts = parse_export_date_parts(value.as_str()?)?; + Some(format_oracle_export_date_parts_literal(&parts)) +} + +fn format_oracle_export_date_parts_literal(parts: &ExportRfc3339Parts) -> String { + if export_temporal_parts_are_midnight(parts) { + format!("DATE '{}'", parts.date) + } else { + format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time) + } +} + +fn export_temporal_parts_are_midnight(parts: &ExportRfc3339Parts) -> bool { + parts.time == "00:00:00" + && parts + .fraction + .as_deref() + .map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0')) + .unwrap_or(true) +} + fn format_export_temporal_literal( value: &Value, database_type: Option, @@ -363,6 +403,53 @@ struct ExportRfc3339Parts { zone: String, } +fn parse_export_date_parts(text: &str) -> Option { + parse_export_rfc3339_parts(text).or_else(|| parse_export_local_temporal_parts(text)) +} + +fn parse_export_local_temporal_parts(text: &str) -> Option { + let bytes = text.as_bytes(); + if bytes.len() < 10 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') { + return None; + } + let date = &text[0..10]; + if !date.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) { + return None; + } + if bytes.len() == 10 { + return Some(ExportRfc3339Parts { + date: date.to_string(), + time: "00:00:00".to_string(), + fraction: None, + zone: String::new(), + }); + } + let separator = *bytes.get(10)?; + if separator != b'T' && separator != b' ' { + return None; + } + if bytes.len() < 19 || bytes.get(13) != Some(&b':') || bytes.get(16) != Some(&b':') { + return None; + } + let time = &text[11..19]; + if !time.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 2 | 5) || byte.is_ascii_digit()) { + return None; + } + let rest = &text[19..]; + let fraction = if let Some(rest) = rest.strip_prefix('.') { + let digit_count = rest.chars().take_while(|ch| ch.is_ascii_digit()).count(); + if digit_count == 0 || digit_count > 9 || digit_count != rest.len() { + return None; + } + Some(format!(".{}", &rest[..digit_count])) + } else if rest.is_empty() { + None + } else { + return None; + }; + Some(ExportRfc3339Parts { date: date.to_string(), time: time.to_string(), fraction, zone: String::new() }) +} + fn parse_export_rfc3339_parts(text: &str) -> Option { let bytes = text.as_bytes(); if bytes.len() < 20 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') { @@ -1635,6 +1722,33 @@ mod tests { ); } + #[test] + fn oracle_date_columns_export_as_date_literals() { + let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(DatabaseType::Oracle), + schema: Some("APP".to_string()), + table_name: Some("EVENTS".to_string()), + qualified_table_name: None, + columns: vec!["ID".to_string(), "CREATED_ON".to_string(), "RAW_TEXT".to_string()], + column_types: vec![Some("NUMBER".to_string()), Some("DATE".to_string()), Some("VARCHAR2(64)".to_string())], + column_extras: Vec::new(), + rows: vec![ + vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")], + vec![json!(2), json!("2022-08-25T00:00:00Z"), json!("2022-08-25T00:00:00Z")], + ], + batch_size: Some(100), + }) + .unwrap(); + + assert_eq!( + statements, + vec![ + "INSERT INTO \"APP\".\"EVENTS\" (\"ID\", \"CREATED_ON\", \"RAW_TEXT\") VALUES (1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z');", + "INSERT INTO \"APP\".\"EVENTS\" (\"ID\", \"CREATED_ON\", \"RAW_TEXT\") VALUES (2, DATE '2022-08-25', '2022-08-25T00:00:00Z');", + ] + ); + } + #[test] fn mysql_bit_columns_export_without_quoted_string_values() { let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { diff --git a/crates/dbx-core/src/db/duckdb_worker_process.rs b/crates/dbx-core/src/db/duckdb_worker_process.rs index ca2f0fc0af..ae29bd4b85 100644 --- a/crates/dbx-core/src/db/duckdb_worker_process.rs +++ b/crates/dbx-core/src/db/duckdb_worker_process.rs @@ -547,7 +547,7 @@ fn is_transient_duckdb_file_lock_error(message: &str) -> bool { || lower.contains("file is already open"); let mentions_lock = lower.contains("file is already open") || lower.contains("being used by another process") - || lower.contains("conflicting lock is held") + || lower.contains("conflicting lock") || lower.contains("process cannot access the file") || lower.contains("sharing violation") || lower.contains("resource temporarily unavailable") diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index c2724f716d..46baecf678 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -1014,6 +1014,9 @@ pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, colum if let Some(numeric_literal) = format_mysql_numeric_string_literal(s, db_type, column_type) { return numeric_literal; } + if let Some(date_literal) = format_oracle_date_sql_literal(s, db_type, column_type) { + return date_literal; + } let literal = format_literal_string(s, db_type, column_type); let escaped = if is_postgres_family_target(db_type) { @@ -1102,6 +1105,76 @@ fn format_postgres_binary_sql_literal( Some(format!("decode('{hex}', 'hex')")) } +fn format_oracle_date_sql_literal(value: &str, db_type: &DatabaseType, column_type: Option<&str>) -> Option { + if !matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) { + return None; + } + if temporal_column_kind(column_type) != Some("date") { + return None; + } + let parts = oracle_export_date_parts(value)?; + Some(format_oracle_date_sql_literal_parts(&parts)) +} + +struct OracleExportDateParts<'a> { + date: &'a str, + time: &'a str, + fraction: Option<&'a str>, +} + +fn format_oracle_date_sql_literal_parts(parts: &OracleExportDateParts<'_>) -> String { + if oracle_export_date_parts_are_midnight(parts) { + format!("DATE '{}'", parts.date) + } else { + format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time) + } +} + +fn oracle_export_date_parts_are_midnight(parts: &OracleExportDateParts<'_>) -> bool { + parts.time == "00:00:00" + && parts.fraction.map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0')).unwrap_or(true) +} + +fn oracle_export_date_parts(value: &str) -> Option> { + let bytes = value.as_bytes(); + if bytes.len() < 10 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') { + return None; + } + let date = &value[..10]; + if !date.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) { + return None; + } + if bytes.len() == 10 { + return Some(OracleExportDateParts { date, time: "00:00:00", fraction: None }); + } + let separator = *bytes.get(10)?; + if separator != b'T' && separator != b' ' { + return None; + } + if bytes.len() < 19 || bytes.get(13) != Some(&b':') || bytes.get(16) != Some(&b':') { + return None; + } + let time = &value[11..19]; + if !time.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 2 | 5) || byte.is_ascii_digit()) { + return None; + } + let rest = &value[19..]; + if rest.is_empty() || is_timezone_suffix(rest) { + return Some(OracleExportDateParts { date, time, fraction: None }); + } + if let Some(after_dot) = rest.strip_prefix('.') { + let digit_count = after_dot.bytes().take_while(|byte| byte.is_ascii_digit()).count(); + if digit_count == 0 { + return None; + } + let zone = &after_dot[digit_count..]; + if zone.is_empty() || is_timezone_suffix(zone) { + return Some(OracleExportDateParts { date, time, fraction: Some(&value[19..19 + 1 + digit_count]) }); + } + } + None +} + pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String { if arr.is_empty() { return "'{}'".to_string(); @@ -5641,6 +5714,37 @@ mod tests { ); } + #[test] + fn oracle_insert_uses_date_literals_for_date_columns() { + let sql = generate_insert_typed( + &[String::from("id"), String::from("created_on"), String::from("created_at"), String::from("raw_text")], + &[ + Some(String::from("NUMBER")), + Some(String::from("DATE")), + Some(String::from("TIMESTAMP(6)")), + Some(String::from("VARCHAR2(64)")), + ], + &[vec![ + json!(1), + json!("2022-08-25T09:58:43Z"), + json!("2022-08-25T09:58:43Z"), + json!("2022-08-25T09:58:43Z"), + ]], + "events", + "APP", + &DatabaseType::Oracle, + ); + + assert_eq!( + sql, + "INSERT INTO \"APP\".\"events\" (\"id\", \"created_on\", \"created_at\", \"raw_text\") VALUES\n(1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z', '2022-08-25T09:58:43Z')" + ); + assert_eq!( + escape_value_typed(&json!("2022-08-25T00:00:00Z"), &DatabaseType::Oracle, Some("DATE")), + "DATE '2022-08-25'" + ); + } + #[test] fn mysql_insert_formats_numeric_strings_from_numeric_columns_as_numeric_literals() { let sql = generate_insert_typed( diff --git a/crates/dbx-core/tests/duckdb_worker_process.rs b/crates/dbx-core/tests/duckdb_worker_process.rs index 5836c4fdac..04b3589f85 100644 --- a/crates/dbx-core/tests/duckdb_worker_process.rs +++ b/crates/dbx-core/tests/duckdb_worker_process.rs @@ -321,12 +321,11 @@ async fn worker_process_is_killed_after_connect_timeout() { async fn worker_process_is_killed_after_connect_error() { let _guard = duckdb_worker_process_test_guard().await; let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-pid-test-host")); - let db_path = temp_duckdb_path(); + let missing_dir = temp_duckdb_path(); + let db_path = missing_dir.join("missing.duckdb"); let pid_file = temp_pid_path(); let _ = std::fs::remove_file(&pid_file); - let _ = std::fs::remove_file(&db_path); - - let _file_owner = duckdb::Connection::open(&db_path).expect("open lock owner connection"); + let _ = std::fs::remove_dir_all(&missing_dir); std::env::set_var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE", &pid_file); let client = DuckDbWorkerClient::new_unconnected_with_timeouts( @@ -341,15 +340,13 @@ async fn worker_process_is_killed_after_connect_error() { let err = client .execute(None, "SELECT 1".to_string(), Some(10), None, Some(Duration::from_secs(5))) .await - .expect_err("connect should fail while another process owns the DuckDB file"); + .expect_err("connect should fail with an invalid DuckDB path"); std::env::remove_var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE"); assert!( err.contains("Cannot open file") - || err.contains("Could not set lock") - || err.contains("Conflicting lock is held") - || err.contains("already open") - || err.contains("timed out") - || err.contains("另一个程序正在使用"), + || err.contains("No such file") + || err.contains("Parent directory does not exist") + || err.contains("不存在"), "unexpected error: {err}" ); @@ -357,45 +354,33 @@ async fn worker_process_is_killed_after_connect_error() { wait_until_process_exits(pid, Duration::from_secs(5)).await; let _ = std::fs::remove_file(&pid_file); - drop(_file_owner); - let _ = std::fs::remove_file(&db_path); + let _ = std::fs::remove_dir_all(&missing_dir); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn worker_process_retries_connect_after_transient_file_lock() { let _guard = duckdb_worker_process_test_guard().await; let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host")); + let lock_owner_executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-file-lock-owner")); let db_path = temp_duckdb_path(); let _ = std::fs::remove_file(&db_path); - let mut owner_child = Command::new(&executable) - .stdin(Stdio::piped()) + let mut lock_owner = Command::new(lock_owner_executable) + .arg(&db_path) + .arg("500") .stdout(Stdio::piped()) .stderr(Stdio::inherit()) + .kill_on_drop(true) .spawn() - .expect("spawn lock owner worker"); - let mut owner_stdin = owner_child.stdin.take().expect("owner worker stdin"); - let owner_stdout = owner_child.stdout.take().expect("owner worker stdout"); - let mut owner_lines = BufReader::new(owner_stdout).lines(); - write_worker_request( - &mut owner_stdin, - DuckDbWorkerRequest::new( - "connect-owner", - DuckDbWorkerMethod::Connect, - DuckDbWorkerConnectParams { path: db_path.to_string_lossy().to_string(), attached_databases: Vec::new() }, - ) - .expect("owner connect request"), - ) - .await; - let connected = read_worker_response(&mut owner_lines).await; - assert!(connected.ok, "owner connect failed: {connected:?}"); - drop(owner_lines); - - let release_owner = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(200)).await; - drop(owner_stdin); - let _ = tokio::time::timeout(Duration::from_secs(5), owner_child.wait()).await; - }); + .expect("spawn lock owner"); + let lock_owner_stdout = lock_owner.stdout.take().expect("lock owner stdout"); + let mut lock_owner_lines = BufReader::new(lock_owner_stdout).lines(); + let ready = tokio::time::timeout(Duration::from_secs(5), lock_owner_lines.next_line()) + .await + .expect("lock owner ready timed out") + .expect("read lock owner ready line") + .expect("lock owner ready line"); + assert_eq!(ready, "ready"); let client = DuckDbWorkerClient::new_unconnected_with_timeouts( executable, @@ -411,7 +396,10 @@ async fn worker_process_retries_connect_after_transient_file_lock() { ) .await; - let _ = release_owner.await; + tokio::time::timeout(Duration::from_secs(5), lock_owner.wait()) + .await + .expect("lock owner should exit") + .expect("wait for lock owner"); client.shutdown().await; let _ = std::fs::remove_file(&db_path); diff --git a/crates/dbx-core/tests/live_postgres_tsvector.rs b/crates/dbx-core/tests/live_postgres_tsvector.rs index 57132406cf..d85994e63b 100644 --- a/crates/dbx-core/tests/live_postgres_tsvector.rs +++ b/crates/dbx-core/tests/live_postgres_tsvector.rs @@ -71,6 +71,7 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser database_type: Some(DatabaseType::Postgres), table_meta: Some(table_meta), columns: result.columns.clone(), + column_types: None, source_columns: None, rows: result.rows.clone(), exclude_primary_keys: false, diff --git a/crates/dbx-core/tests/support/duckdb_worker_file_lock_owner.rs b/crates/dbx-core/tests/support/duckdb_worker_file_lock_owner.rs new file mode 100644 index 0000000000..25ac9890b9 --- /dev/null +++ b/crates/dbx-core/tests/support/duckdb_worker_file_lock_owner.rs @@ -0,0 +1,16 @@ +#![cfg(feature = "duckdb-bundled")] + +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; + +fn main() { + let mut args = std::env::args().skip(1); + let path = args.next().map(PathBuf::from).expect("DuckDB path argument"); + let hold_ms = args.next().expect("hold duration argument").parse::().expect("hold duration ms"); + + let _connection = duckdb::Connection::open(&path).expect("open lock owner connection"); + println!("ready"); + std::io::stdout().flush().expect("flush ready line"); + std::thread::sleep(Duration::from_millis(hold_ms)); +}