fix(oracle): preserve DATE time in display and export

This commit is contained in:
zipg
2026-07-09 15:04:34 +08:00
committed by GitHub
parent aa0abcd670
commit ce8cb2b509
17 changed files with 504 additions and 58 deletions
+35 -11
View File
@@ -174,6 +174,7 @@ type queryOptions struct {
type queryResult struct {
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
@@ -186,6 +187,9 @@ func (r queryResult) MarshalJSON() ([]byte, error) {
if value.Columns == nil {
value.Columns = []string{}
}
if value.ColumnTypes == nil {
value.ColumnTypes = []string{}
}
if value.Rows == nil {
value.Rows = [][]any{}
}
@@ -194,6 +198,7 @@ func (r queryResult) MarshalJSON() ([]byte, error) {
type queryPageResult struct {
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
@@ -208,6 +213,9 @@ func (r queryPageResult) MarshalJSON() ([]byte, error) {
if value.Columns == nil {
value.Columns = []string{}
}
if value.ColumnTypes == nil {
value.ColumnTypes = []string{}
}
if value.Rows == nil {
value.Rows = [][]any{}
}
@@ -215,10 +223,11 @@ func (r queryPageResult) MarshalJSON() ([]byte, error) {
}
type querySession struct {
rows *sql.Rows
columns []string
pending []any
remaining int
rows *sql.Rows
columns []string
columnTypes []string
pending []any
remaining int
}
type oracleColumnMeta struct {
@@ -1618,6 +1627,7 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes
result, err := s.executeQuery(opts)
return queryPageResult{
Columns: result.Columns,
ColumnTypes: result.ColumnTypes,
Rows: result.Rows,
AffectedRows: result.AffectedRows,
ExecutionTimeMS: result.ExecutionTimeMS,
@@ -1635,11 +1645,12 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes
rows.Close()
return queryPageResult{}, err
}
columnTypes := columnTypeNames(rows)
maxRows := opts.MaxRows
if maxRows <= 0 {
maxRows = defaultMaxRows
}
session := &querySession{rows: rows, columns: columns, remaining: maxRows}
session := &querySession{rows: rows, columns: columns, columnTypes: columnTypes, remaining: maxRows}
result, err := readQuerySessionPage(session, pageSize)
result.ExecutionTimeMS = time.Since(start).Milliseconds()
if err != nil {
@@ -1658,7 +1669,7 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes
func (s *server) fetchQueryPage(sessionID string, pageSize int) (queryPageResult, error) {
session := s.sessions[sessionID]
if session == nil {
return queryPageResult{Columns: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil
return queryPageResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil
}
result, err := readQuerySessionPage(session, pageSize)
if err != nil {
@@ -1700,11 +1711,12 @@ func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResul
rows.Close()
return queryPageResult{}, err
}
columnTypes := columnTypeNames(rows)
maxRows := opts.MaxRows
if maxRows <= 0 {
maxRows = defaultMaxRows
}
session := &querySession{rows: rows, columns: columns, remaining: maxRows}
session := &querySession{rows: rows, columns: columns, columnTypes: columnTypes, remaining: maxRows}
result, err := readQuerySessionPage(session, pageSize)
result.ExecutionTimeMS = time.Since(start).Milliseconds()
if err != nil {
@@ -1723,7 +1735,7 @@ func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResul
func (s *server) fetchTableReadPage(sessionID string, pageSize int) (queryPageResult, error) {
session := s.tableReadSessions[sessionID]
if session == nil {
return queryPageResult{Columns: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil
return queryPageResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil
}
result, err := readQuerySessionPage(session, pageSize)
if err != nil {
@@ -1778,7 +1790,7 @@ func readQuerySessionPage(session *querySession, pageSize int) (queryPageResult,
if pageSize <= 0 {
pageSize = defaultMaxRows
}
result := queryPageResult{Columns: session.columns, Rows: [][]any{}, SessionID: nil, HasMore: false}
result := queryPageResult{Columns: session.columns, ColumnTypes: session.columnTypes, Rows: [][]any{}, SessionID: nil, HasMore: false}
for len(result.Rows) < pageSize && session.remaining > 0 {
if session.pending != nil {
result.Rows = append(result.Rows, session.pending)
@@ -1838,7 +1850,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
return queryResult{}, err
}
affected, _ := execResult.RowsAffected()
return queryResult{Columns: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil
return queryResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil
}
func (s *server) executeSelect(sqlText string, maxRows int) (queryResult, error) {
@@ -1851,7 +1863,7 @@ func (s *server) executeSelect(sqlText string, maxRows int) (queryResult, error)
if err != nil {
return queryResult{}, err
}
result := queryResult{Columns: columns, Rows: [][]any{}}
result := queryResult{Columns: columns, ColumnTypes: columnTypeNames(rows), Rows: [][]any{}}
for rows.Next() {
if len(result.Rows) >= maxRows {
result.Truncated = true
@@ -1881,6 +1893,18 @@ func scanRow(rows *sql.Rows, columnCount int) ([]any, error) {
return values, nil
}
func columnTypeNames(rows *sql.Rows) []string {
types, err := rows.ColumnTypes()
if err != nil {
return []string{}
}
result := make([]string, 0, len(types))
for _, columnType := range types {
result = append(result, columnType.DatabaseTypeName())
}
return result
}
func (s *server) queryRowsWithXMLTypeRewriteIfNeeded(sqlText string) (*sql.Rows, error) {
rows, err := s.queryRows(sqlText, nil)
if err != nil {
+5 -2
View File
@@ -79,7 +79,7 @@ func TestMissingTableReadSessionMethodsReturnEmptyOrFalse(t *testing.T) {
if err := json.Unmarshal(data, &page); err != nil {
t.Fatal(err)
}
if len(page.Columns) != 0 || len(page.Rows) != 0 || page.HasMore || page.SessionID != nil {
if len(page.Columns) != 0 || len(page.ColumnTypes) != 0 || len(page.Rows) != 0 || page.HasMore || page.SessionID != nil {
t.Fatalf("missing table read session should return empty page, got %+v", page)
}
@@ -101,9 +101,12 @@ func TestEmptyResultSlicesMarshalAsArrays(t *testing.T) {
t.Fatal(err)
}
text := string(data)
if strings.Contains(text, `"columns":null`) || strings.Contains(text, `"rows":null`) {
if strings.Contains(text, `"columns":null`) || strings.Contains(text, `"column_types":null`) || strings.Contains(text, `"rows":null`) {
t.Fatalf("query result should marshal nil slices as arrays: %s", text)
}
if !strings.Contains(text, `"column_types":[]`) {
t.Fatalf("query result should marshal empty column types array: %s", text)
}
data, err = json.Marshal(indexInfo{})
if err != nil {
+14 -2
View File
@@ -4024,6 +4024,11 @@ function tableColumnForGridColumn(columnIndex: number): ColumnInfo | undefined {
return props.tableMeta?.columns.find((column) => column.name.toLowerCase() === columnName.toLowerCase());
}
function resultColumnInfoForGridColumn(columnIndex: number): Pick<ColumnInfo, "data_type"> | undefined {
const dataType = props.result.column_types?.[columnIndex]?.trim();
return dataType ? { data_type: dataType } : undefined;
}
function temporalEditorKindForColumn(columnIndex: number): TemporalCellEditorKind | undefined {
return temporalCellEditorKind(tableColumnForGridColumn(columnIndex)?.data_type, props.databaseType);
}
@@ -5351,7 +5356,8 @@ function formatCell(value: CellValue, columnIndex?: number): string {
const formatter = columnIndex === undefined ? undefined : resolvedColumnFormatters.value[columnIndex];
const columnName = columnIndex === undefined ? undefined : props.result.columns[columnIndex];
const columnInfo = columnIndex === undefined ? undefined : tableColumnForGridColumn(columnIndex);
const arrayDisplay = formatter ? undefined : dataGridCellDisplayText({ value, databaseType: props.databaseType, columnInfo });
const displayColumnInfo = columnInfo ?? (columnIndex === undefined ? undefined : resultColumnInfoForGridColumn(columnIndex));
const arrayDisplay = formatter ? undefined : dataGridCellDisplayText({ value, databaseType: props.databaseType, columnInfo: displayColumnInfo });
if (arrayDisplay !== undefined) return arrayDisplay;
const binaryDisplay = formatter ? null : binaryCellDisplayText(value, columnInfo?.data_type ?? (columnName ? columnTypeMap.value.get(columnName) : undefined));
if (binaryDisplay) return binaryDisplay;
@@ -5392,7 +5398,13 @@ function draftCellPlaceholder(item: RowItem | undefined, columnIndex: number): s
return item?.isDraft && item.data[columnIndex] === null ? t("grid.quickEntryDraftPlaceholder") : null;
}
watch(() => [props.result.columns.join("\u0000"), columnFormatterSignatures.value.join("\u0000")], clearCellFormatCache);
function columnTypeCacheSignature(): string {
const resultTypes = props.result.column_types?.join("\u0000") ?? "";
const tableTypes = props.tableMeta?.columns?.map((column) => `${column.name}:${column.data_type}`).join("\u0000") ?? "";
return `${resultTypes}\u0001${tableTypes}`;
}
watch(() => [props.result.columns.join("\u0000"), columnFormatterSignatures.value.join("\u0000"), columnTypeCacheSignature()], clearCellFormatCache);
function quoteIdent(name: string): string {
return quoteTableIdentifier(props.databaseType, name);
@@ -237,6 +237,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
tableName: tableMeta.value?.tableName ?? null,
copyInsertTargetLabel: copyInsertTargetLabel?.value ?? null,
columns: columns.value,
columnTypes: columnTypes.value ?? null,
sourceColumns: sourceColumns.value ?? null,
excludePrimaryKeys,
insertMode,
@@ -311,6 +312,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
databaseType: databaseType.value,
tableMeta: tableMeta.value,
columns: columns.value,
columnTypes: columnTypes.value,
sourceColumns: sourceColumns.value,
rows: rows.map((item) => item.data),
excludePrimaryKeys,
@@ -1075,9 +1077,10 @@ export function useDataGridExport(options: UseDataGridExportOptions) {
} {
const exportColumns = tableMeta.value ? effectiveColumns(sourceColumns.value, result.columns) : result.columns;
const columnIndexes = exportColumns.map((column, index) => ({ column, index })).filter((item): item is { column: string; index: number } => !!item.column);
const exportColumnTypes = columnTypes.value?.length === result.columns.length ? columnTypes.value : undefined;
return {
columns: columnIndexes.map((item) => item.column),
columnTypes: tableMeta.value ? columnIndexes.map((item) => columnTypes.value?.[item.index]) : undefined,
columnTypes: exportColumnTypes ? columnIndexes.map((item) => exportColumnTypes[item.index]) : undefined,
rows: result.rows.map((row) => columnIndexes.map((item) => row[item.index] ?? null)),
};
}
@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import { dataGridCellDisplayText } from "@/lib/dataGrid/dataGridCellCoercion";
describe("dataGridCellDisplayText", () => {
it("formats Oracle DATE values without RFC3339 separators", () => {
expect(
dataGridCellDisplayText({
value: "2022-08-25T09:58:43Z",
databaseType: "oracle",
columnInfo: { data_type: "DATE" },
}),
).toBe("2022-08-25 09:58:43");
});
it("formats midnight Oracle DATE values as a date", () => {
expect(
dataGridCellDisplayText({
value: "2022-08-25T00:00:00Z",
databaseType: "oracle",
columnInfo: { data_type: "DATE" },
}),
).toBe("2022-08-25");
});
it("does not format non-date Oracle strings", () => {
expect(
dataGridCellDisplayText({
value: "2022-08-25T09:58:43Z",
databaseType: "oracle",
columnInfo: { data_type: "VARCHAR2(64)" },
}),
).toBeUndefined();
});
});
@@ -45,6 +45,9 @@ export function dataGridCellDisplayText(options: { value: GridCellValue; databas
if (Array.isArray(options.value) && options.databaseType === "postgres" && isPostgresArrayColumn(options.columnInfo, options.value)) {
return formatPostgresArrayText(options.value);
}
if (typeof options.value === "string" && isOracleDateColumn(options.databaseType, options.columnInfo)) {
return formatOracleDateDisplayText(options.value);
}
return undefined;
}
@@ -113,6 +116,39 @@ function normalizeDataType(dataType: string | undefined): string {
return (dataType ?? "").trim().toLowerCase();
}
function isOracleDateColumn(databaseType: DatabaseType | undefined, columnInfo: Pick<ColumnInfo, "data_type"> | undefined): boolean {
if (databaseType !== "oracle" && databaseType !== "oceanbase-oracle") return false;
const base = normalizeDataType(columnInfo?.data_type).split(/[()\s\t\n]/)[0] ?? "";
return base === "date";
}
function formatOracleDateDisplayText(value: string): string | undefined {
const parts = parseOracleDateLikeText(value);
if (!parts) return undefined;
if (parts.time === "00:00:00" && !parts.fraction) return parts.date;
return `${parts.date} ${parts.time}${parts.fraction ?? ""}`;
}
function parseOracleDateLikeText(value: string): { date: string; time: string; fraction?: string } | undefined {
if (!/^\d{4}-\d{2}-\d{2}/.test(value)) return undefined;
const date = value.slice(0, 10);
if (value.length === 10) return { date, time: "00:00:00" };
const separator = value[10];
if (separator !== "T" && separator !== " ") return undefined;
if (!/^\d{2}:\d{2}:\d{2}/.test(value.slice(11))) return undefined;
const time = value.slice(11, 19);
let rest = value.slice(19);
let fraction: string | undefined;
if (rest.startsWith(".")) {
const match = rest.match(/^(\.\d{1,9})(.*)$/);
if (!match) return undefined;
fraction = match[1];
rest = match[2];
}
if (rest && !/^z$/i.test(rest) && !/^[+-]\d{2}:\d{2}$/.test(rest)) return undefined;
return { date, time, fraction };
}
function isExactDecimalDataType(dataType: string): boolean {
return /\b(?:decimal|numeric|number|dec|money|smallmoney|bigdecimal|bignumeric|big_numeric|fixed)\b/.test(dataType);
}
@@ -45,6 +45,7 @@ export interface DataGridCopyInsertStatementOptions {
databaseType?: DatabaseType;
tableMeta?: DataGridTableMeta;
columns: string[];
columnTypes?: Array<string | null | undefined>;
sourceColumns?: Array<string | undefined>;
rows: GridCellValue[][];
excludePrimaryKeys?: boolean;
@@ -156,6 +156,7 @@ describe("queryStore database open state", () => {
vi.unstubAllGlobals();
const storage = installLocalStorage();
storage.set("dbx-editor-settings", JSON.stringify({ openTabsRestoreMode: "none" }));
storage.set("dbx-app-state:open_tabs", JSON.stringify({ tabs: JSON.parse(persistedTabs), activeTabId: "tab-1" }));
storage.set(OPEN_TABS_STORAGE_KEY, persistedTabs);
storage.set(ACTIVE_TAB_STORAGE_KEY, "tab-1");
setActivePinia(createPinia());
@@ -170,6 +171,7 @@ describe("queryStore database open state", () => {
expect(store.activeTabId).toBeNull();
expect(storage.get(OPEN_TABS_STORAGE_KEY)).toBeUndefined();
expect(storage.get(ACTIVE_TAB_STORAGE_KEY)).toBeUndefined();
expect(JSON.parse(storage.get("dbx-app-state:open_tabs") ?? "{}")).toEqual({ tabs: [], activeTabId: null });
});
it("restores only pinned tabs when launch restore mode is pinned", async () => {
+6
View File
@@ -681,6 +681,12 @@ export const useQueryStore = defineStore("query", () => {
if (saved?.tabs && Array.isArray(saved.tabs)) {
const restored = restoreSavedTabsFromPayload(saved);
applyRestoredOpenTabs(restored);
if (useSettingsStore().editorSettings.openTabsRestoreMode === "none") {
// Restore is explicitly disabled, so stale saved payloads should not
// reappear if the user later changes the setting.
clearLegacySavedTabs();
await saveTabs(tabs.value, activeTabId.value).catch(() => undefined);
}
isOpenTabsLoaded.value = true;
return;
}
+7
View File
@@ -24,6 +24,13 @@ required-features = ["duckdb-bundled"]
test = false
bench = false
[[bin]]
name = "duckdb-worker-file-lock-owner"
path = "tests/support/duckdb_worker_file_lock_owner.rs"
required-features = ["duckdb-bundled"]
test = false
bench = false
[features]
default = ["duckdb-bundled", "mq-admin", "sqlite-sqlcipher"]
duckdb-bundled = ["duckdb/bundled"]
+98 -3
View File
@@ -93,6 +93,8 @@ pub struct DataGridCopyInsertStatementOptions {
pub table_meta: Option<DataGridTableMeta>,
pub columns: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub column_types: Option<Vec<Option<String>>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_columns: Option<Vec<Option<String>>>,
#[serde(default)]
pub rows: Vec<Vec<Value>>,
@@ -372,10 +374,19 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
insert_columns
.iter()
.map(|(column, index)| {
format_grid_sql_literal(
format_grid_copy_insert_sql_literal(
row.get(*index).unwrap_or(&Value::Null),
options.database_type,
column_info_for(column_info, column),
copy_column_info(
column_info,
column,
options
.column_types
.as_deref()
.and_then(|types| types.get(*index))
.and_then(|value| value.as_deref()),
)
.as_ref(),
)
})
.collect::<Vec<_>>()
@@ -1020,6 +1031,24 @@ fn effective_copy_columns(source_columns: Option<&[Option<String>]>, columns: &[
}
}
fn copy_column_info(
column_info: &[DataGridColumnInfo],
column: &str,
fallback_type: Option<&str>,
) -> Option<DataGridColumnInfo> {
if let Some(info) = column_info_for(column_info, column) {
return Some(info.clone());
}
fallback_type.map(|data_type| DataGridColumnInfo {
name: column.to_string(),
data_type: data_type.to_string(),
is_nullable: true,
is_primary_key: false,
column_default: None,
extra: None,
})
}
fn effective_column(options: &DataGridSaveStatementOptions, index: usize) -> Option<&str> {
match &options.source_columns {
Some(source_columns) if source_columns.len() == options.columns.len() => source_columns.get(index)?.as_deref(),
@@ -1046,6 +1075,23 @@ pub fn normalize_data_grid_save_error(database_type: Option<DatabaseType>, error
error.to_string()
}
fn format_grid_copy_insert_sql_literal(
value: &Value,
database_type: Option<DatabaseType>,
column_info: Option<&DataGridColumnInfo>,
) -> String {
if is_oracle_temporal_literal_database(database_type) {
if let Some(text) = value.as_str() {
if let Some(literal) =
format_oracle_temporal_literal(text, column_info.map(|column| column.data_type.as_str()))
{
return literal;
}
}
}
format_grid_sql_literal(value, database_type, column_info)
}
pub fn format_grid_sql_literal(
value: &Value,
database_type: Option<DatabaseType>,
@@ -1169,9 +1215,12 @@ fn is_oracle_temporal_literal_database(database_type: Option<DatabaseType>) -> b
fn format_oracle_temporal_literal(text: &str, data_type: Option<&str>) -> Option<String> {
let kind = oracle_temporal_column_kind(data_type?)?;
let parts = regex_like_oracle_temporal(text)?;
let fraction = parts.fraction.unwrap_or_default();
let fraction = parts.fraction.as_deref().unwrap_or_default();
let datetime = format!("{} {}{}", parts.date, parts.time, fraction);
match kind {
OracleTemporalKind::Date if oracle_temporal_parts_are_midnight(&parts) => {
Some(format!("DATE '{}'", parts.date))
}
OracleTemporalKind::Date => Some(format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time)),
OracleTemporalKind::Timestamp => {
let mask = oracle_timestamp_format_mask(datetime.contains('.'));
@@ -1189,6 +1238,15 @@ fn format_oracle_temporal_literal(text: &str, data_type: Option<&str>) -> Option
}
}
fn oracle_temporal_parts_are_midnight(parts: &Rfc3339Parts) -> bool {
parts.time == "00:00:00"
&& parts
.fraction
.as_deref()
.map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0'))
.unwrap_or(true)
}
fn oracle_temporal_column_kind(data_type: &str) -> Option<OracleTemporalKind> {
let lower = data_type.trim().to_ascii_lowercase();
let base = lower.split(['(', ' ']).next().unwrap_or("");
@@ -2103,6 +2161,7 @@ mod tests {
columns: None,
}),
columns: vec!["id".to_string(), "login_name".to_string(), "display_name".to_string()],
column_types: None,
source_columns: None,
rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]],
exclude_primary_keys: true,
@@ -2126,6 +2185,7 @@ mod tests {
columns: None,
}),
columns: vec!["id".to_string(), "login_name".to_string(), "display_name".to_string()],
column_types: None,
source_columns: None,
rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]],
exclude_primary_keys: false,
@@ -2151,6 +2211,7 @@ mod tests {
columns: None,
}),
columns: vec!["ID".to_string(), "NAME".to_string()],
column_types: None,
source_columns: None,
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]],
exclude_primary_keys: false,
@@ -2179,6 +2240,7 @@ mod tests {
database_type: Some(DatabaseType::Mysql),
table_meta: Some(table_meta.clone()),
columns: columns.clone(),
column_types: None,
source_columns: None,
rows: rows.clone(),
exclude_primary_keys: false,
@@ -2241,6 +2303,7 @@ mod tests {
]),
}),
columns: vec!["id".to_string(), "title".to_string(), "search_vector".to_string()],
column_types: None,
source_columns: None,
rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]],
exclude_primary_keys: false,
@@ -2253,6 +2316,29 @@ mod tests {
);
}
#[test]
fn oracle_copy_insert_uses_result_column_types_for_date_literals() {
let statement = build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions {
database_type: Some(DatabaseType::Oracle),
table_meta: None,
columns: vec!["ID".to_string(), "CREATED_ON".to_string(), "RAW_TEXT".to_string()],
column_types: Some(vec![
Some("NUMBER".to_string()),
Some("DATE".to_string()),
Some("VARCHAR2".to_string()),
]),
source_columns: None,
rows: vec![vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")]],
exclude_primary_keys: false,
insert_mode: DataGridCopyInsertMode::Merged,
});
assert_eq!(
statement.as_deref(),
Some("INSERT INTO table_name (\"ID\", \"CREATED_ON\", \"RAW_TEXT\") VALUES (1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z');")
);
}
#[test]
fn builds_filter_conditions() {
assert_eq!(
@@ -2601,6 +2687,14 @@ mod tests {
format_grid_sql_literal(&json!("2022-08-25 09:58:43"), Some(DatabaseType::Oracle), Some(&date)),
"TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS')"
);
assert_eq!(
format_grid_sql_literal(&json!("2022-08-25"), Some(DatabaseType::Oracle), Some(&date)),
"DATE '2022-08-25'"
);
assert_eq!(
format_grid_sql_literal(&json!("2022-08-25T00:00:00Z"), Some(DatabaseType::Oracle), Some(&date)),
"DATE '2022-08-25'"
);
}
#[test]
@@ -3032,6 +3126,7 @@ mod tests {
database_type: Some(DatabaseType::Doris),
table_meta: Some(table_meta.clone()),
columns: vec!["id".to_string(), "status".to_string()],
column_types: None,
source_columns: None,
rows: vec![vec![json!(2), json!("new")]],
exclude_primary_keys: false,
+114
View File
@@ -206,6 +206,9 @@ fn format_export_sql_literal_typed(
return format_ch_array_sql_literal(arr);
}
}
if let Some(literal) = format_oracle_export_date_literal(value, database_type, column_type) {
return literal;
}
if let Some(literal) = format_export_temporal_literal(value, database_type, column_type) {
return literal;
}
@@ -295,6 +298,43 @@ fn is_mysql_compatible_export_literal_target(database_type: Option<DatabaseType>
)
}
fn format_oracle_export_date_literal(
value: &Value,
database_type: Option<DatabaseType>,
column_type: Option<&str>,
) -> Option<String> {
if !matches!(database_type, Some(DatabaseType::Oracle | DatabaseType::OceanbaseOracle)) {
return None;
}
if export_temporal_column_kind(database_type, column_type?)? != ExportTemporalKind::DateTime {
return None;
}
let lower = column_type?.trim().trim_matches('"').to_ascii_lowercase();
let base = lower.split(['(', ' ', '\t', '\n']).next().unwrap_or("");
if base != "date" {
return None;
}
let parts = parse_export_date_parts(value.as_str()?)?;
Some(format_oracle_export_date_parts_literal(&parts))
}
fn format_oracle_export_date_parts_literal(parts: &ExportRfc3339Parts) -> String {
if export_temporal_parts_are_midnight(parts) {
format!("DATE '{}'", parts.date)
} else {
format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time)
}
}
fn export_temporal_parts_are_midnight(parts: &ExportRfc3339Parts) -> bool {
parts.time == "00:00:00"
&& parts
.fraction
.as_deref()
.map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0'))
.unwrap_or(true)
}
fn format_export_temporal_literal(
value: &Value,
database_type: Option<DatabaseType>,
@@ -363,6 +403,53 @@ struct ExportRfc3339Parts {
zone: String,
}
fn parse_export_date_parts(text: &str) -> Option<ExportRfc3339Parts> {
parse_export_rfc3339_parts(text).or_else(|| parse_export_local_temporal_parts(text))
}
fn parse_export_local_temporal_parts(text: &str) -> Option<ExportRfc3339Parts> {
let bytes = text.as_bytes();
if bytes.len() < 10 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') {
return None;
}
let date = &text[0..10];
if !date.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) {
return None;
}
if bytes.len() == 10 {
return Some(ExportRfc3339Parts {
date: date.to_string(),
time: "00:00:00".to_string(),
fraction: None,
zone: String::new(),
});
}
let separator = *bytes.get(10)?;
if separator != b'T' && separator != b' ' {
return None;
}
if bytes.len() < 19 || bytes.get(13) != Some(&b':') || bytes.get(16) != Some(&b':') {
return None;
}
let time = &text[11..19];
if !time.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 2 | 5) || byte.is_ascii_digit()) {
return None;
}
let rest = &text[19..];
let fraction = if let Some(rest) = rest.strip_prefix('.') {
let digit_count = rest.chars().take_while(|ch| ch.is_ascii_digit()).count();
if digit_count == 0 || digit_count > 9 || digit_count != rest.len() {
return None;
}
Some(format!(".{}", &rest[..digit_count]))
} else if rest.is_empty() {
None
} else {
return None;
};
Some(ExportRfc3339Parts { date: date.to_string(), time: time.to_string(), fraction, zone: String::new() })
}
fn parse_export_rfc3339_parts(text: &str) -> Option<ExportRfc3339Parts> {
let bytes = text.as_bytes();
if bytes.len() < 20 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') {
@@ -1635,6 +1722,33 @@ mod tests {
);
}
#[test]
fn oracle_date_columns_export_as_date_literals() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Oracle),
schema: Some("APP".to_string()),
table_name: Some("EVENTS".to_string()),
qualified_table_name: None,
columns: vec!["ID".to_string(), "CREATED_ON".to_string(), "RAW_TEXT".to_string()],
column_types: vec![Some("NUMBER".to_string()), Some("DATE".to_string()), Some("VARCHAR2(64)".to_string())],
column_extras: Vec::new(),
rows: vec![
vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")],
vec![json!(2), json!("2022-08-25T00:00:00Z"), json!("2022-08-25T00:00:00Z")],
],
batch_size: Some(100),
})
.unwrap();
assert_eq!(
statements,
vec![
"INSERT INTO \"APP\".\"EVENTS\" (\"ID\", \"CREATED_ON\", \"RAW_TEXT\") VALUES (1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z');",
"INSERT INTO \"APP\".\"EVENTS\" (\"ID\", \"CREATED_ON\", \"RAW_TEXT\") VALUES (2, DATE '2022-08-25', '2022-08-25T00:00:00Z');",
]
);
}
#[test]
fn mysql_bit_columns_export_without_quoted_string_values() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
@@ -547,7 +547,7 @@ fn is_transient_duckdb_file_lock_error(message: &str) -> bool {
|| lower.contains("file is already open");
let mentions_lock = lower.contains("file is already open")
|| lower.contains("being used by another process")
|| lower.contains("conflicting lock is held")
|| lower.contains("conflicting lock")
|| lower.contains("process cannot access the file")
|| lower.contains("sharing violation")
|| lower.contains("resource temporarily unavailable")
+104
View File
@@ -1014,6 +1014,9 @@ pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, colum
if let Some(numeric_literal) = format_mysql_numeric_string_literal(s, db_type, column_type) {
return numeric_literal;
}
if let Some(date_literal) = format_oracle_date_sql_literal(s, db_type, column_type) {
return date_literal;
}
let literal = format_literal_string(s, db_type, column_type);
let escaped = if is_postgres_family_target(db_type) {
@@ -1102,6 +1105,76 @@ fn format_postgres_binary_sql_literal(
Some(format!("decode('{hex}', 'hex')"))
}
fn format_oracle_date_sql_literal(value: &str, db_type: &DatabaseType, column_type: Option<&str>) -> Option<String> {
if !matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) {
return None;
}
if temporal_column_kind(column_type) != Some("date") {
return None;
}
let parts = oracle_export_date_parts(value)?;
Some(format_oracle_date_sql_literal_parts(&parts))
}
struct OracleExportDateParts<'a> {
date: &'a str,
time: &'a str,
fraction: Option<&'a str>,
}
fn format_oracle_date_sql_literal_parts(parts: &OracleExportDateParts<'_>) -> String {
if oracle_export_date_parts_are_midnight(parts) {
format!("DATE '{}'", parts.date)
} else {
format!("TO_DATE('{} {}', 'YYYY-MM-DD HH24:MI:SS')", parts.date, parts.time)
}
}
fn oracle_export_date_parts_are_midnight(parts: &OracleExportDateParts<'_>) -> bool {
parts.time == "00:00:00"
&& parts.fraction.map(|fraction| fraction.trim_start_matches('.').chars().all(|ch| ch == '0')).unwrap_or(true)
}
fn oracle_export_date_parts(value: &str) -> Option<OracleExportDateParts<'_>> {
let bytes = value.as_bytes();
if bytes.len() < 10 || bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') {
return None;
}
let date = &value[..10];
if !date.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) {
return None;
}
if bytes.len() == 10 {
return Some(OracleExportDateParts { date, time: "00:00:00", fraction: None });
}
let separator = *bytes.get(10)?;
if separator != b'T' && separator != b' ' {
return None;
}
if bytes.len() < 19 || bytes.get(13) != Some(&b':') || bytes.get(16) != Some(&b':') {
return None;
}
let time = &value[11..19];
if !time.as_bytes().iter().enumerate().all(|(index, byte)| matches!(index, 2 | 5) || byte.is_ascii_digit()) {
return None;
}
let rest = &value[19..];
if rest.is_empty() || is_timezone_suffix(rest) {
return Some(OracleExportDateParts { date, time, fraction: None });
}
if let Some(after_dot) = rest.strip_prefix('.') {
let digit_count = after_dot.bytes().take_while(|byte| byte.is_ascii_digit()).count();
if digit_count == 0 {
return None;
}
let zone = &after_dot[digit_count..];
if zone.is_empty() || is_timezone_suffix(zone) {
return Some(OracleExportDateParts { date, time, fraction: Some(&value[19..19 + 1 + digit_count]) });
}
}
None
}
pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String {
if arr.is_empty() {
return "'{}'".to_string();
@@ -5641,6 +5714,37 @@ mod tests {
);
}
#[test]
fn oracle_insert_uses_date_literals_for_date_columns() {
let sql = generate_insert_typed(
&[String::from("id"), String::from("created_on"), String::from("created_at"), String::from("raw_text")],
&[
Some(String::from("NUMBER")),
Some(String::from("DATE")),
Some(String::from("TIMESTAMP(6)")),
Some(String::from("VARCHAR2(64)")),
],
&[vec![
json!(1),
json!("2022-08-25T09:58:43Z"),
json!("2022-08-25T09:58:43Z"),
json!("2022-08-25T09:58:43Z"),
]],
"events",
"APP",
&DatabaseType::Oracle,
);
assert_eq!(
sql,
"INSERT INTO \"APP\".\"events\" (\"id\", \"created_on\", \"created_at\", \"raw_text\") VALUES\n(1, TO_DATE('2022-08-25 09:58:43', 'YYYY-MM-DD HH24:MI:SS'), '2022-08-25T09:58:43Z', '2022-08-25T09:58:43Z')"
);
assert_eq!(
escape_value_typed(&json!("2022-08-25T00:00:00Z"), &DatabaseType::Oracle, Some("DATE")),
"DATE '2022-08-25'"
);
}
#[test]
fn mysql_insert_formats_numeric_strings_from_numeric_columns_as_numeric_literals() {
let sql = generate_insert_typed(
+26 -38
View File
@@ -321,12 +321,11 @@ async fn worker_process_is_killed_after_connect_timeout() {
async fn worker_process_is_killed_after_connect_error() {
let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-pid-test-host"));
let db_path = temp_duckdb_path();
let missing_dir = temp_duckdb_path();
let db_path = missing_dir.join("missing.duckdb");
let pid_file = temp_pid_path();
let _ = std::fs::remove_file(&pid_file);
let _ = std::fs::remove_file(&db_path);
let _file_owner = duckdb::Connection::open(&db_path).expect("open lock owner connection");
let _ = std::fs::remove_dir_all(&missing_dir);
std::env::set_var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE", &pid_file);
let client = DuckDbWorkerClient::new_unconnected_with_timeouts(
@@ -341,15 +340,13 @@ async fn worker_process_is_killed_after_connect_error() {
let err = client
.execute(None, "SELECT 1".to_string(), Some(10), None, Some(Duration::from_secs(5)))
.await
.expect_err("connect should fail while another process owns the DuckDB file");
.expect_err("connect should fail with an invalid DuckDB path");
std::env::remove_var("DBX_DUCKDB_PID_TEST_HOST_PID_FILE");
assert!(
err.contains("Cannot open file")
|| err.contains("Could not set lock")
|| err.contains("Conflicting lock is held")
|| err.contains("already open")
|| err.contains("timed out")
|| err.contains("另一个程序正在使用"),
|| err.contains("No such file")
|| err.contains("Parent directory does not exist")
|| err.contains("不存在"),
"unexpected error: {err}"
);
@@ -357,45 +354,33 @@ async fn worker_process_is_killed_after_connect_error() {
wait_until_process_exits(pid, Duration::from_secs(5)).await;
let _ = std::fs::remove_file(&pid_file);
drop(_file_owner);
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_dir_all(&missing_dir);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_process_retries_connect_after_transient_file_lock() {
let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-test-host"));
let lock_owner_executable = PathBuf::from(env!("CARGO_BIN_EXE_duckdb-worker-file-lock-owner"));
let db_path = temp_duckdb_path();
let _ = std::fs::remove_file(&db_path);
let mut owner_child = Command::new(&executable)
.stdin(Stdio::piped())
let mut lock_owner = Command::new(lock_owner_executable)
.arg(&db_path)
.arg("500")
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.expect("spawn lock owner worker");
let mut owner_stdin = owner_child.stdin.take().expect("owner worker stdin");
let owner_stdout = owner_child.stdout.take().expect("owner worker stdout");
let mut owner_lines = BufReader::new(owner_stdout).lines();
write_worker_request(
&mut owner_stdin,
DuckDbWorkerRequest::new(
"connect-owner",
DuckDbWorkerMethod::Connect,
DuckDbWorkerConnectParams { path: db_path.to_string_lossy().to_string(), attached_databases: Vec::new() },
)
.expect("owner connect request"),
)
.await;
let connected = read_worker_response(&mut owner_lines).await;
assert!(connected.ok, "owner connect failed: {connected:?}");
drop(owner_lines);
let release_owner = tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(200)).await;
drop(owner_stdin);
let _ = tokio::time::timeout(Duration::from_secs(5), owner_child.wait()).await;
});
.expect("spawn lock owner");
let lock_owner_stdout = lock_owner.stdout.take().expect("lock owner stdout");
let mut lock_owner_lines = BufReader::new(lock_owner_stdout).lines();
let ready = tokio::time::timeout(Duration::from_secs(5), lock_owner_lines.next_line())
.await
.expect("lock owner ready timed out")
.expect("read lock owner ready line")
.expect("lock owner ready line");
assert_eq!(ready, "ready");
let client = DuckDbWorkerClient::new_unconnected_with_timeouts(
executable,
@@ -411,7 +396,10 @@ async fn worker_process_retries_connect_after_transient_file_lock() {
)
.await;
let _ = release_owner.await;
tokio::time::timeout(Duration::from_secs(5), lock_owner.wait())
.await
.expect("lock owner should exit")
.expect("wait for lock owner");
client.shutdown().await;
let _ = std::fs::remove_file(&db_path);
@@ -71,6 +71,7 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser
database_type: Some(DatabaseType::Postgres),
table_meta: Some(table_meta),
columns: result.columns.clone(),
column_types: None,
source_columns: None,
rows: result.rows.clone(),
exclude_primary_keys: false,
@@ -0,0 +1,16 @@
#![cfg(feature = "duckdb-bundled")]
use std::io::Write;
use std::path::PathBuf;
use std::time::Duration;
fn main() {
let mut args = std::env::args().skip(1);
let path = args.next().map(PathBuf::from).expect("DuckDB path argument");
let hold_ms = args.next().expect("hold duration argument").parse::<u64>().expect("hold duration ms");
let _connection = duckdb::Connection::open(&path).expect("open lock owner connection");
println!("ready");
std::io::stdout().flush().expect("flush ready line");
std::thread::sleep(Duration::from_millis(hold_ms));
}