diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index 2dc0038e21..811006c3bf 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -17,9 +17,10 @@ use crate::mysql_ddl_normalize::DdlNormalizeOptions; use crate::object_source_sql::build_export_object_source_sql; use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements}; use crate::transfer::{ - format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra, - is_mysql_generated_column_extra, keyset_pagination_sql_with_identifier_quote, quote_identifier, - quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table, + format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal, + is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_vector_type, + keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal, + wrap_dameng_identity_insert_sql_for_table, }; use crate::types::{ObjectSourceKind, SpatialColumn}; @@ -441,8 +442,8 @@ fn format_export_sql_literal_typed( if is_postgres_json_export_column(database_type, column_type) { return format_postgres_json_export_literal(value); } - if is_postgres_vector_export_column(database_type, column_type) { - return format_postgres_vector_export_literal(value); + if database_type == Some(DatabaseType::Postgres) && is_postgres_vector_type(column_type) { + return format_postgres_vector_sql_literal(value); } if matches!(database_type, Some(DatabaseType::Mysql)) && column_type.is_some_and(is_mysql_bit_type) { return format_mysql_bit_literal(value); @@ -509,35 +510,6 @@ fn format_postgres_json_export_literal(value: &Value) -> String { quote_postgres_string_literal(&text) } -fn format_postgres_vector_export_literal(value: &Value) -> String { - if value.is_null() { - return "NULL".to_string(); - } - let text = match value { - // pgvector vector/halfvec are scalar extension types whose importable - // literal grammar uses square brackets, unlike PostgreSQL arrays. - Value::Array(arr) => format_postgres_vector_export_text(arr), - Value::String(text) => text.to_string(), - _ => value.to_string(), - }; - quote_postgres_string_literal(&text) -} - -fn format_postgres_vector_export_text(arr: &[Value]) -> String { - let elements = arr.iter().map(format_postgres_vector_export_element).collect::>(); - format!("[{}]", elements.join(",")) -} - -fn format_postgres_vector_export_element(value: &Value) -> String { - match value { - Value::String(text) => text.trim().to_string(), - Value::Number(number) => number.to_string(), - Value::Bool(value) => value.to_string(), - Value::Null => "NULL".to_string(), - _ => value.to_string(), - } -} - fn quote_export_sql_string(text: &str) -> String { format!("'{}'", text.replace('\\', "\\\\").replace('\'', "''")) } @@ -1274,17 +1246,6 @@ fn is_postgres_bytea_export_column(database_type: Option, column_t .unwrap_or(false) } -fn is_postgres_vector_export_column(database_type: Option, column_type: Option<&str>) -> bool { - database_type == Some(DatabaseType::Postgres) - && column_type - .map(|column_type| { - let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase(); - let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"'); - matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec") - }) - .unwrap_or(false) -} - pub fn build_export_sql_insert(options: BuildExportSqlInsertOptions) -> Result { build_export_insert_statements(options.insert).map(|statements| statements.join("\n")) } @@ -4399,18 +4360,29 @@ mod tests { "id".to_string(), "embedding".to_string(), "qualified_embedding".to_string(), + "compact_embedding".to_string(), "labels".to_string(), + "embedding_history".to_string(), ], column_types: vec![ Some("integer".to_string()), Some("vector(2)".to_string()), Some("public.vector".to_string()), + Some("halfvec(2)".to_string()), Some("text[]".to_string()), + Some("public.vector(2)[]".to_string()), ], column_extras: Vec::new(), spatial_columns: Vec::new(), spatial_values: Vec::new(), - rows: vec![vec![json!(1), json!([1.2, 3.4]), json!(["5", "6"]), json!(["x", "y"])]], + rows: vec![vec![ + json!(1), + json!([1.2, 3.4]), + json!(["5", "6"]), + json!([-0.25, 4]), + json!(["x", "y"]), + json!([[1.2, 3.4], [5, 6]]), + ]], batch_size: Some(10), }) .unwrap(); @@ -4418,7 +4390,7 @@ mod tests { assert_eq!( statements, vec![ - r#"INSERT INTO "public"."items" ("id", "embedding", "qualified_embedding", "labels") VALUES (1, '[1.2,3.4]', '[5,6]', '{"x","y"}');"# + r#"INSERT INTO "public"."items" ("id", "embedding", "qualified_embedding", "compact_embedding", "labels", "embedding_history") VALUES (1, '[1.2,3.4]', '[5,6]', '[-0.25,4]', '{"x","y"}', '{{1.2,3.4},{5,6}}');"# ] ); } diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index d225073283..3334565730 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -2613,10 +2613,15 @@ pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, colum _ => format!("'{escaped}'"), } } - serde_json::Value::Array(arr) => match db_type { - DatabaseType::ClickHouse | DatabaseType::Databend => format_ch_array_sql_literal(arr), - _ => format_pg_array_sql_literal(arr), - }, + serde_json::Value::Array(arr) => { + if *db_type == DatabaseType::Postgres && is_postgres_vector_type(column_type) { + return format_postgres_vector_sql_literal(val); + } + match db_type { + DatabaseType::ClickHouse | DatabaseType::Databend => format_ch_array_sql_literal(arr), + _ => format_pg_array_sql_literal(arr), + } + } _ => { let s = val.to_string(); format!("'{}'", s.replace('\\', "\\\\").replace('\'', "''")) @@ -2830,6 +2835,46 @@ pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String { format!("'{}'", inner.replace('\\', "\\\\").replace('\'', "''")) } +pub(crate) fn is_postgres_vector_type(column_type: Option<&str>) -> bool { + column_type + .map(|column_type| { + let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase(); + if normalized.trim_end().ends_with("[]") { + return false; + } + let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"'); + matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec") + }) + .unwrap_or(false) +} + +pub(crate) fn format_postgres_vector_sql_literal(value: &serde_json::Value) -> String { + if value.is_null() { + return "NULL".to_string(); + } + let text = match value { + // pgvector vector/halfvec are scalar extension types whose importable + // literal grammar uses square brackets, unlike PostgreSQL arrays. + serde_json::Value::Array(arr) => { + let elements = arr.iter().map(format_postgres_vector_element).collect::>(); + format!("[{}]", elements.join(",")) + } + serde_json::Value::String(text) => text.to_string(), + _ => value.to_string(), + }; + quote_postgres_string_literal(&text) +} + +fn format_postgres_vector_element(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(text) => text.trim().to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::Bool(value) => value.to_string(), + serde_json::Value::Null => "NULL".to_string(), + _ => value.to_string(), + } +} + fn format_pg_array_element(val: &serde_json::Value) -> String { match val { serde_json::Value::Null => "NULL".to_string(), @@ -12176,6 +12221,50 @@ mod tests { ); } + #[test] + fn postgres_insert_preserves_pgvector_literals_and_array_controls() { + let sql = generate_insert_typed( + &[ + String::from("embedding"), + String::from("qualified_embedding"), + String::from("compact_embedding"), + String::from("scores"), + String::from("embedding_history"), + ], + &[ + Some(String::from("vector(3)")), + Some(String::from("public.vector(3)")), + Some(String::from("extensions.halfvec(3)")), + Some(String::from("real[]")), + Some(String::from("vector(3)[]")), + ], + &[vec![ + json!([1.25, -2.5, 3.75]), + json!([0, 0.875, -0.014]), + json!(["0.5", "-0.25", "4"]), + json!([1.25, -2.5, 3.75]), + json!([[1.25, -2.5, 3.75], [0, 0.875, -0.014]]), + ]], + "pgvector_probe", + "target_7955", + &DatabaseType::Postgres, + None, + ); + + assert_eq!( + sql, + r#"INSERT INTO "target_7955"."pgvector_probe" ("embedding", "qualified_embedding", "compact_embedding", "scores", "embedding_history") VALUES +('[1.25,-2.5,3.75]', '[0,0.875,-0.014]', '[0.5,-0.25,4]', '{1.25,-2.5,3.75}', '{{1.25,-2.5,3.75},{0,0.875,-0.014}}')"# + ); + } + + #[test] + fn postgres_vector_literal_keeps_null_and_preformatted_string_behavior() { + assert_eq!(escape_value_typed(&serde_json::Value::Null, &DatabaseType::Postgres, Some("vector(3)")), "NULL"); + assert_eq!(escape_value_typed(&json!("[1,2,3]"), &DatabaseType::Postgres, Some("halfvec(3)")), "'[1,2,3]'"); + assert_eq!(escape_value_typed(&json!([1, 2, 3]), &DatabaseType::Postgres, Some("integer[]")), "'{1,2,3}'"); + } + #[test] fn postgres_insert_formats_bytea_prefixed_hex_as_binary_literal() { let sql = generate_insert_typed(