fix(transfer): preserve PostgreSQL pgvector literals
This commit is contained in:
1 parent
c28248cd67
commit
767cc809a1
2 files changed
+112
-51
No files matched your search
@@ -17,9 +17,10 @@ use crate::mysql_ddl_normalize::DdlNormalizeOptions;
|
|||||||
use crate::object_source_sql::build_export_object_source_sql;
|
use crate::object_source_sql::build_export_object_source_sql;
|
||||||
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
|
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
|
||||||
use crate::transfer::{
|
use crate::transfer::{
|
||||||
format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra,
|
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
|
||||||
is_mysql_generated_column_extra, keyset_pagination_sql_with_identifier_quote, quote_identifier,
|
is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_vector_type,
|
||||||
quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table,
|
keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal,
|
||||||
|
wrap_dameng_identity_insert_sql_for_table,
|
||||||
};
|
};
|
||||||
use crate::types::{ObjectSourceKind, SpatialColumn};
|
use crate::types::{ObjectSourceKind, SpatialColumn};
|
||||||
|
|
||||||
@@ -441,8 +442,8 @@ fn format_export_sql_literal_typed(
|
|||||||
if is_postgres_json_export_column(database_type, column_type) {
|
if is_postgres_json_export_column(database_type, column_type) {
|
||||||
return format_postgres_json_export_literal(value);
|
return format_postgres_json_export_literal(value);
|
||||||
}
|
}
|
||||||
if is_postgres_vector_export_column(database_type, column_type) {
|
if database_type == Some(DatabaseType::Postgres) && is_postgres_vector_type(column_type) {
|
||||||
return format_postgres_vector_export_literal(value);
|
return format_postgres_vector_sql_literal(value);
|
||||||
}
|
}
|
||||||
if matches!(database_type, Some(DatabaseType::Mysql)) && column_type.is_some_and(is_mysql_bit_type) {
|
if matches!(database_type, Some(DatabaseType::Mysql)) && column_type.is_some_and(is_mysql_bit_type) {
|
||||||
return format_mysql_bit_literal(value);
|
return format_mysql_bit_literal(value);
|
||||||
@@ -509,35 +510,6 @@ fn format_postgres_json_export_literal(value: &Value) -> String {
|
|||||||
quote_postgres_string_literal(&text)
|
quote_postgres_string_literal(&text)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_postgres_vector_export_literal(value: &Value) -> String {
|
|
||||||
if value.is_null() {
|
|
||||||
return "NULL".to_string();
|
|
||||||
}
|
|
||||||
let text = match value {
|
|
||||||
// pgvector vector/halfvec are scalar extension types whose importable
|
|
||||||
// literal grammar uses square brackets, unlike PostgreSQL arrays.
|
|
||||||
Value::Array(arr) => format_postgres_vector_export_text(arr),
|
|
||||||
Value::String(text) => text.to_string(),
|
|
||||||
_ => value.to_string(),
|
|
||||||
};
|
|
||||||
quote_postgres_string_literal(&text)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn format_postgres_vector_export_text(arr: &[Value]) -> String {
|
|
||||||
let elements = arr.iter().map(format_postgres_vector_export_element).collect::<Vec<_>>();
|
|
||||||
format!("[{}]", elements.join(","))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn format_postgres_vector_export_element(value: &Value) -> String {
|
|
||||||
match value {
|
|
||||||
Value::String(text) => text.trim().to_string(),
|
|
||||||
Value::Number(number) => number.to_string(),
|
|
||||||
Value::Bool(value) => value.to_string(),
|
|
||||||
Value::Null => "NULL".to_string(),
|
|
||||||
_ => value.to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn quote_export_sql_string(text: &str) -> String {
|
fn quote_export_sql_string(text: &str) -> String {
|
||||||
format!("'{}'", text.replace('\\', "\\\\").replace('\'', "''"))
|
format!("'{}'", text.replace('\\', "\\\\").replace('\'', "''"))
|
||||||
}
|
}
|
||||||
@@ -1274,17 +1246,6 @@ fn is_postgres_bytea_export_column(database_type: Option<DatabaseType>, column_t
|
|||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_postgres_vector_export_column(database_type: Option<DatabaseType>, column_type: Option<&str>) -> bool {
|
|
||||||
database_type == Some(DatabaseType::Postgres)
|
|
||||||
&& column_type
|
|
||||||
.map(|column_type| {
|
|
||||||
let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase();
|
|
||||||
let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"');
|
|
||||||
matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec")
|
|
||||||
})
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn build_export_sql_insert(options: BuildExportSqlInsertOptions) -> Result<String, String> {
|
pub fn build_export_sql_insert(options: BuildExportSqlInsertOptions) -> Result<String, String> {
|
||||||
build_export_insert_statements(options.insert).map(|statements| statements.join("\n"))
|
build_export_insert_statements(options.insert).map(|statements| statements.join("\n"))
|
||||||
}
|
}
|
||||||
@@ -4399,18 +4360,29 @@ mod tests {
|
|||||||
"id".to_string(),
|
"id".to_string(),
|
||||||
"embedding".to_string(),
|
"embedding".to_string(),
|
||||||
"qualified_embedding".to_string(),
|
"qualified_embedding".to_string(),
|
||||||
|
"compact_embedding".to_string(),
|
||||||
"labels".to_string(),
|
"labels".to_string(),
|
||||||
|
"embedding_history".to_string(),
|
||||||
],
|
],
|
||||||
column_types: vec![
|
column_types: vec![
|
||||||
Some("integer".to_string()),
|
Some("integer".to_string()),
|
||||||
Some("vector(2)".to_string()),
|
Some("vector(2)".to_string()),
|
||||||
Some("public.vector".to_string()),
|
Some("public.vector".to_string()),
|
||||||
|
Some("halfvec(2)".to_string()),
|
||||||
Some("text[]".to_string()),
|
Some("text[]".to_string()),
|
||||||
|
Some("public.vector(2)[]".to_string()),
|
||||||
],
|
],
|
||||||
column_extras: Vec::new(),
|
column_extras: Vec::new(),
|
||||||
spatial_columns: Vec::new(),
|
spatial_columns: Vec::new(),
|
||||||
spatial_values: Vec::new(),
|
spatial_values: Vec::new(),
|
||||||
rows: vec![vec![json!(1), json!([1.2, 3.4]), json!(["5", "6"]), json!(["x", "y"])]],
|
rows: vec![vec![
|
||||||
|
json!(1),
|
||||||
|
json!([1.2, 3.4]),
|
||||||
|
json!(["5", "6"]),
|
||||||
|
json!([-0.25, 4]),
|
||||||
|
json!(["x", "y"]),
|
||||||
|
json!([[1.2, 3.4], [5, 6]]),
|
||||||
|
]],
|
||||||
batch_size: Some(10),
|
batch_size: Some(10),
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -4418,7 +4390,7 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
statements,
|
statements,
|
||||||
vec![
|
vec![
|
||||||
r#"INSERT INTO "public"."items" ("id", "embedding", "qualified_embedding", "labels") VALUES (1, '[1.2,3.4]', '[5,6]', '{"x","y"}');"#
|
r#"INSERT INTO "public"."items" ("id", "embedding", "qualified_embedding", "compact_embedding", "labels", "embedding_history") VALUES (1, '[1.2,3.4]', '[5,6]', '[-0.25,4]', '{"x","y"}', '{{1.2,3.4},{5,6}}');"#
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2613,10 +2613,15 @@ pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, colum
|
|||||||
_ => format!("'{escaped}'"),
|
_ => format!("'{escaped}'"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
serde_json::Value::Array(arr) => match db_type {
|
serde_json::Value::Array(arr) => {
|
||||||
DatabaseType::ClickHouse | DatabaseType::Databend => format_ch_array_sql_literal(arr),
|
if *db_type == DatabaseType::Postgres && is_postgres_vector_type(column_type) {
|
||||||
_ => format_pg_array_sql_literal(arr),
|
return format_postgres_vector_sql_literal(val);
|
||||||
},
|
}
|
||||||
|
match db_type {
|
||||||
|
DatabaseType::ClickHouse | DatabaseType::Databend => format_ch_array_sql_literal(arr),
|
||||||
|
_ => format_pg_array_sql_literal(arr),
|
||||||
|
}
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
let s = val.to_string();
|
let s = val.to_string();
|
||||||
format!("'{}'", s.replace('\\', "\\\\").replace('\'', "''"))
|
format!("'{}'", s.replace('\\', "\\\\").replace('\'', "''"))
|
||||||
@@ -2830,6 +2835,46 @@ pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String {
|
|||||||
format!("'{}'", inner.replace('\\', "\\\\").replace('\'', "''"))
|
format!("'{}'", inner.replace('\\', "\\\\").replace('\'', "''"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_postgres_vector_type(column_type: Option<&str>) -> bool {
|
||||||
|
column_type
|
||||||
|
.map(|column_type| {
|
||||||
|
let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase();
|
||||||
|
if normalized.trim_end().ends_with("[]") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"');
|
||||||
|
matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec")
|
||||||
|
})
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn format_postgres_vector_sql_literal(value: &serde_json::Value) -> String {
|
||||||
|
if value.is_null() {
|
||||||
|
return "NULL".to_string();
|
||||||
|
}
|
||||||
|
let text = match value {
|
||||||
|
// pgvector vector/halfvec are scalar extension types whose importable
|
||||||
|
// literal grammar uses square brackets, unlike PostgreSQL arrays.
|
||||||
|
serde_json::Value::Array(arr) => {
|
||||||
|
let elements = arr.iter().map(format_postgres_vector_element).collect::<Vec<_>>();
|
||||||
|
format!("[{}]", elements.join(","))
|
||||||
|
}
|
||||||
|
serde_json::Value::String(text) => text.to_string(),
|
||||||
|
_ => value.to_string(),
|
||||||
|
};
|
||||||
|
quote_postgres_string_literal(&text)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_postgres_vector_element(value: &serde_json::Value) -> String {
|
||||||
|
match value {
|
||||||
|
serde_json::Value::String(text) => text.trim().to_string(),
|
||||||
|
serde_json::Value::Number(number) => number.to_string(),
|
||||||
|
serde_json::Value::Bool(value) => value.to_string(),
|
||||||
|
serde_json::Value::Null => "NULL".to_string(),
|
||||||
|
_ => value.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn format_pg_array_element(val: &serde_json::Value) -> String {
|
fn format_pg_array_element(val: &serde_json::Value) -> String {
|
||||||
match val {
|
match val {
|
||||||
serde_json::Value::Null => "NULL".to_string(),
|
serde_json::Value::Null => "NULL".to_string(),
|
||||||
@@ -12176,6 +12221,50 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn postgres_insert_preserves_pgvector_literals_and_array_controls() {
|
||||||
|
let sql = generate_insert_typed(
|
||||||
|
&[
|
||||||
|
String::from("embedding"),
|
||||||
|
String::from("qualified_embedding"),
|
||||||
|
String::from("compact_embedding"),
|
||||||
|
String::from("scores"),
|
||||||
|
String::from("embedding_history"),
|
||||||
|
],
|
||||||
|
&[
|
||||||
|
Some(String::from("vector(3)")),
|
||||||
|
Some(String::from("public.vector(3)")),
|
||||||
|
Some(String::from("extensions.halfvec(3)")),
|
||||||
|
Some(String::from("real[]")),
|
||||||
|
Some(String::from("vector(3)[]")),
|
||||||
|
],
|
||||||
|
&[vec![
|
||||||
|
json!([1.25, -2.5, 3.75]),
|
||||||
|
json!([0, 0.875, -0.014]),
|
||||||
|
json!(["0.5", "-0.25", "4"]),
|
||||||
|
json!([1.25, -2.5, 3.75]),
|
||||||
|
json!([[1.25, -2.5, 3.75], [0, 0.875, -0.014]]),
|
||||||
|
]],
|
||||||
|
"pgvector_probe",
|
||||||
|
"target_7955",
|
||||||
|
&DatabaseType::Postgres,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sql,
|
||||||
|
r#"INSERT INTO "target_7955"."pgvector_probe" ("embedding", "qualified_embedding", "compact_embedding", "scores", "embedding_history") VALUES
|
||||||
|
('[1.25,-2.5,3.75]', '[0,0.875,-0.014]', '[0.5,-0.25,4]', '{1.25,-2.5,3.75}', '{{1.25,-2.5,3.75},{0,0.875,-0.014}}')"#
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn postgres_vector_literal_keeps_null_and_preformatted_string_behavior() {
|
||||||
|
assert_eq!(escape_value_typed(&serde_json::Value::Null, &DatabaseType::Postgres, Some("vector(3)")), "NULL");
|
||||||
|
assert_eq!(escape_value_typed(&json!("[1,2,3]"), &DatabaseType::Postgres, Some("halfvec(3)")), "'[1,2,3]'");
|
||||||
|
assert_eq!(escape_value_typed(&json!([1, 2, 3]), &DatabaseType::Postgres, Some("integer[]")), "'{1,2,3}'");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn postgres_insert_formats_bytea_prefixed_hex_as_binary_literal() {
|
fn postgres_insert_formats_bytea_prefixed_hex_as_binary_literal() {
|
||||||
let sql = generate_insert_typed(
|
let sql = generate_insert_typed(
|
||||||
|
|||||||
Reference in new issue
Block a user