fix(mysql): correct geometry export lon/lat order via WKB round-trip
This commit is contained in:
1 parent
00bbfad055
commit
c835deb3f9
5 files changed
+553
-90
No files matched your search
@@ -380,6 +380,9 @@ fn format_export_sql_literal_typed(
|
||||
if matches!(database_type, Some(DatabaseType::Mysql)) && column_type.is_some_and(is_mysql_bit_type) {
|
||||
return format_mysql_bit_literal(value);
|
||||
}
|
||||
if let Some(literal) = format_mysql_spatial_export_literal(value, database_type, column_type) {
|
||||
return literal;
|
||||
}
|
||||
if is_mysql_compatible_export_literal_target(database_type) {
|
||||
if column_type.is_some_and(is_mysql_binary_export_type) {
|
||||
if let Some(literal) = format_mysql_binary_export_literal(value) {
|
||||
@@ -769,6 +772,72 @@ fn is_mysql_binary_export_type(column_type: &str) -> bool {
|
||||
matches!(base, "binary" | "varbinary" | "blob" | "tinyblob" | "mediumblob" | "longblob")
|
||||
}
|
||||
|
||||
pub(crate) fn is_mysql_spatial_export_type(column_type: &str) -> bool {
|
||||
let base = column_type
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.split(['(', ':', ' ', '\t', '\n'])
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
matches!(
|
||||
base.as_str(),
|
||||
"geometry"
|
||||
| "point"
|
||||
| "linestring"
|
||||
| "polygon"
|
||||
| "multipoint"
|
||||
| "multilinestring"
|
||||
| "multipolygon"
|
||||
| "geometrycollection"
|
||||
| "geomcollection"
|
||||
)
|
||||
}
|
||||
|
||||
/// Database exports encode MySQL spatial cells as `DBX_WKB:<srid>:<hex>` while
|
||||
/// reading them. Keeping this marker internal lets the normal JSON row shape
|
||||
/// and all non-export query paths continue to expose readable WKT values.
|
||||
fn format_mysql_spatial_export_literal(
|
||||
value: &Value,
|
||||
database_type: Option<DatabaseType>,
|
||||
column_type: Option<&str>,
|
||||
) -> Option<String> {
|
||||
if database_type != Some(DatabaseType::Mysql) || !column_type.is_some_and(is_mysql_spatial_export_type) {
|
||||
return None;
|
||||
}
|
||||
let Value::String(value) = value else {
|
||||
return value.is_null().then(|| "NULL".to_string());
|
||||
};
|
||||
let marker = value.strip_prefix("DBX_WKB:")?;
|
||||
let (srid, hex) = marker.split_once(':')?;
|
||||
if srid.is_empty()
|
||||
|| !srid.as_bytes().iter().all(u8::is_ascii_digit)
|
||||
|| hex.is_empty()
|
||||
|| hex.len() % 2 != 0
|
||||
|| !hex.as_bytes().iter().all(u8::is_ascii_hexdigit)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let wkb = decode_mysql_spatial_export_wkb(hex)?;
|
||||
crate::db::wkb::decode_wkb_geometry(&wkb)?;
|
||||
let srid = srid.parse::<u32>().ok()?;
|
||||
Some(if srid == 0 { format!("ST_GeomFromWKB(0x{hex})") } else { format!("ST_GeomFromWKB(0x{hex}, {srid})") })
|
||||
}
|
||||
|
||||
fn decode_mysql_spatial_export_wkb(hex: &str) -> Option<Vec<u8>> {
|
||||
fn nibble(byte: u8) -> Option<u8> {
|
||||
match byte {
|
||||
b'0'..=b'9' => Some(byte - b'0'),
|
||||
b'a'..=b'f' => Some(byte - b'a' + 10),
|
||||
b'A'..=b'F' => Some(byte - b'A' + 10),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
hex.as_bytes().chunks_exact(2).map(|pair| Some((nibble(pair[0])? << 4) | nibble(pair[1])?)).collect()
|
||||
}
|
||||
|
||||
fn format_mysql_binary_export_literal(value: &Value) -> Option<String> {
|
||||
match value {
|
||||
Value::Null => Some("NULL".to_string()),
|
||||
@@ -1376,8 +1445,59 @@ fn record_export_error<W: Write>(file: &mut W, fail_on_error: bool, message: Str
|
||||
}
|
||||
}
|
||||
|
||||
fn database_export_select_sql(columns: &[String], table: &str, schema: &str, db_type: &DatabaseType) -> String {
|
||||
let columns = columns.iter().map(|column| quote_identifier(column, db_type)).collect::<Vec<_>>().join(", ");
|
||||
fn mysql_spatial_export_marker_expression(column: &str) -> String {
|
||||
let quoted = quote_identifier(column, &DatabaseType::Mysql);
|
||||
format!(
|
||||
"CASE WHEN {quoted} IS NULL THEN NULL ELSE CONCAT('DBX_WKB:', ST_SRID({quoted}), ':', HEX(ST_AsWKB({quoted}))) END AS {quoted}"
|
||||
)
|
||||
}
|
||||
|
||||
fn database_export_select_list(columns: &[String], column_types: &[Option<String>], db_type: &DatabaseType) -> String {
|
||||
columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, column)| {
|
||||
if *db_type == DatabaseType::Mysql
|
||||
&& column_types.get(index).and_then(|value| value.as_deref()).is_some_and(is_mysql_spatial_export_type)
|
||||
{
|
||||
mysql_spatial_export_marker_expression(column)
|
||||
} else {
|
||||
quote_identifier(column, db_type)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
fn replace_database_export_select_list(
|
||||
sql: String,
|
||||
columns: &[String],
|
||||
column_types: &[Option<String>],
|
||||
db_type: &DatabaseType,
|
||||
) -> String {
|
||||
let original = columns.iter().map(|column| quote_identifier(column, db_type)).collect::<Vec<_>>().join(", ");
|
||||
let replacement = database_export_select_list(columns, column_types, db_type);
|
||||
if replacement == original {
|
||||
return sql;
|
||||
}
|
||||
let prefix = format!("SELECT {original}");
|
||||
if !sql.starts_with(&prefix) {
|
||||
log::warn!(
|
||||
"MySQL spatial database export could not replace its SELECT list; geometry columns will be exported as WKT"
|
||||
);
|
||||
return sql;
|
||||
}
|
||||
format!("SELECT {replacement}{}", &sql[prefix.len()..])
|
||||
}
|
||||
|
||||
fn database_export_select_sql(
|
||||
columns: &[String],
|
||||
column_types: &[Option<String>],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
) -> String {
|
||||
let columns = database_export_select_list(columns, column_types, db_type);
|
||||
let table = crate::transfer::qualified_table(table, schema, db_type, None);
|
||||
format!("SELECT {columns} FROM {table}")
|
||||
}
|
||||
@@ -1966,7 +2086,7 @@ pub async fn export_database_sql_core(
|
||||
|
||||
if !col_names.is_empty() {
|
||||
if let Some(snapshot_session_id) = request.snapshot_session_id.as_deref() {
|
||||
let sql = database_export_select_sql(&col_names, table_name, &request.schema, &db_type);
|
||||
let sql = database_export_select_sql(&col_names, &col_types, table_name, &request.schema, &db_type);
|
||||
crate::query::stream_rows_in_manual_transaction(
|
||||
state,
|
||||
snapshot_session_id,
|
||||
@@ -2037,7 +2157,7 @@ pub async fn export_database_sql_core(
|
||||
}
|
||||
|
||||
let sql = if use_keyset {
|
||||
keyset_pagination_sql_with_identifier_quote(
|
||||
let sql = keyset_pagination_sql_with_identifier_quote(
|
||||
&col_names,
|
||||
table_name,
|
||||
&request.schema,
|
||||
@@ -2046,16 +2166,18 @@ pub async fn export_database_sql_core(
|
||||
&last_primary_key_values,
|
||||
batch_size,
|
||||
None,
|
||||
)
|
||||
);
|
||||
replace_database_export_select_list(sql, &col_names, &col_types, &db_type)
|
||||
} else {
|
||||
crate::transfer::pagination_sql(
|
||||
let sql = crate::transfer::pagination_sql(
|
||||
&col_names,
|
||||
table_name,
|
||||
&request.schema,
|
||||
&db_type,
|
||||
offset,
|
||||
batch_size,
|
||||
)
|
||||
);
|
||||
replace_database_export_select_list(sql, &col_names, &col_types, &db_type)
|
||||
};
|
||||
let result = match crate::transfer::execute_read_on_pool(state, &pool_key, &sql).await {
|
||||
Ok(result) => result,
|
||||
@@ -2359,15 +2481,16 @@ fn build_database_export_object_source_sql(
|
||||
mod tests {
|
||||
use super::{
|
||||
build_database_export_object_source_sql, build_database_sql_export, build_export_insert_statements,
|
||||
database_export_total_objects, drop_table_if_exists_sql, filter_export_table_infos, format_export_sql_literal,
|
||||
format_export_table_ddl, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl,
|
||||
generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql,
|
||||
is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows,
|
||||
mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_error,
|
||||
sort_export_views_by_dependencies, write_database_export_rows, BuildDatabaseSqlExportOptions,
|
||||
BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions,
|
||||
ExportedTableSql, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers,
|
||||
DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
|
||||
database_export_select_sql, database_export_total_objects, drop_table_if_exists_sql, filter_export_table_infos,
|
||||
format_export_sql_literal, format_export_table_ddl, format_mysql_spatial_export_literal,
|
||||
generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl,
|
||||
generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble,
|
||||
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl,
|
||||
record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies,
|
||||
write_database_export_rows, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions,
|
||||
DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions, ExportedTableSql,
|
||||
PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE,
|
||||
DATABASE_EXPORT_ROW_LIMIT,
|
||||
};
|
||||
use super::{concurrent_metadata_prefetch_allowed, database_export_metadata_prefetch_concurrency};
|
||||
use crate::models::connection::DatabaseType;
|
||||
@@ -2670,6 +2793,83 @@ mod tests {
|
||||
assert_eq!(format_export_sql_literal(&json!("O'Hara")), "'O''Hara'");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_spatial_export_uses_wkb_constructor_and_preserves_srid() {
|
||||
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
|
||||
database_type: Some(DatabaseType::Mysql),
|
||||
schema: None,
|
||||
table_name: Some("places".to_string()),
|
||||
qualified_table_name: None,
|
||||
columns: vec!["location".to_string(), "shape".to_string()],
|
||||
column_types: vec![Some("point".to_string()), Some("geometry".to_string())],
|
||||
column_extras: Vec::new(),
|
||||
rows: vec![vec![
|
||||
json!("DBX_WKB:4326:0101000000AE47E17A14AE5C4052B81E85EBF34240"),
|
||||
json!("DBX_WKB:0:0101000000000000000000F03F0000000000000040"),
|
||||
]],
|
||||
batch_size: Some(10),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
statements,
|
||||
vec!["INSERT INTO `places` (`location`, `shape`) VALUES (ST_GeomFromWKB(0x0101000000AE47E17A14AE5C4052B81E85EBF34240, 4326), ST_GeomFromWKB(0x0101000000000000000000F03F0000000000000040));"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_spatial_export_rejects_markers_for_unknown_or_nonspatial_types() {
|
||||
let marker = json!("DBX_WKB:4326:0101000000AE47E17A14AE5C4052B81E85EBF34240");
|
||||
assert!(format_mysql_spatial_export_literal(&marker, Some(DatabaseType::Mysql), None).is_none());
|
||||
assert!(format_mysql_spatial_export_literal(&marker, Some(DatabaseType::Mysql), Some("varchar")).is_none());
|
||||
assert!(format_mysql_spatial_export_literal(
|
||||
&json!("DBX_WKB:4326:0101000000"),
|
||||
Some(DatabaseType::Mysql),
|
||||
Some("geometry"),
|
||||
)
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_spatial_export_select_normalizes_geometry_columns_to_wkb_markers() {
|
||||
let sql = database_export_select_sql(
|
||||
&["id".to_string(), "location".to_string(), "name".to_string()],
|
||||
&[Some("int".to_string()), Some("point".to_string()), Some("varchar(32)".to_string())],
|
||||
"places",
|
||||
"app",
|
||||
&DatabaseType::Mysql,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
sql,
|
||||
"SELECT `id`, CASE WHEN `location` IS NULL THEN NULL ELSE CONCAT('DBX_WKB:', ST_SRID(`location`), ':', HEX(ST_AsWKB(`location`))) END AS `location`, `name` FROM `places`"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_spatial_export_keeps_keyset_pagination_when_replacing_select_list() {
|
||||
let columns = vec!["id".to_string(), "location".to_string()];
|
||||
let column_types = vec![Some("bigint".to_string()), Some("geometry".to_string())];
|
||||
let sql = crate::transfer::keyset_pagination_sql_with_identifier_quote(
|
||||
&columns,
|
||||
"places",
|
||||
"app",
|
||||
&DatabaseType::Mysql,
|
||||
&["id".to_string()],
|
||||
&[json!(7)],
|
||||
1000,
|
||||
None,
|
||||
);
|
||||
|
||||
let sql = replace_database_export_select_list(sql, &columns, &column_types, &DatabaseType::Mysql);
|
||||
|
||||
assert!(sql.starts_with(
|
||||
"SELECT `id`, CASE WHEN `location` IS NULL THEN NULL ELSE CONCAT('DBX_WKB:', ST_SRID(`location`), ':', HEX(ST_AsWKB(`location`))) END AS `location` FROM `places`"
|
||||
));
|
||||
assert!(sql.contains("WHERE `id` > 7"), "sql: {sql}");
|
||||
assert!(sql.contains("ORDER BY `id` ASC LIMIT 1000"), "sql: {sql}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_specific_boolean_export_literals() {
|
||||
let sqlserver_statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
|
||||
|
||||
Reference in new issue
Block a user