fix(metadata): unify enum column metadata

This commit is contained in:
onenewcode authored and GitHub committed 2026-07-08 21:02:53 +08:00
1 parent c32d0b7d6e
commit 1b13d2176b
21 files changed
+675 -202

No files matched your search

@@ -348,6 +348,7 @@ pub async fn get_columns(client: &ChClient, database: &str, table: &str) -> Resu
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
})
.collect())
@@ -333,6 +333,7 @@ fn push_mapping_column(
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
});
}
@@ -423,6 +423,7 @@ pub async fn get_columns(client: &InfluxdbClient, database: &str, table: &str) -
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
};
let cols: Vec<ColumnInfo> = std::iter::once(time_col)
@@ -437,6 +438,7 @@ pub async fn get_columns(client: &InfluxdbClient, database: &str, table: &str) -
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}))
.chain(field_series.first().into_iter().flat_map(|s| s.values.iter()).map(|row| {
let data_type = row.get(1).and_then(|v| v.as_str()).unwrap_or("unknown").to_string();
@@ -451,6 +453,7 @@ pub async fn get_columns(client: &InfluxdbClient, database: &str, table: &str) -
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
}))
.collect();
@@ -515,6 +518,7 @@ async fn get_columns_v2(client: &InfluxdbClient, bucket: &str, measurement: &str
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
};
let tag_cols = flux_column_values(&tag_result, "_value")
.into_iter()
@@ -530,6 +534,7 @@ async fn get_columns_v2(client: &InfluxdbClient, bucket: &str, measurement: &str
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
});
let field_cols = flux_column_values(&field_result, "_value").into_iter().map(|name| ColumnInfo {
name,
@@ -542,6 +547,7 @@ async fn get_columns_v2(client: &InfluxdbClient, bucket: &str, measurement: &str
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
});
Ok(std::iter::once(time_col).chain(tag_cols).chain(field_cols).collect())
}
+91 -3
View File
@@ -2054,8 +2054,8 @@ pub async fn list_completion_objects(pool: &MySqlPool, database: &str) -> Result
fn columns_sql(database: &str, table: &str) -> String {
format!(
"SELECT c.COLUMN_NAME, c.COLUMN_TYPE, c.IS_NULLABLE, c.COLUMN_DEFAULT, c.EXTRA, \
c.COLUMN_COMMENT, c.COLUMN_KEY, c.NUMERIC_PRECISION, c.NUMERIC_SCALE, c.CHARACTER_MAXIMUM_LENGTH \
"SELECT c.COLUMN_NAME, c.DATA_TYPE, c.COLUMN_TYPE, c.IS_NULLABLE, c.COLUMN_DEFAULT, c.EXTRA, \
c.COLUMN_COMMENT, c.COLUMN_KEY, c.NUMERIC_PRECISION, c.NUMERIC_SCALE, c.CHARACTER_MAXIMUM_LENGTH, \
FROM information_schema.COLUMNS c \
WHERE c.TABLE_SCHEMA = {} AND c.TABLE_NAME = {} \
ORDER BY c.ORDINAL_POSITION",
@@ -2128,6 +2128,65 @@ fn fix_potential_double_encoding(s: &str) -> String {
}
}
fn parse_mysql_enum_values(column_type: &str) -> Option<Vec<String>> {
let trimmed = column_type.trim();
if !trimmed.get(..5)?.eq_ignore_ascii_case("enum(") || !trimmed.ends_with(')') {
return None;
}
let inner = &trimmed[5..trimmed.len() - 1];
let mut chars = inner.chars().peekable();
let mut values = Vec::new();
loop {
while matches!(chars.peek(), Some(c) if c.is_whitespace()) {
chars.next();
}
match chars.next() {
Some('\'') => {}
None if values.is_empty() => return Some(values),
_ => return None,
}
let mut value = String::new();
loop {
match chars.next() {
Some('\'') => {
if matches!(chars.peek(), Some('\'')) {
chars.next();
value.push('\'');
} else {
break;
}
}
Some('\\') => match chars.next() {
Some('0') => value.push('\0'),
Some('b') => value.push('\u{0008}'),
Some('n') => value.push('\n'),
Some('r') => value.push('\r'),
Some('t') => value.push('\t'),
Some('Z') => value.push('\u{001A}'),
Some(c @ ('\\' | '\'' | '"')) => value.push(c),
Some(c) => value.push(c),
None => return None,
},
Some(c) => value.push(c),
None => return None,
}
}
values.push(value);
while matches!(chars.peek(), Some(c) if c.is_whitespace()) {
chars.next();
}
match chars.next() {
Some(',') => continue,
None => return Some(values),
_ => return None,
}
}
}
pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
let sql = columns_sql(database, table);
let mut conn = get_conn_with_health_check(pool).await?;
@@ -2150,10 +2209,19 @@ pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Resul
return None;
}
let column_key = get_str_by_name(row, "COLUMN_KEY");
let data_type = get_str_by_name(row, "DATA_TYPE");
let column_type = get_str_by_name(row, "COLUMN_TYPE");
let enum_values = if data_type.eq_ignore_ascii_case("enum") {
// MySQL exposes enum literals only through COLUMN_TYPE. Parse the SQL literal
// syntax in Rust so empty values, quotes, and backslash escapes survive intact.
parse_mysql_enum_values(&column_type)
} else {
None
};
Some(ColumnInfo {
is_primary_key: column_key.eq_ignore_ascii_case("PRI"),
name,
data_type: get_str_by_name(row, "COLUMN_TYPE"),
data_type: column_type,
is_nullable: get_str_by_name(row, "IS_NULLABLE") == "YES",
column_default: get_opt_str(row, "COLUMN_DEFAULT"),
extra: get_opt_str(row, "EXTRA"),
@@ -2163,6 +2231,7 @@ pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Resul
numeric_precision: get_opt_i32(row, "NUMERIC_PRECISION"),
numeric_scale: get_opt_i32(row, "NUMERIC_SCALE"),
character_maximum_length: get_opt_i32(row, "CHARACTER_MAXIMUM_LENGTH"),
enum_values,
})
})
.collect();
@@ -2209,6 +2278,7 @@ pub async fn get_columns_show(pool: &MySqlPool, database: &str, table: &str) ->
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
})
})
.collect())
@@ -3625,7 +3695,25 @@ mod tests {
assert!(!sql.contains("KEY_COLUMN_USAGE"));
assert!(!sql.contains("CONSTRAINT_NAME = 'PRIMARY'"));
assert!(sql.contains("c.COLUMN_KEY"));
assert!(sql.contains("c.DATA_TYPE"));
assert!(sql.contains("c.COLUMN_TYPE"));
assert!(!sql.contains("COLLATE"));
assert!(!sql.contains("AS ENUM_VALUES"));
}
#[test]
fn parse_mysql_enum_values_preserves_mysql_literal_edges() {
assert_eq!(
parse_mysql_enum_values("enum('pending','active','archived')"),
Some(vec!["pending".to_string(), "active".to_string(), "archived".to_string()])
);
assert_eq!(parse_mysql_enum_values("ENUM('','a')"), Some(vec!["".to_string(), "a".to_string()]));
assert_eq!(parse_mysql_enum_values("enum('x'',''y','z')"), Some(vec!["x','y".to_string(), "z".to_string()]));
assert_eq!(
parse_mysql_enum_values(r#"enum('it''s','quote\"d','back\\slash')"#),
Some(vec!["it's".to_string(), "quote\"d".to_string(), "back\\slash".to_string()])
);
assert_eq!(parse_mysql_enum_values("varchar(255)"), None);
}
#[test]
+1
View File
@@ -154,6 +154,7 @@ pub async fn get_columns(pool: &mysql_async::Pool, schema: &str, table: &str) ->
numeric_precision: precision,
numeric_scale: scale,
character_maximum_length: length,
enum_values: None,
}
})
.collect())
+125 -3
View File
@@ -2059,9 +2059,13 @@ const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
CASE WHEN t.typname = 'numeric' AND a.atttypmod > 0 \
THEN (a.atttypmod - 4) & 65535 ELSE NULL END AS numeric_scale, \
CASE WHEN t.typname IN ('varchar', 'bpchar') AND a.atttypmod > 0 \
THEN a.atttypmod - 4 ELSE NULL END AS character_maximum_length \
THEN a.atttypmod - 4 ELSE NULL END AS character_maximum_length, \
CASE WHEN enum_t.oid IS NULL THEN NULL \
ELSE COALESCE((SELECT array_to_json(array_agg(e.enumlabel ORDER BY e.enumsortorder))::text \
FROM pg_enum e WHERE e.enumtypid = enum_t.oid), '[]') END AS enum_values \
FROM pg_attribute a \
JOIN pg_type t ON t.oid = a.atttypid \
LEFT JOIN pg_type enum_t ON enum_t.oid = CASE WHEN t.typtype = 'd' THEN t.typbasetype WHEN t.typtype = 'e' THEN t.oid ELSE NULL END AND enum_t.typtype = 'e' \
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
LEFT JOIN pg_depend dep ON dep.refobjid = a.attrelid AND dep.refobjsubid = a.attnum AND dep.deptype = 'i' \
LEFT JOIN pg_sequence pseq ON pseq.seqrelid = dep.objid \
@@ -2088,7 +2092,8 @@ const POSTGRES_COLUMNS_COMPAT_SQL: &str = "SELECT a.attname AS column_name, \
CASE WHEN t.typname = 'numeric' AND a.atttypmod > 0 \
THEN (a.atttypmod - 4) & 65535 ELSE NULL END AS numeric_scale, \
CASE WHEN t.typname IN ('varchar', 'bpchar') AND a.atttypmod > 0 \
THEN a.atttypmod - 4 ELSE NULL END AS character_maximum_length \
THEN a.atttypmod - 4 ELSE NULL END AS character_maximum_length, \
NULL::text AS enum_values \
FROM pg_attribute a \
JOIN pg_type t ON t.oid = a.atttypid \
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
@@ -2119,11 +2124,17 @@ const POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL: &str = "SELECT c.column_name, \
NULL::text AS column_extra, \
CAST(c.numeric_precision AS int) AS numeric_precision, \
CAST(c.numeric_scale AS int) AS numeric_scale, \
CAST(c.character_maximum_length AS int) AS character_maximum_length \
CAST(c.character_maximum_length AS int) AS character_maximum_length, \
NULL::text AS enum_values \
FROM information_schema.columns c \
WHERE c.table_schema = $1 AND c.table_name = $2 \
ORDER BY c.ordinal_position";
fn parse_enum_values_from_row(row: &Row, index: usize) -> Option<Vec<String>> {
let raw = row.try_get::<_, Option<String>>(index).ok().flatten()?;
serde_json::from_str::<Vec<String>>(&raw).ok()
}
/// Read a boolean column from a PostgreSQL row, tolerating databases that
/// encode booleans as integers (0/1) or text ('t'/'f') instead of the standard
/// `bool` OID. Returns `None` when the column is NULL or truly unreadable.
@@ -2181,6 +2192,7 @@ fn column_info_from_row(row: &Row) -> ColumnInfo {
numeric_precision: row.try_get::<_, Option<i32>>(7).ok().flatten(),
numeric_scale: row.try_get::<_, Option<i32>>(8).ok().flatten(),
character_maximum_length: row.try_get::<_, Option<i32>>(9).ok().flatten(),
enum_values: parse_enum_values_from_row(row, 10),
}
}
@@ -3184,8 +3196,82 @@ pub async fn copy_in(pool: &Pool, sql: &str, data: &[u8]) -> Result<(), String>
#[cfg(test)]
mod tests {
use super::*;
use std::process::Command;
use std::time::Instant;
use tokio_postgres::types::FromSql;
struct DockerPostgres {
name: String,
port: u16,
}
impl DockerPostgres {
fn url(&self) -> String {
format!("postgres://postgres:postgres@127.0.0.1:{}/postgres?sslmode=disable", self.port)
}
}
impl Drop for DockerPostgres {
fn drop(&mut self) {
let _ = Command::new("docker").args(["rm", "-f", &self.name]).status();
}
}
fn docker_ready() -> bool {
Command::new("docker")
.args(["version", "--format", "{{.Server.Version}}"])
.output()
.map(|output| output.status.success())
.unwrap_or(false)
}
async fn start_docker_postgres() -> Option<DockerPostgres> {
if !docker_ready() {
eprintln!("skipping docker-backed postgres test because Docker is unavailable");
return None;
}
let port = portpicker::pick_unused_port().expect("pick unused postgres port");
let container = DockerPostgres { name: format!("dbx-postgres-enum-{}", uuid::Uuid::new_v4()), port };
let status = Command::new("docker")
.args([
"run",
"-d",
"--rm",
"--name",
&container.name,
"-e",
"POSTGRES_PASSWORD=postgres",
"-e",
"POSTGRES_USER=postgres",
"-e",
"POSTGRES_DB=postgres",
"-p",
&format!("{port}:5432"),
"postgres:16-alpine",
])
.status()
.expect("start docker postgres");
assert!(status.success(), "docker run postgres container should succeed");
let deadline = Instant::now() + Duration::from_secs(60);
loop {
match connect(&container.url(), Duration::from_secs(2)).await {
Ok(pool) => {
drop(pool);
return Some(container);
}
Err(_) if Instant::now() < deadline => tokio::time::sleep(Duration::from_millis(500)).await,
Err(error) => panic!("docker postgres did not become ready: {error}"),
}
}
}
fn state_enum_values(columns: &[ColumnInfo]) -> Option<Vec<String>> {
columns.iter().find(|column| column.name == "state").and_then(|column| column.enum_values.clone())
}
// --- pg_quote_ident ---
#[test]
@@ -3653,6 +3739,8 @@ mod tests {
assert!(POSTGRES_COLUMNS_SQL.contains("generated always as identity"));
assert!(POSTGRES_COLUMNS_SQL.contains("COALESCE(c.is_nullable = 'YES', NOT a.attnotnull)"));
assert!(POSTGRES_COLUMNS_SQL.contains("LEFT JOIN information_schema.columns"));
assert!(POSTGRES_COLUMNS_SQL.contains("pg_enum"));
assert!(POSTGRES_COLUMNS_SQL.contains("AS enum_values"));
}
#[test]
@@ -3663,6 +3751,8 @@ mod tests {
assert!(POSTGRES_COLUMNS_COMPAT_SQL.contains("col_description"));
assert!(POSTGRES_COLUMNS_COMPAT_SQL.contains("COALESCE(c.is_nullable = 'YES', NOT a.attnotnull)"));
assert!(POSTGRES_COLUMNS_COMPAT_SQL.contains("LEFT JOIN information_schema.columns"));
assert!(POSTGRES_COLUMNS_COMPAT_SQL.contains("NULL::text AS enum_values"));
assert!(!POSTGRES_COLUMNS_COMPAT_SQL.contains("pg_enum"));
}
#[test]
@@ -3670,10 +3760,42 @@ mod tests {
assert!(POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL.contains("information_schema.columns"));
assert!(POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL.contains("information_schema.table_constraints"));
assert!(POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL.contains("information_schema.key_column_usage"));
assert!(POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL.contains("NULL::text AS enum_values"));
assert!(!POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL.contains("pg_attribute"));
assert!(!POSTGRES_COLUMNS_INFORMATION_SCHEMA_SQL.contains("regclass"));
}
#[tokio::test]
async fn postgres_column_metadata_query_returns_enum_values_against_real_postgres() {
let Some(container) = start_docker_postgres().await else {
return;
};
let pool = connect(&container.url(), Duration::from_secs(5)).await.expect("connect postgres");
let schema = format!("dbx_enum_meta_{}", std::process::id());
let schema_ident = format!("\"{}\"", schema.replace('\"', "\"\""));
let table = format!("{schema_ident}.orders");
let type_ident = format!("{schema_ident}.\"status\"");
execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema");
execute_query(&pool, &format!("CREATE TYPE {type_ident} AS ENUM ('pending', 'active', 'archived')"))
.await
.expect("create enum type");
execute_query(&pool, &format!("CREATE TABLE {table} (id integer PRIMARY KEY, state {type_ident} NOT NULL)"))
.await
.expect("create table");
let client =
checkout_postgres_client(&pool, None, crate::db::connection_timeout()).await.expect("checkout client");
let columns =
get_columns_with_sql(&client, POSTGRES_COLUMNS_SQL, &schema, "orders").await.expect("primary columns");
assert_eq!(
state_enum_values(&columns),
Some(vec!["pending".to_string(), "active".to_string(), "archived".to_string()])
);
}
#[tokio::test]
#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"]
async fn postgres_column_metadata_decode_type_mismatch_uses_fallbacks() {
+1
View File
@@ -111,6 +111,7 @@ pub async fn get_columns(pool: &Pool, _schema: &str, table: &str) -> Result<Vec<
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
})
.collect())
+1
View File
@@ -145,6 +145,7 @@ pub async fn get_columns(client: &RqliteClient, _schema: &str, table: &str) -> R
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
})
.collect())
}
+1
View File
@@ -1141,6 +1141,7 @@ pub async fn get_columns(pool: &SqliteHandle, _schema: &str, table: &str) -> Res
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
})
})
.map_err(|e| e.to_string())?;
+2
View File
@@ -876,6 +876,7 @@ pub async fn get_linked_server_columns(
numeric_precision: column_size,
numeric_scale,
character_maximum_length: linked_i32(row, 15),
enum_values: None,
})
})
.collect())
@@ -1436,6 +1437,7 @@ pub async fn get_columns(client: &mut SqlServerClient, schema: &str, table: &str
numeric_precision: num_prec,
numeric_scale: num_scale,
character_maximum_length: max_len,
enum_values: None,
}
})
.collect())
+1
View File
@@ -184,6 +184,7 @@ pub async fn get_columns(client: &TursoClient, _schema: &str, table: &str) -> Re
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
})
.collect())
}
+7
View File
@@ -236,6 +236,7 @@ pub fn duckdb_query_columns_in_database_with_attached(
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
})
})
.map_err(|e| e.to_string())?;
@@ -1047,6 +1048,7 @@ fn oracle_columns_from_query_result(result: db::QueryResult) -> Vec<db::ColumnIn
numeric_precision: precision,
numeric_scale: scale,
character_maximum_length: length,
enum_values: None,
})
})
.collect()
@@ -1980,6 +1982,7 @@ fn presto_like_columns_from_query_result(result: &db::QueryResult) -> Vec<db::Co
numeric_precision: presto_like_numeric_precision(&data_type),
numeric_scale: presto_like_numeric_scale(&data_type),
character_maximum_length: presto_like_character_maximum_length(&data_type),
enum_values: None,
})
})
.collect()
@@ -2080,6 +2083,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
}
@@ -5263,6 +5267,7 @@ mod object_source_tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
};
let mut ignored = column.clone();
ignored.name = "EMPTY_COMMENT".to_string();
@@ -5295,6 +5300,7 @@ mod object_source_tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
};
let ddl = append_oracle_comments_to_ddl(
@@ -5328,6 +5334,7 @@ mod ddl_tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
}
+2
View File
@@ -1454,6 +1454,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
}
@@ -1853,6 +1854,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}),
target: None,
changes: Vec::new(),
+8
View File
@@ -2317,6 +2317,7 @@ fn mongo_columns_from_documents(documents: &[serde_json::Value]) -> Vec<db::Colu
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
})
.collect()
@@ -3491,6 +3492,7 @@ where
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
});
}
sql_target_column_names = sql_target_columns.iter().map(|column| column.name.clone()).collect();
@@ -4388,6 +4390,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
}
}
@@ -5216,6 +5219,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
},
db::ColumnInfo {
name: "identity_id".to_string(),
@@ -5228,6 +5232,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
},
db::ColumnInfo {
name: "computed_id".to_string(),
@@ -5240,6 +5245,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
},
],
"users",
@@ -5313,6 +5319,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
},
db::ColumnInfo {
name: "name".to_string(),
@@ -5325,6 +5332,7 @@ mod tests {
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
},
];
let source_ddl = crate::schema::render_postgres_table_ddl("public", "it_quick_entry", &columns, &[], &[]);
+2
View File
@@ -92,6 +92,8 @@ pub struct ColumnInfo {
pub numeric_precision: Option<i32>,
pub numeric_scale: Option<i32>,
pub character_maximum_length: Option<i32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub enum_values: Option<Vec<String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]