From 3be523b2f5237b4033ec3ee1502250d23563cba3 Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Sat, 5 Sep 2026 01:22:58 +0800 Subject: [PATCH] fix(jdbc): render generic table DDL for unmatched external JDBC drivers Closes #8187 --- crates/dbx-core/src/schema.rs | 79 +++++- .../main/java/app/dbx/jdbc/DbxJdbcPlugin.java | 67 +++++ .../app/dbx/jdbc/GenericJdbcDdlBuilder.java | 236 ++++++++++++++++++ .../java/app/dbx/jdbc/DbxJdbcPluginTest.java | 88 +++++++ 4 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 plugins/jdbc/src/main/java/app/dbx/jdbc/GenericJdbcDdlBuilder.java diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema.rs index 1358d7ad3a..1d0e57f28a 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema.rs @@ -3166,12 +3166,12 @@ mod tests { clickhouse_metadata_database, dameng_object_statistics_dba_segments_sql, dameng_object_statistics_rows_only_sql, dameng_object_statistics_user_segments_sql, deduplicate_column_infos, ephemeral_agent_metadata_session_id, external_driver_statistics_dialect, external_driver_statistics_query_plan, - external_driver_uses_mysql_ddl, filter_mongodb_agent_collections, filter_mysql_system_databases_for_config, - filter_object_infos, filter_table_infos, filter_visible_schema_names, finalize_object_source, - gaussdb_m_view_object_source_sql, gbase8a_object_statistics_sql, is_agent_postgres_metadata_fallback_config, - is_mysql_external_driver_config, is_oracle_external_driver_config, is_retryable_metadata_error, - metadata_error_action, metadata_name_or_comment_matches, mysql_database_list_timeout, - mysql_external_driver_ddl_from_query_result, mysql_external_driver_ddl_sql, + external_driver_uses_generic_ddl, external_driver_uses_mysql_ddl, filter_mongodb_agent_collections, + filter_mysql_system_databases_for_config, filter_object_infos, filter_table_infos, filter_visible_schema_names, + finalize_object_source, gaussdb_m_view_object_source_sql, gbase8a_object_statistics_sql, + is_agent_postgres_metadata_fallback_config, is_mysql_external_driver_config, is_oracle_external_driver_config, + is_retryable_metadata_error, metadata_error_action, metadata_name_or_comment_matches, + mysql_database_list_timeout, mysql_external_driver_ddl_from_query_result, mysql_external_driver_ddl_sql, mysql_object_source_ddl_column_index, mysql_object_source_sql, mysql_table_list_source_for_config, mysql_table_metadata_catalog, normalize_information_schema_table_type, oracle_columns_from_query_result, oracle_columns_sql, oracle_columns_sql_for_resolved_owner, oracle_current_schema_from_query_result, @@ -3603,6 +3603,30 @@ mod tests { assert!(!is_oracle_external_driver_config(&config)); } + #[test] + fn external_driver_generic_ddl_applies_to_unmatched_jdbc_connections() { + // JDBCX-style wrappers match no vendor DDL dialect and fall back to the + // plugin's generic DatabaseMetaData renderer. + let mut config = test_connection_config(DatabaseType::Jdbc); + config.connection_string = Some("jdbcx:wrap-jdbc:jdbc:mysql://127.0.0.1:3306/demo".to_string()); + config.jdbc_driver_class = Some("io.github.jdbcx.WrappedDriver".to_string()); + assert!(external_driver_uses_generic_ddl(&config)); + assert!(!external_driver_uses_mysql_ddl(&config)); + assert!(!is_oracle_external_driver_config(&config)); + + // Vendor dialects keep their dedicated SQL paths. + config.connection_string = Some("jdbc:mysql://127.0.0.1:3306/demo".to_string()); + config.jdbc_driver_class = None; + assert!(!external_driver_uses_generic_ddl(&config)); + + config.connection_string = Some("jdbc:oracle:thin:@//127.0.0.1:1521/ORCL".to_string()); + assert!(!external_driver_uses_generic_ddl(&config)); + + // Vendor-typed database never routes through the generic renderer. + let oracle = test_connection_config(DatabaseType::Oracle); + assert!(!external_driver_uses_generic_ddl(&oracle)); + } + #[test] fn external_driver_statistics_dialect_matches_jdbc_url_vendor() { let mut config = test_connection_config(DatabaseType::Jdbc); @@ -7804,6 +7828,11 @@ async fn get_table_ddl_once( let session = session.clone(); return external_driver_mysql_ddl(session, config.as_ref(), database, schema, table).await; } + if external_driver_uses_generic_ddl(config.as_ref()) { + let config = config.clone(); + let session = session.clone(); + return external_driver_jdbc_ddl(session, config.as_ref(), database, schema, table).await; + } } #[cfg(feature = "duckdb-sidecar")] if let Some(client) = extract_pool!(pool_handle.as_ref(), DuckDbWorker) { @@ -11001,6 +11030,44 @@ async fn external_driver_oracle_ddl( .ok_or_else(|| "JDBC Oracle plugin returned no table DDL".to_string()) } +/// External JDBC connections that match no vendor DDL dialect (JDBCX wrappers, +/// custom protocol drivers, …) have table DDL rendered by the plugin from +/// plain `DatabaseMetaData` via `getObjectSource`, mirroring the agent-side +/// `DdlBuilder` output. +fn external_driver_uses_generic_ddl(config: &ConnectionConfig) -> bool { + config.db_type == DatabaseType::Jdbc + && !is_oracle_external_driver_config(config) + && !external_driver_uses_mysql_ddl(config) +} + +async fn external_driver_jdbc_ddl( + session: std::sync::Arc, + config: &ConnectionConfig, + database: &str, + schema: &str, + table: &str, +) -> Result { + let result: serde_json::Value = session + .invoke_with_timeout( + "getObjectSource", + serde_json::json!({ + "connection": config, + "database": database, + "schema": schema, + "name": table, + "object_type": "TABLE", + }), + agent_metadata_timeout(Some(config)), + ) + .await?; + result + .get("source") + .and_then(serde_json::Value::as_str) + .filter(|source| !source.trim().is_empty()) + .map(str::to_string) + .ok_or_else(|| "JDBC plugin returned no table DDL".to_string()) +} + fn normalize_mysql_display_ddl(sql: String) -> String { ensure_display_ddl_terminated(repair_mysql_ddl_comments(&sql)) } diff --git a/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java b/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java index a262148bea..a7cfc12fbe 100644 --- a/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java +++ b/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java @@ -4028,9 +4028,76 @@ public final class DbxJdbcPlugin { throw new SQLException("Object source not found"); } + if ("TABLE".equals(normalizeObjectType(objectType))) { + return genericTableObjectSource(connection, database, schema, name); + } + throw new SQLException("Object source is not supported by this JDBC driver"); } + /** + * Table source for generic external JDBC drivers (JDBCX wrappers, custom + * protocol drivers) that expose no vendor DDL statement: assemble CREATE + * TABLE from {@code DatabaseMetaData} via the same readers the browse RPCs + * use, so driver quirks (catalog fallback, PK marking, tolerance of + * unimplemented metadata methods) apply identically. + */ + private static JsonNode genericTableObjectSource(JsonNode connection, String database, String schema, String table) + throws SQLException { + JsonNode columns = getColumns(connection, database, schema, table); + if (!columns.isArray() || columns.isEmpty()) { + throw new SQLException("Object source not found"); + } + JsonNode indexes = listIndexes(connection, database, schema, table); + + JsonNode foreignKeys; + try (Connection conn = openConnection(connection)) { + DatabaseMetaData meta = conn.getMetaData(); + JdbcDriverQuirks quirks = driverQuirks(connection); + String catalog = metadataCatalog(database, quirks); + String schemaPattern = resolveSchemaPattern(meta, database, schema, quirks); + foreignKeys = listGenericForeignKeys(meta, catalog, schemaPattern, table); + } + + String source = GenericJdbcDdlBuilder.buildTableDdl( + emptyToNull(schema) != null ? schema : emptyToNull(database), + table, + columns, + indexes, + foreignKeys + ); + ObjectNode item = MAPPER.createObjectNode(); + item.put("name", table); + item.put("object_type", "TABLE"); + putNullable(item, "schema", emptyToNull(schema)); + item.put("source", source); + return item; + } + + private static JsonNode listGenericForeignKeys(DatabaseMetaData meta, String catalog, String schemaPattern, String table) { + ArrayNode result = MAPPER.createArrayNode(); + try (ResultSet rs = meta.getImportedKeys(catalog, schemaPattern, table)) { + while (rs != null && rs.next()) { + String column = rs.getString("FKCOLUMN_NAME"); + String refTable = rs.getString("PKTABLE_NAME"); + String refColumn = rs.getString("PKCOLUMN_NAME"); + if (column == null || column.isBlank() || refTable == null || refTable.isBlank() + || refColumn == null || refColumn.isBlank()) { + continue; + } + ObjectNode item = MAPPER.createObjectNode(); + item.put("name", rs.getString("FK_NAME")); + item.put("column", column); + item.put("ref_table", refTable); + item.put("ref_column", refColumn); + result.add(item); + } + } catch (SQLException | AbstractMethodError | UnsupportedOperationException ignored) { + // Foreign keys are optional detail; generic DDL must still succeed. + } + return result; + } + private static JsonNode hive2ShowCreateObjectSource( Connection conn, String database, diff --git a/plugins/jdbc/src/main/java/app/dbx/jdbc/GenericJdbcDdlBuilder.java b/plugins/jdbc/src/main/java/app/dbx/jdbc/GenericJdbcDdlBuilder.java new file mode 100644 index 0000000000..f76a4abec9 --- /dev/null +++ b/plugins/jdbc/src/main/java/app/dbx/jdbc/GenericJdbcDdlBuilder.java @@ -0,0 +1,236 @@ +package app.dbx.jdbc; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Renders a CREATE TABLE statement from the plugin's generic JDBC metadata + * nodes. Used for external drivers that expose no vendor DDL statement, where + * {@code DatabaseMetaData} is the only portable metadata source. Identifiers + * are double-quoted per the SQL standard, matching the agent-side DdlBuilder. + */ +final class GenericJdbcDdlBuilder { + + private GenericJdbcDdlBuilder() { + } + + static String buildTableDdl(String schema, String table, JsonNode columns, JsonNode indexes, JsonNode foreignKeys) { + String tableRef = qualifiedName(schema, table); + List columnLines = new ArrayList<>(); + for (JsonNode column : columns) { + String name = column.path("name").asText(""); + if (!name.isBlank()) { + columnLines.add(columnLine(column, name)); + } + } + + Set primaryKeys = primaryKeysFromIndexes(indexes); + if (primaryKeys.isEmpty()) { + primaryKeys = primaryKeysFromColumnFlags(columns); + } + if (!primaryKeys.isEmpty()) { + columnLines.add(" PRIMARY KEY (" + joinQuoted(primaryKeys) + ")"); + } + + List foreignKeyGroups = groupForeignKeys(foreignKeys); + for (ForeignKeyGroup group : foreignKeyGroups) { + String constraint = notBlank(group.name) ? "CONSTRAINT " + quoteIdent(group.name) + " " : ""; + columnLines.add( + " " + constraint + + "FOREIGN KEY (" + joinQuoted(group.columns) + ") " + + "REFERENCES " + quoteIdent(group.refTable) + + " (" + joinQuoted(group.refColumns) + ")" + ); + } + Set foreignKeyColumnSets = new LinkedHashSet<>(); + for (ForeignKeyGroup group : foreignKeyGroups) { + foreignKeyColumnSets.add(String.join("\u0000", group.columns)); + } + + StringBuilder ddl = new StringBuilder(); + ddl.append("CREATE TABLE ").append(tableRef).append(" (\n"); + ddl.append(String.join(",\n", columnLines)); + ddl.append("\n);\n"); + + for (JsonNode index : indexes) { + if (index.path("is_primary").asBoolean()) { + continue; + } + String name = index.path("name").asText(""); + List indexColumns = textValues(index.path("columns")); + if (name.isBlank() || indexColumns.isEmpty()) { + continue; + } + if (!index.path("is_unique").asBoolean() && foreignKeyColumnSets.contains(String.join("\u0000", indexColumns))) { + // Many engines auto-create this index for the FOREIGN KEY + // constraint itself; re-creating it would collide on restore. + continue; + } + String unique = index.path("is_unique").asBoolean() ? "UNIQUE " : ""; + ddl.append("\nCREATE ").append(unique).append("INDEX ").append(quoteIdent(name)); + ddl.append(" ON ").append(tableRef); + ddl.append(" (").append(String.join(", ", quoteAll(indexColumns))).append(");\n"); + } + + return ddl.toString(); + } + + private static String columnLine(JsonNode column, String name) { + StringBuilder line = new StringBuilder(" ").append(quoteIdent(name)); + String type = column.path("data_type").asText(""); + if (!type.isBlank()) { + line.append(" ").append(typeWithLength(column, type)); + } + if (!column.path("is_nullable").asBoolean(true)) { + line.append(" NOT NULL"); + } + String defaultValue = column.path("column_default").asText(""); + if (!defaultValue.isBlank()) { + line.append(" DEFAULT ").append(defaultValue); + } + return line.toString(); + } + + private static String typeWithLength(JsonNode column, String type) { + String normalized = type.toLowerCase(java.util.Locale.ROOT); + Integer characterLength = nullableInt(column, "character_maximum_length"); + if (isCharacterType(normalized) && characterLength != null && characterLength > 0) { + return type + "(" + characterLength + ")"; + } + Integer precision = nullableInt(column, "numeric_precision"); + if (isNumericType(normalized) && precision != null) { + Integer scale = nullableInt(column, "numeric_scale"); + return scale != null ? type + "(" + precision + ", " + scale + ")" : type + "(" + precision + ")"; + } + return type; + } + + private static boolean isCharacterType(String normalized) { + return "character varying".equals(normalized) + || "varchar".equals(normalized) + || "nvarchar".equals(normalized) + || "char".equals(normalized) + || "nchar".equals(normalized) + || "character".equals(normalized); + } + + private static boolean isNumericType(String normalized) { + return "numeric".equals(normalized) || "decimal".equals(normalized); + } + + private static Set primaryKeysFromIndexes(JsonNode indexes) { + for (JsonNode index : indexes) { + if (index.path("is_primary").asBoolean()) { + Set columns = new LinkedHashSet<>(textValues(index.path("columns"))); + if (!columns.isEmpty()) { + return columns; + } + } + } + return Set.of(); + } + + private static Set primaryKeysFromColumnFlags(JsonNode columns) { + Set primaryKeys = new LinkedHashSet<>(); + for (JsonNode column : columns) { + if (column.path("is_primary_key").asBoolean()) { + String name = column.path("name").asText(""); + if (!name.isBlank()) { + primaryKeys.add(name); + } + } + } + return primaryKeys; + } + + private static List groupForeignKeys(JsonNode foreignKeys) { + List ordered = new ArrayList<>(); + Map namedGroups = new LinkedHashMap<>(); + for (JsonNode foreignKey : foreignKeys) { + String name = foreignKey.path("name").asText(""); + String refTable = foreignKey.path("ref_table").asText(""); + String column = foreignKey.path("column").asText(""); + String refColumn = foreignKey.path("ref_column").asText(""); + if (column.isBlank() || refTable.isBlank() || refColumn.isBlank()) { + continue; + } + ForeignKeyGroup group = null; + if (notBlank(name)) { + group = namedGroups.get(name + "\u0000" + refTable); + } + if (group == null) { + group = new ForeignKeyGroup(name, refTable); + ordered.add(group); + if (notBlank(name)) { + namedGroups.put(name + "\u0000" + refTable, group); + } + } + group.columns.add(column); + group.refColumns.add(refColumn); + } + return ordered; + } + + private static final class ForeignKeyGroup { + private final String name; + private final String refTable; + private final List columns = new ArrayList<>(); + private final List refColumns = new ArrayList<>(); + + private ForeignKeyGroup(String name, String refTable) { + this.name = name; + this.refTable = refTable; + } + } + + private static String qualifiedName(String schema, String name) { + if (!notBlank(schema)) { + return quoteIdent(name); + } + return quoteIdent(schema) + "." + quoteIdent(name); + } + + private static String quoteIdent(String identifier) { + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } + + private static String joinQuoted(Iterable values) { + return String.join(", ", quoteAll(values)); + } + + private static List quoteAll(Iterable values) { + List quoted = new ArrayList<>(); + for (String value : values) { + quoted.add(quoteIdent(value)); + } + return quoted; + } + + private static List textValues(JsonNode array) { + List values = new ArrayList<>(); + if (array.isArray()) { + for (JsonNode value : array) { + String text = value.asText(""); + if (!text.isBlank()) { + values.add(text); + } + } + } + return values; + } + + private static Integer nullableInt(JsonNode node, String field) { + JsonNode value = node.path(field); + return value.isMissingNode() || value.isNull() ? null : value.asInt(); + } + + private static boolean notBlank(String value) { + return value != null && !value.trim().isEmpty(); + } +} diff --git a/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java b/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java index f764d5fc15..e391b5b139 100644 --- a/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java +++ b/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java @@ -5132,4 +5132,92 @@ final class DbxJdbcPluginTest { String once = DbxJdbcPlugin.enrichDriverHint(connection, "Unsupported charset: ZHS16GBK"); assertEquals(once, DbxJdbcPlugin.enrichDriverHint(connection, once)); } + + @Test + void getObjectSourceBuildsExecutableTableDdlForPlainJdbcDrivers() throws Exception { + String sourceDb = "jdbc:h2:mem:dbx_ddl_src;DB_CLOSE_DELAY=-1"; + try (Statement st = DriverManager.getConnection(sourceDb, "sa", "").createStatement()) { + st.execute("CREATE TABLE \"dbx_ddl_parent\"(id BIGINT NOT NULL, CONSTRAINT pk_ddl_parent PRIMARY KEY(id))"); + st.execute("CREATE TABLE \"dbx_ddl_child\"(" + + "id BIGINT NOT NULL, name VARCHAR(50) NOT NULL DEFAULT 'x', parent_id BIGINT, " + + "CONSTRAINT pk_ddl_child PRIMARY KEY(id), " + + "CONSTRAINT fk_ddl_child FOREIGN KEY(parent_id) REFERENCES \"dbx_ddl_parent\"(id))"); + st.execute("CREATE UNIQUE INDEX \"uq_ddl_child_name\" ON \"dbx_ddl_child\"(name)"); + } + + JsonNode response = request("getObjectSource", """ + { + "connection": { "connection_string": "%s", "username": "sa", "connect_timeout_secs": 30 }, + "name": "dbx_ddl_child", + "object_type": "TABLE" + } + """.formatted(sourceDb)); + + assertFalse(response.has("error"), response.toString()); + JsonNode result = response.path("result"); + assertEquals("TABLE", result.path("object_type").asText()); + String source = result.path("source").asText(); + assertTrue(source.startsWith("CREATE TABLE "), source); + assertTrue(source.contains("PRIMARY KEY"), source); + assertTrue(source.contains("NOT NULL"), source); + assertTrue(source.contains("DEFAULT"), source); + assertTrue(source.contains("FOREIGN KEY"), source); + assertTrue(source.contains("REFERENCES"), source); + assertTrue(source.contains("CREATE UNIQUE INDEX"), source); + + // The generated DDL must be executable: rebuild the child table in a + // fresh database that only has the parent, then verify the rebuilt + // structure matches the original (columns, PK, FK, unique index). + String targetUrl = "jdbc:h2:mem:dbx_ddl_tgt;DB_CLOSE_DELAY=-1"; + try (Statement st = DriverManager.getConnection(targetUrl, "sa", "").createStatement()) { + st.execute("CREATE TABLE \"dbx_ddl_parent\"(id BIGINT NOT NULL, CONSTRAINT pk_ddl_parent PRIMARY KEY(id))"); + for (String statement : source.split(";")) { + if (!statement.isBlank()) { + st.execute(statement); + } + } + } + try (Statement st = DriverManager.getConnection(targetUrl, "sa", "").createStatement(); + ResultSet rs = st.executeQuery(""" + SELECT + (SELECT COUNT(*) FROM information_schema.columns + WHERE table_name = 'dbx_ddl_child') AS column_count, + (SELECT COUNT(*) FROM information_schema.table_constraints + WHERE table_name = 'dbx_ddl_child' AND constraint_type = 'PRIMARY KEY') AS pk_count, + (SELECT COUNT(*) FROM information_schema.table_constraints + WHERE table_name = 'dbx_ddl_child' AND constraint_type = 'FOREIGN KEY') AS fk_count, + (SELECT COUNT(*) FROM information_schema.indexes + WHERE table_name = 'dbx_ddl_child' AND index_name = 'uq_ddl_child_name') AS uq_index_count + """)) { + assertTrue(rs.next()); + assertEquals(3, rs.getInt("column_count")); + assertEquals(1, rs.getInt("pk_count")); + assertEquals(1, rs.getInt("fk_count")); + assertEquals(1, rs.getInt("uq_index_count")); + } + + request("close", """ + { "connection": { "connection_string": "%s", "username": "sa" } } + """.formatted(sourceDb)); + request("close", """ + { "connection": { "connection_string": "%s", "username": "sa" } } + """.formatted(targetUrl)); + } + + @Test + void getObjectSourceKeepsUnsupportedObjectTypesOnPlainJdbcDrivers() throws Exception { + JsonNode response = request("getObjectSource", """ + { + "connection": %s, + "name": "some_view", + "object_type": "VIEW" + } + """.formatted(CONNECTION)); + + assertTrue(response.has("error"), response.toString()); + assertEquals( + "Object source is not supported by this JDBC driver", + response.path("error").path("message").asText() + ); + } }