diff --git a/agents/common/src/main/java/com/dbx/agent/DdlBuilder.java b/agents/common/src/main/java/com/dbx/agent/DdlBuilder.java index 222d233a10..a03b1647b3 100644 --- a/agents/common/src/main/java/com/dbx/agent/DdlBuilder.java +++ b/agents/common/src/main/java/com/dbx/agent/DdlBuilder.java @@ -2,8 +2,10 @@ package com.dbx.agent; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -106,20 +108,35 @@ public final class DdlBuilder { } List primaryKeys = new ArrayList<>(); - for (ColumnInfo column : columns) { - if (column.getIs_primary_key()) { - primaryKeys.add(quoteIdent(column.getName(), useBacktick)); + for (IndexInfo index : indexes) { + if (!index.getIs_primary() || index.getColumns().isEmpty()) { + continue; + } + for (String column : index.getColumns()) { + primaryKeys.add(quoteIdent(column, useBacktick)); + } + break; + } + if (primaryKeys.isEmpty()) { + for (ColumnInfo column : columns) { + if (column.getIs_primary_key()) { + primaryKeys.add(quoteIdent(column.getName(), useBacktick)); + } } } if (!primaryKeys.isEmpty()) { columnLines.add(" PRIMARY KEY (" + join(primaryKeys, ", ") + ")"); } - for (ForeignKeyInfo fk : foreignKeys) { + for (ForeignKeyGroup foreignKey : groupForeignKeys(foreignKeys)) { + String constraint = notBlank(foreignKey.name) + ? "CONSTRAINT " + quoteIdent(foreignKey.name, useBacktick) + " " + : ""; columnLines.add( - " CONSTRAINT " + quoteIdent(fk.getName(), useBacktick) - + " FOREIGN KEY (" + quoteIdent(fk.getColumn(), useBacktick) + ") " - + "REFERENCES " + quoteIdent(fk.getRef_table(), useBacktick) + "(" + quoteIdent(fk.getRef_column(), useBacktick) + ")" + " " + constraint + + "FOREIGN KEY (" + joinQuoted(foreignKey.columns, useBacktick) + ") " + + "REFERENCES " + quoteIdent(foreignKey.refTable, useBacktick) + + "(" + joinQuoted(foreignKey.refColumns, useBacktick) + ")" ); } @@ -202,6 +219,50 @@ public final class DdlBuilder { return ddl.toString(); } + private static List groupForeignKeys(List foreignKeys) { + List result = new ArrayList<>(); + Map namedGroups = new LinkedHashMap<>(); + for (ForeignKeyInfo foreignKey : foreignKeys) { + ForeignKeyGroup group = null; + if (notBlank(foreignKey.getName()) && notBlank(foreignKey.getRef_table())) { + String groupKey = foreignKey.getName() + "\u0000" + foreignKey.getRef_table(); + group = namedGroups.get(groupKey); + if (group == null) { + group = new ForeignKeyGroup(foreignKey.getName(), foreignKey.getRef_table()); + namedGroups.put(groupKey, group); + result.add(group); + } + } + if (group == null) { + group = new ForeignKeyGroup(foreignKey.getName(), foreignKey.getRef_table()); + result.add(group); + } + group.columns.add(foreignKey.getColumn()); + group.refColumns.add(foreignKey.getRef_column()); + } + return result; + } + + private static String joinQuoted(List values, boolean useBacktick) { + List quotedValues = new ArrayList<>(); + for (String value : values) { + quotedValues.add(quoteIdent(value, useBacktick)); + } + return join(quotedValues, ", "); + } + + private static final class ForeignKeyGroup { + private final String name; + private final String refTable; + private final List columns = new ArrayList<>(); + private final List refColumns = new ArrayList<>(); + + private ForeignKeyGroup(String name, String refTable) { + this.name = name; + this.refTable = refTable; + } + } + private static String quoteIdent(String identifier, boolean useBacktick) { return useBacktick ? JdbcIdentifiers.INSTANCE.backtick(identifier) diff --git a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java index bba38fe52d..acbd05b197 100644 --- a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java @@ -3,9 +3,11 @@ package com.dbx.agent; import java.sql.ResultSet; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; public abstract class PostgresLikeAgent extends AbstractJdbcAgent { @@ -241,16 +243,23 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { @Override public String getTableDdl(String schema, String table) { + TableAttributeCache attributeCache = profile.mapsCatalogAttributeArraysInJava() + ? new TableAttributeCache(schema, table) + : null; List indexes; try { - indexes = listIndexes(schema, table); + indexes = attributeCache == null + ? listIndexes(schema, table) + : listIndexes(schema, table, attributeCache); } catch (RuntimeException e) { indexes = Collections.emptyList(); } List foreignKeys; try { - foreignKeys = listForeignKeys(schema, table); + foreignKeys = attributeCache == null + ? listForeignKeys(schema, table) + : listForeignKeys(schema, table, attributeCache); } catch (RuntimeException e) { foreignKeys = Collections.emptyList(); } @@ -274,7 +283,9 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { return DdlBuilder.buildTableDdl( schema, table, - getColumns(schema, table), + attributeCache == null + ? getColumns(schema, table) + : getColumns(schema, table, attributeCache), indexes, foreignKeys, checkConstraints, @@ -380,8 +391,19 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { @Override public List getColumns(String schema, String table) { + TableAttributeCache attributeCache = profile.mapsCatalogAttributeArraysInJava() + ? new TableAttributeCache(schema, table) + : null; + return getColumns(schema, table, attributeCache); + } + + private List getColumns( + String schema, + String table, + TableAttributeCache attributeCache + ) { return unchecked(() -> { - Set primaryKeys = primaryKeys(schema, table); + Set primaryKeys = primaryKeys(schema, table, attributeCache); List result = new ArrayList<>(); String sql = "SELECT a.attname AS column_name, " + profile.catalogBuiltinFunction("format_type") + "(a.atttypid, a.atttypmod) AS data_type, " + @@ -427,7 +449,14 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { }); } - private Set primaryKeys(String schema, String table) { + private Set primaryKeys( + String schema, + String table, + TableAttributeCache attributeCache + ) { + if (profile.mapsCatalogAttributeArraysInJava()) { + return primaryKeysFromCatalogArrays(schema, table, attributeCache); + } return unchecked(() -> { Set primaryKeys = new LinkedHashSet<>(); String sql = "SELECT a.attname AS column_name " + @@ -453,8 +482,56 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { }); } + private Set primaryKeysFromCatalogArrays( + String schema, + String table, + TableAttributeCache attributeCache + ) { + return unchecked(() -> { + List attributeNumbers = Collections.emptyList(); + String sql = "SELECT co.conkey AS column_numbers " + + "FROM " + profile.catalogRelation("constraint") + " co " + + "JOIN " + profile.catalogRelation("class") + " c ON c.oid = co.conrelid " + + "JOIN " + profile.catalogRelation("namespace") + " n ON n.oid = c.relnamespace " + + "WHERE co.contype = 'p' " + + "AND n.nspname = ? " + + "AND c.relname = ? " + + "ORDER BY co.conname"; + try (java.sql.PreparedStatement stmt = requireConnection().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + attributeNumbers = readAttributeNumbers(rs, "column_numbers"); + } + } + } + if (attributeNumbers.isEmpty()) { + return new LinkedHashSet<>(); + } + return new LinkedHashSet<>(mapRequiredAttributeNumbers( + attributeNumbers, + attributeCache.get() + )); + }); + } + @Override public List listIndexes(String schema, String table) { + TableAttributeCache attributeCache = profile.mapsCatalogAttributeArraysInJava() + ? new TableAttributeCache(schema, table) + : null; + return listIndexes(schema, table, attributeCache); + } + + private List listIndexes( + String schema, + String table, + TableAttributeCache attributeCache + ) { + if (profile.mapsCatalogAttributeArraysInJava()) { + return listIndexesFromCatalogArrays(schema, table, attributeCache); + } return unchecked(() -> { List result = new ArrayList<>(); String sql = "SELECT i.relname AS index_name, am.amname AS index_type, " + @@ -497,8 +574,80 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { }); } + private List listIndexesFromCatalogArrays( + String schema, + String table, + TableAttributeCache attributeCache + ) { + return unchecked(() -> { + List catalogIndexes = new ArrayList<>(); + String sql = "SELECT i.relname AS index_name, am.amname AS index_type, " + + "ix.indisunique AS is_unique, ix.indisprimary AS is_primary, " + + "ix.indkey AS column_numbers " + + "FROM " + profile.catalogRelation("index") + " ix " + + "JOIN " + profile.catalogRelation("class") + " t ON t.oid = ix.indrelid " + + "JOIN " + profile.catalogRelation("class") + " i ON i.oid = ix.indexrelid " + + "JOIN " + profile.catalogRelation("namespace") + " n ON n.oid = t.relnamespace " + + "JOIN " + profile.catalogRelation("am") + " am ON am.oid = i.relam " + + "WHERE n.nspname = ? AND t.relname = ? " + + "ORDER BY i.relname"; + try (java.sql.PreparedStatement stmt = requireConnection().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + catalogIndexes.add(new CatalogIndex( + rs.getString("index_name"), + rs.getString("index_type"), + rs.getBoolean("is_unique"), + rs.getBoolean("is_primary"), + readAttributeNumbers(rs, "column_numbers") + )); + } + } + } + if (catalogIndexes.isEmpty()) { + return Collections.emptyList(); + } + + Map attributes = attributeCache.get(); + List result = new ArrayList<>(); + for (CatalogIndex index : catalogIndexes) { + List columns = mapRequiredAttributeNumbers(index.attributeNumbers, attributes); + if (columns.isEmpty()) { + continue; + } + result.add(new IndexInfo( + index.name, + columns, + index.unique, + index.primary, + null, + index.type, + null, + null + )); + } + return result; + }); + } + @Override public List listForeignKeys(String schema, String table) { + TableAttributeCache attributeCache = profile.mapsCatalogAttributeArraysInJava() + ? new TableAttributeCache(schema, table) + : null; + return listForeignKeys(schema, table, attributeCache); + } + + private List listForeignKeys( + String schema, + String table, + TableAttributeCache attributeCache + ) { + if (profile.mapsCatalogAttributeArraysInJava()) { + return listForeignKeysFromCatalogArrays(schema, table, attributeCache); + } return unchecked(() -> { List result = new ArrayList<>(); String sql = "SELECT co.conname AS constraint_name, " + @@ -535,6 +684,309 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { }); } + private List listForeignKeysFromCatalogArrays( + String schema, + String table, + TableAttributeCache attributeCache + ) { + return unchecked(() -> { + List catalogForeignKeys = new ArrayList<>(); + String sql = "SELECT co.conname AS constraint_name, " + + "co.conkey AS column_numbers, co.confkey AS ref_column_numbers, " + + "rc.relname AS ref_table, rc.oid AS ref_table_oid " + + "FROM " + profile.catalogRelation("constraint") + " co " + + "JOIN " + profile.catalogRelation("class") + " c ON c.oid = co.conrelid " + + "JOIN " + profile.catalogRelation("namespace") + " n ON n.oid = c.relnamespace " + + "JOIN " + profile.catalogRelation("class") + " rc ON rc.oid = co.confrelid " + + "WHERE co.contype = 'f' " + + "AND n.nspname = ? " + + "AND c.relname = ? " + + "ORDER BY co.conname"; + try (java.sql.PreparedStatement stmt = requireConnection().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + Object refTableOid = rs.getObject("ref_table_oid"); + catalogForeignKeys.add(new CatalogForeignKey( + rs.getString("constraint_name"), + rs.getString("ref_table"), + refTableOid instanceof Number + ? ((Number) refTableOid).longValue() + : Long.parseLong(String.valueOf(refTableOid)), + readAttributeNumbers(rs, "column_numbers"), + readAttributeNumbers(rs, "ref_column_numbers") + )); + } + } + } + if (catalogForeignKeys.isEmpty()) { + return Collections.emptyList(); + } + + Map attributes = attributeCache.get(); + Set referencedRelationOids = new LinkedHashSet<>(); + for (CatalogForeignKey foreignKey : catalogForeignKeys) { + if (!foreignKey.attributeNumbers.isEmpty() + && foreignKey.attributeNumbers.size() == foreignKey.refAttributeNumbers.size()) { + referencedRelationOids.add(foreignKey.refTableOid); + } + } + Map> referencedAttributes = relationAttributeNames(referencedRelationOids); + List result = new ArrayList<>(); + for (CatalogForeignKey foreignKey : catalogForeignKeys) { + if (foreignKey.attributeNumbers.isEmpty() + || foreignKey.attributeNumbers.size() != foreignKey.refAttributeNumbers.size()) { + continue; + } + Map refAttributes = referencedAttributes.get(foreignKey.refTableOid); + if (refAttributes == null) { + continue; + } + List columns = mapRequiredAttributeNumbers( + foreignKey.attributeNumbers, + attributes + ); + List refColumns = mapRequiredAttributeNumbers( + foreignKey.refAttributeNumbers, + refAttributes + ); + if (columns.isEmpty() || refColumns.isEmpty()) { + continue; + } + for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { + result.add(new ForeignKeyInfo( + foreignKey.name, + columns.get(columnIndex), + foreignKey.refTable, + refColumns.get(columnIndex) + )); + } + } + return result; + }); + } + + private Map tableAttributeNames(String schema, String table) throws java.sql.SQLException { + Map result = new LinkedHashMap<>(); + String sql = "SELECT a.attnum AS attribute_number, a.attname AS column_name " + + "FROM " + profile.catalogRelation("attribute") + " a " + + "JOIN " + profile.catalogRelation("class") + " c ON c.oid = a.attrelid " + + "JOIN " + profile.catalogRelation("namespace") + " n ON n.oid = c.relnamespace " + + "WHERE n.nspname = ? AND c.relname = ? " + + "AND a.attnum > 0 AND NOT a.attisdropped " + + "ORDER BY a.attnum"; + try (java.sql.PreparedStatement stmt = requireConnection().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + result.put(rs.getInt("attribute_number"), rs.getString("column_name")); + } + } + } + return result; + } + + private Map> relationAttributeNames(Set relationOids) + throws java.sql.SQLException { + if (relationOids.isEmpty()) { + return Collections.emptyMap(); + } + + StringBuilder placeholders = new StringBuilder(); + for (int relationIndex = 0; relationIndex < relationOids.size(); relationIndex++) { + if (relationIndex > 0) { + placeholders.append(", "); + } + placeholders.append("?"); + } + + Map> result = new LinkedHashMap<>(); + String sql = "SELECT a.attrelid AS relation_oid, " + + "a.attnum AS attribute_number, a.attname AS column_name " + + "FROM " + profile.catalogRelation("attribute") + " a " + + "WHERE a.attrelid IN (" + placeholders + ") " + + "AND a.attnum > 0 AND NOT a.attisdropped " + + "ORDER BY a.attrelid, a.attnum"; + try (java.sql.PreparedStatement stmt = requireConnection().prepareStatement(sql)) { + int parameterIndex = 1; + for (Long relationOid : relationOids) { + stmt.setLong(parameterIndex, relationOid); + parameterIndex += 1; + } + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + long relationOid = rs.getLong("relation_oid"); + Map attributes = result.get(relationOid); + if (attributes == null) { + attributes = new LinkedHashMap<>(); + result.put(relationOid, attributes); + } + attributes.put(rs.getInt("attribute_number"), rs.getString("column_name")); + } + } + } + return result; + } + + private static List mapRequiredAttributeNumbers( + List attributeNumbers, + Map attributes + ) { + List result = new ArrayList<>(); + for (Integer attributeNumber : attributeNumbers) { + if (attributeNumber == null || attributeNumber <= 0) { + return Collections.emptyList(); + } + String attribute = attributes.get(attributeNumber); + if (attribute == null) { + return Collections.emptyList(); + } + result.add(attribute); + } + return result.size() == attributeNumbers.size() + ? result + : Collections.emptyList(); + } + + private static List readAttributeNumbers(ResultSet rs, String columnName) + throws java.sql.SQLException { + Object rawValue = null; + java.sql.Array sqlArray = null; + try { + sqlArray = rs.getArray(columnName); + if (sqlArray != null) { + rawValue = sqlArray.getArray(); + } + } catch (java.sql.SQLException | UnsupportedOperationException | IncompatibleClassChangeError ignored) { + } finally { + if (sqlArray != null) { + freeSqlArray(sqlArray); + } + } + if (rawValue == null) { + try { + rawValue = rs.getObject(columnName); + } catch (java.sql.SQLException | UnsupportedOperationException | IncompatibleClassChangeError ignored) { + } + } + if (rawValue == null) { + rawValue = rs.getString(columnName); + } + return parseAttributeNumbers(rawValue); + } + + private static List parseAttributeNumbers(Object rawValue) throws java.sql.SQLException { + if (rawValue == null) { + return Collections.emptyList(); + } + if (rawValue instanceof java.sql.Array) { + java.sql.Array sqlArray = (java.sql.Array) rawValue; + try { + return parseAttributeNumbers(sqlArray.getArray()); + } finally { + freeSqlArray(sqlArray); + } + } + + List result = new ArrayList<>(); + if (rawValue.getClass().isArray()) { + int length = java.lang.reflect.Array.getLength(rawValue); + for (int arrayIndex = 0; arrayIndex < length; arrayIndex++) { + appendAttributeNumbers(result, java.lang.reflect.Array.get(rawValue, arrayIndex)); + } + return result; + } + appendAttributeNumbers(result, rawValue); + return result; + } + + private static void freeSqlArray(java.sql.Array sqlArray) { + try { + sqlArray.free(); + } catch (java.sql.SQLException | UnsupportedOperationException | IncompatibleClassChangeError ignored) { + } + } + + private static void appendAttributeNumbers(List result, Object rawValue) { + if (rawValue == null) { + return; + } + if (rawValue instanceof Number) { + result.add(((Number) rawValue).intValue()); + return; + } + java.util.regex.Matcher matcher = java.util.regex.Pattern + .compile("-?\\d+") + .matcher(String.valueOf(rawValue)); + while (matcher.find()) { + result.add(Integer.parseInt(matcher.group())); + } + } + + private final class TableAttributeCache { + private final String schema; + private final String table; + private Map attributes; + + private TableAttributeCache(String schema, String table) { + this.schema = schema; + this.table = table; + } + + private Map get() throws java.sql.SQLException { + if (attributes == null) { + attributes = tableAttributeNames(schema, table); + } + return attributes; + } + } + + private static final class CatalogIndex { + private final String name; + private final String type; + private final boolean unique; + private final boolean primary; + private final List attributeNumbers; + + private CatalogIndex( + String name, + String type, + boolean unique, + boolean primary, + List attributeNumbers + ) { + this.name = name; + this.type = type; + this.unique = unique; + this.primary = primary; + this.attributeNumbers = attributeNumbers; + } + } + + private static final class CatalogForeignKey { + private final String name; + private final String refTable; + private final long refTableOid; + private final List attributeNumbers; + private final List refAttributeNumbers; + + private CatalogForeignKey( + String name, + String refTable, + long refTableOid, + List attributeNumbers, + List refAttributeNumbers + ) { + this.name = name; + this.refTable = refTable; + this.refTableOid = refTableOid; + this.attributeNumbers = attributeNumbers; + this.refAttributeNumbers = refAttributeNumbers; + } + } + @Override public List listTriggers(String schema, String table) { return unchecked(() -> { diff --git a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgentProfile.java b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgentProfile.java index afea7880cb..4c4e5db6d1 100644 --- a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgentProfile.java +++ b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgentProfile.java @@ -6,6 +6,7 @@ public final class PostgresLikeAgentProfile { private final int defaultPort; private final String catalogSchema; private final String catalogPrefix; + private final boolean mapCatalogAttributeArraysInJava; public PostgresLikeAgentProfile(String driverClass, String urlTemplate) { this(driverClass, urlTemplate, 0, "pg_catalog", "pg_"); @@ -26,6 +27,17 @@ public final class PostgresLikeAgentProfile { int defaultPort, String catalogSchema, String catalogPrefix + ) { + this(driverClass, urlTemplate, defaultPort, catalogSchema, catalogPrefix, false); + } + + private PostgresLikeAgentProfile( + String driverClass, + String urlTemplate, + int defaultPort, + String catalogSchema, + String catalogPrefix, + boolean mapCatalogAttributeArraysInJava ) { this.driverClass = driverClass; this.urlTemplate = urlTemplate; @@ -35,6 +47,7 @@ public final class PostgresLikeAgentProfile { // the full metadata implementation for each compatible database. this.catalogSchema = catalogSchema; this.catalogPrefix = catalogPrefix; + this.mapCatalogAttributeArraysInJava = mapCatalogAttributeArraysInJava; } public String getDriverClass() { @@ -61,6 +74,21 @@ public final class PostgresLikeAgentProfile { return catalogPrefix + "toast_temp_"; } + public PostgresLikeAgentProfile withCatalogAttributeArraysMappedInJava() { + return new PostgresLikeAgentProfile( + driverClass, + urlTemplate, + defaultPort, + catalogSchema, + catalogPrefix, + true + ); + } + + public boolean mapsCatalogAttributeArraysInJava() { + return mapCatalogAttributeArraysInJava; + } + public String catalogRelation(String name) { return catalogSchema + "." + catalogPrefix + name; } diff --git a/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java b/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java index d0bdcd98cd..60a10d6040 100644 --- a/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java +++ b/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java @@ -595,6 +595,40 @@ class CommonJavaCompatibilityTest { ); } + @Test + void buildsCompositeKeysFromOrderedIndexAndForeignKeyMetadata() { + String ddl = DdlBuilder.buildTableDdl( + "avatar_asset", + "app_config_info", + Arrays.asList( + new ColumnInfo("id", "bigint", false, null, true), + new ColumnInfo("tenant_id", "bigint", false, null, true), + new ColumnInfo("payload", "text", true, null, false) + ), + Collections.singletonList(new IndexInfo( + "app_config_info_pkey", + Arrays.asList("tenant_id", "id"), + true, + true + )), + Arrays.asList( + new ForeignKeyInfo("fk_account_region", "tenant_id", "accounts", "account_id"), + new ForeignKeyInfo("fk_account_region", "id", "accounts", "region_id") + ) + ); + + assertEquals( + "CREATE TABLE \"avatar_asset\".\"app_config_info\" (\n" + + " \"id\" bigint NOT NULL,\n" + + " \"tenant_id\" bigint NOT NULL,\n" + + " \"payload\" text,\n" + + " PRIMARY KEY (\"tenant_id\", \"id\"),\n" + + " CONSTRAINT \"fk_account_region\" FOREIGN KEY (\"tenant_id\", \"id\") REFERENCES \"accounts\"(\"account_id\", \"region_id\")\n" + + ");\n", + ddl + ); + } + @Test void buildsTableDdlWithColumnComments() { String ddl = DdlBuilder.buildTableDdl( diff --git a/agents/common/src/test/java/com/dbx/agent/PostgresLikeAgentTest.java b/agents/common/src/test/java/com/dbx/agent/PostgresLikeAgentTest.java index 5984bffd40..cbcb973108 100644 --- a/agents/common/src/test/java/com/dbx/agent/PostgresLikeAgentTest.java +++ b/agents/common/src/test/java/com/dbx/agent/PostgresLikeAgentTest.java @@ -52,6 +52,8 @@ class PostgresLikeAgentTest { assertFalse(sql.contains("FROM pg_index"), sql); assertFalse(sql.contains(" AS key "), sql); assertFalse(sql.contains(" key."), sql); + assertTrue(sql.contains("JOIN LATERAL"), sql); + assertFalse(agent.getProfile().mapsCatalogAttributeArraysInJava()); } @Test @@ -86,6 +88,8 @@ class PostgresLikeAgentTest { assertTrue(sql.contains("ux_catalog.ux_get_functiondef"), sql); assertTrue(sql.contains("ux_catalog.ux_get_expr"), sql); assertFalse(sql.contains("pg_catalog"), sql); + assertTrue(sql.contains("JOIN LATERAL"), sql); + assertFalse(agent.getProfile().mapsCatalogAttributeArraysInJava()); } @Test diff --git a/agents/drivers/vastbase/src/main/java/com/dbx/agent/vastbase/VastbaseAgent.java b/agents/drivers/vastbase/src/main/java/com/dbx/agent/vastbase/VastbaseAgent.java index 5f4e3bb6ca..6ebe37fc8d 100644 --- a/agents/drivers/vastbase/src/main/java/com/dbx/agent/vastbase/VastbaseAgent.java +++ b/agents/drivers/vastbase/src/main/java/com/dbx/agent/vastbase/VastbaseAgent.java @@ -9,7 +9,7 @@ public final class VastbaseAgent extends PostgresLikeAgent { public static final PostgresLikeAgentProfile VASTBASE_PROFILE = new PostgresLikeAgentProfile( "cn.com.vastbase.Driver", "jdbc:vastbase://{host}:{port}/{database}" - ); + ).withCatalogAttributeArraysMappedInJava(); public VastbaseAgent() { super(VASTBASE_PROFILE); diff --git a/agents/drivers/vastbase/src/test/java/com/dbx/agent/vastbase/VastbaseAgentTest.java b/agents/drivers/vastbase/src/test/java/com/dbx/agent/vastbase/VastbaseAgentTest.java index ab8cb9a55e..049766fffa 100644 --- a/agents/drivers/vastbase/src/test/java/com/dbx/agent/vastbase/VastbaseAgentTest.java +++ b/agents/drivers/vastbase/src/test/java/com/dbx/agent/vastbase/VastbaseAgentTest.java @@ -1,10 +1,25 @@ package com.dbx.agent.vastbase; +import com.dbx.agent.AbstractJdbcAgent; +import com.dbx.agent.ColumnInfo; import com.dbx.agent.DatabaseAgent; +import com.dbx.agent.ForeignKeyInfo; +import com.dbx.agent.IndexInfo; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + class VastbaseAgentTest extends JdbcFakeExecutionBehaviorTest { @Override protected DatabaseAgent createAgent() { @@ -22,5 +37,485 @@ class VastbaseAgentTest extends JdbcFakeExecutionBehaviorTest { Assertions.assertEquals("cn.com.vastbase.Driver", agent.getProfile().getDriverClass()); Assertions.assertEquals("jdbc:vastbase://{host}:{port}/{database}", agent.getProfile().getUrlTemplate()); + Assertions.assertTrue(agent.getProfile().mapsCatalogAttributeArraysInJava()); + } + + @Test + void mapsTablesWithoutPrimaryKeysFromCatalogArrays() throws Exception { + VastbaseMetadataFake metadata = new VastbaseMetadataFake(null); + VastbaseAgent agent = agentWithConnection(metadata.connection()); + + List columns = agent.getColumns("avatar_asset", "app_config_info"); + + Assertions.assertFalse(column(columns, "id").getIs_primary_key()); + Assertions.assertFalse(column(columns, "tenant_id").getIs_primary_key()); + metadata.assertUsesPostgres92CompatibleSql(); + } + + @Test + void mapsSinglePrimaryKeysFromCatalogArrays() throws Exception { + VastbaseMetadataFake metadata = new VastbaseMetadataFake(new Short[]{1}); + VastbaseAgent agent = agentWithConnection(metadata.connection()); + + List columns = agent.getColumns("avatar_asset", "app_config_info"); + + Assertions.assertTrue(column(columns, "id").getIs_primary_key()); + Assertions.assertFalse(column(columns, "tenant_id").getIs_primary_key()); + metadata.assertUsesPostgres92CompatibleSql(); + } + + @Test + void rejectsEntirePrimaryKeyWhenAnyAttributeNumberIsUnknown() throws Exception { + VastbaseMetadataFake metadata = new VastbaseMetadataFake(new Short[]{1, 99}); + VastbaseAgent agent = agentWithConnection(metadata.connection()); + + List columns = agent.getColumns("avatar_asset", "app_config_info"); + + Assertions.assertFalse(column(columns, "id").getIs_primary_key()); + Assertions.assertFalse(column(columns, "tenant_id").getIs_primary_key()); + Assertions.assertFalse(column(columns, "payload").getIs_primary_key()); + metadata.assertUsesPostgres92CompatibleSql(); + } + + @Test + void mapsCompositeKeysIndexesAndForeignKeysInCatalogOrder() throws Exception { + VastbaseMetadataFake metadata = new VastbaseMetadataFake(new Short[]{2, 1}); + VastbaseAgent agent = agentWithConnection(metadata.connection()); + + List columns = agent.getColumns("avatar_asset", "app_config_info"); + List indexes = agent.listIndexes("avatar_asset", "app_config_info"); + List foreignKeys = agent.listForeignKeys("avatar_asset", "app_config_info"); + metadata.assertSqlArraysFreed(); + metadata.clearRecordedOperations(); + String ddl = agent.getTableDdl("avatar_asset", "app_config_info"); + + Assertions.assertTrue(column(columns, "id").getIs_primary_key()); + Assertions.assertTrue(column(columns, "tenant_id").getIs_primary_key()); + Assertions.assertFalse(column(columns, "payload").getIs_primary_key()); + Assertions.assertEquals(Arrays.asList( + new IndexInfo("idx_payload", Arrays.asList("payload"), false, false, null, "btree", null, null), + new IndexInfo("idx_tenant_id", Arrays.asList("tenant_id", "id"), true, false, null, "btree", null, null), + new IndexInfo("idx_primitive", Arrays.asList("id", "payload"), false, false, null, "btree", null, null), + new IndexInfo("idx_object_array_null", Arrays.asList("tenant_id", "payload"), false, false, null, "btree", null, null), + new IndexInfo("idx_object_array_unsupported", Arrays.asList("id", "tenant_id"), false, false, null, "btree", null, null), + new IndexInfo("app_config_info_pkey", Arrays.asList("tenant_id", "id"), true, true, null, "btree", null, null) + ), indexes); + Assertions.assertEquals(Arrays.asList( + new ForeignKeyInfo("fk_account_region", "tenant_id", "accounts", "account_id"), + new ForeignKeyInfo("fk_account_region", "id", "accounts", "region_id"), + new ForeignKeyInfo("fk_owner", "id", "users", "user_id") + ), foreignKeys); + Assertions.assertTrue(ddl.contains("PRIMARY KEY (\"tenant_id\", \"id\")"), ddl); + Assertions.assertTrue( + ddl.contains("CONSTRAINT \"fk_account_region\" FOREIGN KEY (\"tenant_id\", \"id\") REFERENCES \"accounts\"(\"account_id\", \"region_id\")"), + ddl + ); + Assertions.assertEquals(1, occurrences(ddl, "CONSTRAINT \"fk_account_region\""), ddl); + Assertions.assertFalse(ddl.contains("fk_length_mismatch"), ddl); + Assertions.assertFalse(ddl.contains("fk_unknown_ref"), ddl); + Assertions.assertTrue( + ddl.contains("CREATE UNIQUE INDEX \"idx_tenant_id\" ON \"avatar_asset\".\"app_config_info\" USING btree (\"tenant_id\", \"id\")"), + ddl + ); + Assertions.assertEquals(8, metadata.queryCount(), metadata.sql()); + Assertions.assertEquals(1, metadata.tableAttributeQueryCount(), metadata.sql()); + Assertions.assertEquals(1, metadata.referencedAttributeQueryCount(), metadata.sql()); + Assertions.assertTrue(metadata.sql().contains("WHERE a.attrelid IN (?, ?)"), metadata.sql()); + metadata.assertSqlArraysFreed(); + metadata.assertUsesPostgres92CompatibleSql(); + Assertions.assertTrue(metadata.sql().contains("co.conkey AS column_numbers"), metadata.sql()); + Assertions.assertTrue(metadata.sql().contains("co.confkey AS ref_column_numbers"), metadata.sql()); + Assertions.assertTrue(metadata.sql().contains("ix.indkey AS column_numbers"), metadata.sql()); + } + + private static int occurrences(String value, String fragment) { + int result = 0; + int offset = 0; + while (true) { + int found = value.indexOf(fragment, offset); + if (found < 0) { + return result; + } + result += 1; + offset = found + fragment.length(); + } + } + + private static VastbaseAgent agentWithConnection(Connection connection) throws Exception { + VastbaseAgent agent = new VastbaseAgent(); + Field connectionField = AbstractJdbcAgent.class.getDeclaredField("connection"); + connectionField.setAccessible(true); + connectionField.set(agent, connection); + return agent; + } + + private static ColumnInfo column(List columns, String name) { + for (ColumnInfo column : columns) { + if (name.equals(column.getName())) { + return column; + } + } + throw new AssertionError("Missing column: " + name); + } + + private static final class VastbaseMetadataFake { + private final Object primaryKeyNumbers; + private final List statements = new ArrayList<>(); + private final List sqlArrays = new ArrayList<>(); + + private VastbaseMetadataFake(Object primaryKeyNumbers) { + this.primaryKeyNumbers = primaryKeyNumbers; + } + + private Connection connection() { + return proxy(Connection.class, new MethodHandler() { + @Override + public Object handle(Method method, Object[] args) { + String methodName = method.getName(); + if ("prepareStatement".equals(methodName)) { + String sql = (String) args[0]; + statements.add(sql); + return preparedStatement(sql); + } + if ("isClosed".equals(methodName)) { + return false; + } + if ("close".equals(methodName)) { + return null; + } + return defaultValue(method.getReturnType()); + } + }); + } + + private PreparedStatement preparedStatement(String sql) { + return proxy(PreparedStatement.class, new MethodHandler() { + @Override + public Object handle(Method method, Object[] args) { + String methodName = method.getName(); + if ("executeQuery".equals(methodName)) { + return resultFor(sql); + } + if ("setLong".equals(methodName)) { + return null; + } + if ("setString".equals(methodName) || "close".equals(methodName)) { + return null; + } + return defaultValue(method.getReturnType()); + } + }); + } + + private ResultSet resultFor(String sql) { + if (sql.contains("co.contype = 'p'")) { + if (primaryKeyNumbers == null) { + return resultSet(new String[]{"column_numbers"}, new Object[0][]); + } + return resultSet( + new String[]{"column_numbers"}, + new Object[][]{{primaryKeyNumbers}} + ); + } + if (sql.contains("ix.indkey AS column_numbers")) { + return resultSet( + new String[]{"index_name", "index_type", "is_unique", "is_primary", "column_numbers"}, + new Object[][]{ + {"idx_payload", "btree", false, false, sqlArray(new int[]{3})}, + {"idx_tenant_id", "btree", true, false, "2 1"}, + {"idx_primitive", "btree", false, false, new short[]{1, 3}}, + {"idx_object_array_null", "btree", false, false, objectSqlArray(new int[]{2, 3}, false)}, + {"idx_object_array_unsupported", "btree", false, false, objectSqlArray(new int[]{1, 2}, true)}, + {"app_config_info_pkey", "btree", true, true, new Short[]{2, 1}}, + {"idx_expression", "btree", false, false, new short[]{0, 3}}, + {"idx_unknown", "btree", false, false, "{2,99}"} + } + ); + } + if (sql.contains("co.confkey AS ref_column_numbers")) { + return resultSet( + new String[]{ + "constraint_name", + "column_numbers", + "ref_column_numbers", + "ref_table", + "ref_table_oid" + }, + new Object[][]{ + {"fk_account_region", "{2,1}", "{1,3}", "accounts", 42L}, + {"fk_length_mismatch", "{1,2}", "{1}", "accounts", 42L}, + {"fk_owner", new Short[]{1}, new Short[]{1}, "users", 43L}, + {"fk_unknown_ref", "{1}", "{99}", "users", 43L} + } + ); + } + if (sql.contains("WHERE a.attrelid IN (")) { + return resultSet( + new String[]{"relation_oid", "attribute_number", "column_name"}, + new Object[][]{ + {42L, 1, "account_id"}, + {42L, 3, "region_id"}, + {43L, 1, "user_id"} + } + ); + } + if (sql.contains("a.attnum AS attribute_number")) { + return attributeResult(new Object[][]{ + {1, "id"}, + {2, "tenant_id"}, + {3, "payload"} + }); + } + if (sql.contains("AS constraint_definition")) { + return resultSet( + new String[]{"constraint_name", "constraint_definition"}, + new Object[0][] + ); + } + if (sql.contains("AS table_comment")) { + return resultSet(new String[]{"table_comment"}, new Object[][]{{null}}); + } + if (sql.contains("AS data_type")) { + return resultSet( + new String[]{ + "column_name", + "data_type", + "is_nullable", + "column_default", + "column_comment", + "numeric_precision", + "numeric_scale", + "character_maximum_length" + }, + new Object[][]{ + {"id", "bigint", false, null, null, null, null, null}, + {"tenant_id", "bigint", false, null, null, null, null, null}, + {"payload", "text", true, null, null, null, null, null} + } + ); + } + throw new AssertionError("Unexpected SQL: " + sql); + } + + private ResultSet attributeResult(Object[][] rows) { + return resultSet(new String[]{"attribute_number", "column_name"}, rows); + } + + private String sql() { + return String.join("\n", statements); + } + + private java.sql.Array sqlArray(Object values) { + TrackingSqlArray sqlArray = new TrackingSqlArray(values); + sqlArrays.add(sqlArray); + return sqlArray.value(); + } + + private Object objectSqlArray(Object values, boolean unsupported) { + TrackingSqlArray sqlArray = new TrackingSqlArray(values); + sqlArrays.add(sqlArray); + return new ObjectSqlArray(sqlArray.value(), unsupported); + } + + private void clearRecordedOperations() { + statements.clear(); + sqlArrays.clear(); + } + + private int queryCount() { + return statements.size(); + } + + private int tableAttributeQueryCount() { + int result = 0; + for (String statement : statements) { + if (statement.contains("a.attnum AS attribute_number") + && statement.contains("JOIN pg_catalog.pg_class c")) { + result += 1; + } + } + return result; + } + + private int referencedAttributeQueryCount() { + int result = 0; + for (String statement : statements) { + if (statement.contains("a.attrelid AS relation_oid")) { + result += 1; + } + } + return result; + } + + private void assertSqlArraysFreed() { + Assertions.assertFalse(sqlArrays.isEmpty()); + for (TrackingSqlArray sqlArray : sqlArrays) { + Assertions.assertTrue(sqlArray.freed); + } + } + + private void assertUsesPostgres92CompatibleSql() { + Assertions.assertFalse(sql().contains("LATERAL"), sql()); + Assertions.assertFalse(sql().contains("WITH ORDINALITY"), sql()); + } + } + + private static ResultSet resultSet(String[] columns, Object[][] rows) { + int[] rowIndex = {-1}; + boolean[] wasNull = {false}; + return proxy(ResultSet.class, new MethodHandler() { + @Override + public Object handle(Method method, Object[] args) { + String methodName = method.getName(); + if ("next".equals(methodName)) { + rowIndex[0] += 1; + return rowIndex[0] < rows.length; + } + if ("getString".equals(methodName)) { + Object value = columnValue(columns, rows[rowIndex[0]], args[0]); + wasNull[0] = value == null; + return value == null ? null : String.valueOf(value); + } + if ("getBoolean".equals(methodName)) { + Object value = columnValue(columns, rows[rowIndex[0]], args[0]); + wasNull[0] = value == null; + return value instanceof Boolean && (Boolean) value; + } + if ("getInt".equals(methodName)) { + Object value = columnValue(columns, rows[rowIndex[0]], args[0]); + wasNull[0] = value == null; + return value == null ? 0 : ((Number) value).intValue(); + } + if ("getLong".equals(methodName)) { + Object value = columnValue(columns, rows[rowIndex[0]], args[0]); + wasNull[0] = value == null; + return value == null ? 0L : ((Number) value).longValue(); + } + if ("getObject".equals(methodName)) { + Object value = columnValue(columns, rows[rowIndex[0]], args[0]); + if (value instanceof ObjectSqlArray) { + value = ((ObjectSqlArray) value).value; + } + wasNull[0] = value == null; + return value; + } + if ("getArray".equals(methodName)) { + Object value = columnValue(columns, rows[rowIndex[0]], args[0]); + if (value instanceof ObjectSqlArray) { + ObjectSqlArray objectSqlArray = (ObjectSqlArray) value; + if (objectSqlArray.unsupported) { + throw new UnsupportedOperationException(); + } + return null; + } + wasNull[0] = value == null; + return value instanceof java.sql.Array ? value : null; + } + if ("wasNull".equals(methodName)) { + return wasNull[0]; + } + if ("close".equals(methodName)) { + return null; + } + return defaultValue(method.getReturnType()); + } + }); + } + + private static Object columnValue(String[] columns, Object[] row, Object key) { + if (key instanceof Number) { + return row[((Number) key).intValue() - 1]; + } + for (int columnIndex = 0; columnIndex < columns.length; columnIndex++) { + if (columns[columnIndex].equalsIgnoreCase(String.valueOf(key))) { + return row[columnIndex]; + } + } + return null; + } + + private static T proxy(Class type, MethodHandler handler) { + InvocationHandler invocationHandler = new InvocationHandler() { + @Override + public Object invoke(Object proxy, Method method, Object[] args) { + return handler.handle(method, args); + } + }; + return type.cast(Proxy.newProxyInstance( + type.getClassLoader(), + new Class[]{type}, + invocationHandler + )); + } + + private static Object defaultValue(Class type) { + if (Boolean.TYPE.equals(type)) { + return false; + } + if (Byte.TYPE.equals(type)) { + return (byte) 0; + } + if (Short.TYPE.equals(type)) { + return (short) 0; + } + if (Integer.TYPE.equals(type)) { + return 0; + } + if (Long.TYPE.equals(type)) { + return 0L; + } + if (Float.TYPE.equals(type)) { + return 0.0f; + } + if (Double.TYPE.equals(type)) { + return 0.0; + } + return null; + } + + private static final class TrackingSqlArray { + private final Object values; + private boolean freed; + + private TrackingSqlArray(Object values) { + this.values = values; + } + + private java.sql.Array value() { + return proxy(java.sql.Array.class, new MethodHandler() { + @Override + public Object handle(Method method, Object[] args) { + String methodName = method.getName(); + if ("getArray".equals(methodName) && (args == null || args.length == 0)) { + return values; + } + if ("free".equals(methodName)) { + freed = true; + return null; + } + if ("getBaseType".equals(methodName)) { + return java.sql.Types.SMALLINT; + } + if ("getBaseTypeName".equals(methodName)) { + return "int2"; + } + return defaultValue(method.getReturnType()); + } + }); + } + } + + private static final class ObjectSqlArray { + private final java.sql.Array value; + private final boolean unsupported; + + private ObjectSqlArray(java.sql.Array value, boolean unsupported) { + this.value = value; + this.unsupported = unsupported; + } + } + + private interface MethodHandler { + Object handle(Method method, Object[] args); } }