fix(dameng): speed up table DDL metadata lookup

Closes #10075

agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java

@@ -50,6 +50,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
@@ -2749,15 +2750,39 @@ public final class DamengAgent extends AbstractJdbcAgent {
         if (notBlank(tableComment) && !containsCommentOnTable(result.toString(), schema, table)) {
             appendCommentStatement(result, "COMMENT ON TABLE " + tableRef + " IS '" + sqlStringBody(tableComment) + "';");
         }
-        for (ColumnInfo column : getColumns(schema, table)) {
-            if (!notBlank(column.getComment()) || containsCommentOnColumn(result.toString(), schema, table, column.getName())) {
+        for (Map.Entry<String, String> column : tableColumnComments(schema, table).entrySet()) {
+            if (containsCommentOnColumn(result.toString(), schema, table, column.getKey())) {
                 continue;
             }
-            appendCommentStatement(result, "COMMENT ON COLUMN " + tableRef + "." + JdbcIdentifiers.INSTANCE.doubleQuote(column.getName()) + " IS '" + sqlStringBody(column.getComment()) + "';");
+            appendCommentStatement(result, "COMMENT ON COLUMN " + tableRef + "." + JdbcIdentifiers.INSTANCE.doubleQuote(column.getKey()) + " IS '" + sqlStringBody(column.getValue()) + "';");
         }
         return result.toString();
     }
 
+    private Map<String, String> tableColumnComments(String schema, String table) throws Exception {
+        Map<String, String> result = new LinkedHashMap<>();
+        String sql = """
+            SELECT /*+ PARALLEL(1) */ COLUMN_NAME, COMMENTS
+            FROM ALL_COL_COMMENTS
+            WHERE OWNER = ? AND TABLE_NAME = ? AND COMMENTS IS NOT NULL
+            ORDER BY COLUMN_NAME
+            """.stripIndent().trim();
+        try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
+            stmt.setString(1, schema);
+            stmt.setString(2, table);
+            try (ResultSet rs = stmt.executeQuery()) {
+                while (rs.next()) {
+                    String column = rs.getString(1);
+                    String comment = rs.getString(2);
+                    if (notBlank(column) && notBlank(comment)) {
+                        result.put(column, comment);
+                    }
+                }
+            }
+        }
+        return result;
+    }
+
     private String appendIndependentIndexDdl(String ddl, String schema, String table) throws Exception {
         StringBuilder result = new StringBuilder(ddl == null ? "" : ddl.trim());
         for (IndexInfo index : independentIndexes(schema, table)) {
@@ -2774,6 +2799,84 @@ public final class DamengAgent extends AbstractJdbcAgent {
     }
 
     private List<IndexInfo> independentIndexes(String schema, String table) throws Exception {
+        try {
+            return independentIndexesFromSystemCatalog(schema, table);
+        } catch (Exception systemCatalogError) {
+            if (isDamengConnectionError(systemCatalogError)) {
+                throw systemCatalogError;
+            }
+            try {
+                return independentIndexesFromDictionaryViews(schema, table);
+            } catch (Exception dictionaryViewError) {
+                dictionaryViewError.addSuppressed(systemCatalogError);
+                throw dictionaryViewError;
+            }
+        }
+    }
+
+    private List<IndexInfo> independentIndexesFromSystemCatalog(String schema, String table) throws Exception {
+        List<IndexInfo> result = new ArrayList<>();
+        // DM's ALL_INDEXES view performs privilege filtering across the complete catalog and can
+        // take tens of seconds in installations with very large index dictionaries (#10075).
+        // The DM JDBC driver and DBeaver instead resolve the requested table id first and read its
+        // SYS index rows directly. Keep the ALL_* implementation below as a compatibility and
+        // permission fallback for deployments that do not expose these catalog tables.
+        String sql = """
+            SELECT /*+ PARALLEL(1) */ index_object.NAME,
+                LISTAGG(column_object.NAME, ',') WITHIN GROUP (
+                    ORDER BY SF_GET_INDEX_KEY_SEQ(index_metadata.KEYNUM, index_metadata.KEYINFO, column_object.COLID)
+                ) AS COLUMNS,
+                index_metadata.ISUNIQUE,
+                index_metadata.TYPE$,
+                index_metadata.FLAG
+            FROM SYS.SYSOBJECTS schema_object
+            JOIN SYS.SYSOBJECTS table_object ON table_object.SCHID = schema_object.ID
+                AND table_object.TYPE$ = 'SCHOBJ' AND table_object.SUBTYPE$ = 'UTAB'
+            JOIN SYS.SYSOBJECTS index_object ON index_object.PID = table_object.ID
+                AND index_object.SUBTYPE$ = 'INDEX'
+            JOIN SYS.SYSINDEXES index_metadata ON index_metadata.ID = index_object.ID
+            JOIN SYS.SYSCOLUMNS column_object ON column_object.ID = table_object.ID
+                AND SF_COL_IS_IDX_KEY(
+                    index_metadata.KEYNUM,
+                    index_metadata.KEYINFO,
+                    column_object.COLID
+                ) = 1
+            WHERE schema_object.TYPE$ = 'SCH' AND schema_object.NAME = ? AND table_object.NAME = ?
+                AND NOT EXISTS (
+                    SELECT 1
+                    FROM SYS.SYSCONS constraint_metadata
+                    WHERE constraint_metadata.TABLEID = table_object.ID
+                        AND constraint_metadata.INDEXID = index_metadata.ID
+                        AND constraint_metadata.TYPE$ IN ('P', 'U')
+                )
+            GROUP BY index_object.NAME, index_metadata.ISUNIQUE, index_metadata.TYPE$, index_metadata.FLAG
+            ORDER BY index_object.NAME
+            """.stripIndent().trim();
+        try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
+            stmt.setString(1, schema);
+            stmt.setString(2, table);
+            try (ResultSet rs = stmt.executeQuery()) {
+                while (rs.next()) {
+                    String indexName = rs.getString(1);
+                    if (notBlank(indexName)) {
+                        result.add(new IndexInfo(
+                            indexName,
+                            splitNonEmpty(coalesce(rs.getString(2)), ","),
+                            "Y".equalsIgnoreCase(rs.getString(3)),
+                            false,
+                            null,
+                            damengSystemIndexType(rs.getString(4), rs.getInt(5)),
+                            null,
+                            null
+                        ));
+                    }
+                }
+            }
+        }
+        return result;
+    }
+
+    private List<IndexInfo> independentIndexesFromDictionaryViews(String schema, String table) throws Exception {
         List<IndexInfo> result = new ArrayList<>();
         // Primary-key and unique-constraint backing indexes are already represented in table DDL.
         String sql = """
@@ -2819,6 +2922,13 @@ public final class DamengAgent extends AbstractJdbcAgent {
         return result;
     }
 
+    private static String damengSystemIndexType(String catalogType, int flags) {
+        if ((flags & 3) != 0) {
+            return "INTERNAL";
+        }
+        return "ST".equalsIgnoreCase(coalesce(catalogType).trim()) ? "SPATIAL" : catalogType;
+    }
+
     static String indexDdl(String schema, String table, IndexInfo index) {
         StringBuilder ddl = new StringBuilder("CREATE ");
         if (index.getIs_unique()) {

agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java

@@ -21,6 +21,7 @@ import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
+import java.sql.SQLNonTransientConnectionException;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
@@ -1743,13 +1744,22 @@ class DamengAgentMetadataTest {
     @Test
     void appendsTableAndColumnCommentsToTableDdl() {
         DamengAgent agent = new DamengAgent();
-        TestSupport.setPrivateConnection(agent, metadataConnection());
+        List<String> sqls = new ArrayList<>();
+        TestSupport.setPrivateConnection(agent, metadataConnection(
+            "id comment",
+            null,
+            false,
+            List.of(),
+            sqls
+        ));
 
         String ddl = agent.getTableDdl("APP", "USERS");
 
         Assertions.assertTrue(ddl.contains("CREATE TABLE \"APP\".\"USERS\""), ddl);
         Assertions.assertTrue(ddl.contains("COMMENT ON TABLE \"APP\".\"USERS\" IS '用户示例表';"), ddl);
         Assertions.assertTrue(ddl.contains("COMMENT ON COLUMN \"APP\".\"USERS\".\"ID\" IS 'id comment';"), ddl);
+        Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_COL_COMMENTS")), String.join("\n", sqls));
+        Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("ALL_CONS_COLUMNS")), String.join("\n", sqls));
     }
 
     @Test
@@ -1861,13 +1871,74 @@ class DamengAgentMetadataTest {
             ddl.contains("CREATE UNIQUE INDEX \"APP\".\"UX_USERS_EMAIL\" ON \"APP\".\"USERS\" (\"EMAIL\");"),
             ddl
         );
+        Assertions.assertTrue(
+            ddl.contains("CREATE SPATIAL INDEX \"APP\".\"IDX_USERS_GEO\" ON \"APP\".\"USERS\" (\"GEO\");"),
+            ddl
+        );
         Assertions.assertFalse(ddl.contains("PK_USERS"), ddl);
-        String indexSql = sqls.stream().filter(sql -> sql.contains("ALL_INDEXES")).findFirst().orElseThrow();
-        Assertions.assertTrue(indexSql.contains("CONSTRAINT_TYPE IN ('P', 'U')"), indexSql);
+        String indexSql = sqls.stream().filter(sql -> sql.contains("SYS.SYSINDEXES")).findFirst().orElseThrow();
+        Assertions.assertTrue(indexSql.contains("constraint_metadata.TYPE$ IN ('P', 'U')"), indexSql);
+        Assertions.assertTrue(indexSql.contains("schema_object.NAME = ?"), indexSql);
+        Assertions.assertTrue(indexSql.contains("table_object.NAME = ?"), indexSql);
+        Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("ALL_INDEXES")), String.join("\n", sqls));
         long dbmsMetadataCalls = sqls.stream().filter(sql -> sql.contains("DBMS_METADATA.GET_DDL")).count();
         Assertions.assertEquals(1, dbmsMetadataCalls);
     }
 
+    @Test
+    void fallsBackToDictionaryViewsWhenSystemIndexCatalogIsUnavailable() {
+        DamengAgent agent = new DamengAgent();
+        List<String> sqls = new ArrayList<>();
+        TestSupport.setPrivateConnection(agent, metadataConnection(
+            "id comment",
+            null,
+            false,
+            List.of(indexRow("IDX_USERS_NAME", "NAME", "N", "BT", 0)),
+            sqls,
+            "CREATE TABLE \"APP\".\"USERS\" (\n  \"ID\" NUMBER\n);",
+            defaultColumnMetadataRows("id comment"),
+            null,
+            new SQLException("no SYS index catalog privilege")
+        ));
+
+        String ddl = agent.getTableDdl("APP", "USERS");
+
+        Assertions.assertTrue(
+            ddl.contains("CREATE INDEX \"APP\".\"IDX_USERS_NAME\" ON \"APP\".\"USERS\" (\"NAME\");"),
+            ddl
+        );
+        int systemQuery = indexOfSql(sqls, "SYS.SYSINDEXES");
+        int fallbackQuery = indexOfSql(sqls, "ALL_INDEXES");
+        Assertions.assertTrue(systemQuery >= 0, String.join("\n", sqls));
+        Assertions.assertTrue(fallbackQuery > systemQuery, String.join("\n", sqls));
+    }
+
+    @Test
+    void doesNotRetryIndexCatalogAfterConnectionFailure() {
+        DamengAgent agent = new DamengAgent();
+        List<String> sqls = new ArrayList<>();
+        TestSupport.setPrivateConnection(agent, metadataConnection(
+            "id comment",
+            null,
+            false,
+            List.of(),
+            sqls,
+            "CREATE TABLE \"APP\".\"USERS\" (\n  \"ID\" NUMBER\n);",
+            defaultColumnMetadataRows("id comment"),
+            null,
+            new SQLNonTransientConnectionException("connection lost")
+        ));
+
+        RuntimeException error = Assertions.assertThrows(
+            RuntimeException.class,
+            () -> agent.getTableDdl("APP", "USERS")
+        );
+
+        Assertions.assertEquals("connection lost", error.getCause().getMessage());
+        Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SYS.SYSINDEXES")), String.join("\n", sqls));
+        Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("ALL_INDEXES")), String.join("\n", sqls));
+    }
+
     @Test
     void skipsDamengInternalIndexesWhenAppendingTableDdl() {
         DamengAgent agent = new DamengAgent();
@@ -1877,8 +1948,8 @@ class DamengAgentMetadataTest {
             null,
             false,
             List.of(
-                indexRow("IDX_USERS_NAME", "NAME", "NONUNIQUE", "NORMAL"),
-                indexRow("SYS_INTERNAL_DDL", "ID", "NONUNIQUE", "INNER CLUSTER INDEX")
+                indexRow("IDX_USERS_NAME", "NAME", "N", "BT", 0),
+                indexRow("SYS_INTERNAL_DDL", "ID", "N", "BT", 1)
             ),
             sqls
         ));
@@ -2078,8 +2149,9 @@ class DamengAgentMetadataTest {
             null,
             false,
             List.of(
-                indexRow("IDX_USERS_NAME", "NAME", "NONUNIQUE", "NORMAL"),
-                indexRow("UX_USERS_EMAIL", "EMAIL", "UNIQUE", "NORMAL")
+                indexRow("IDX_USERS_NAME", "NAME", "N", "BT", 0),
+                indexRow("UX_USERS_EMAIL", "EMAIL", "Y", "BT", 0),
+                indexRow("IDX_USERS_GEO", "GEO", "N", "ST", 0)
             ),
             sqls
         );
@@ -2169,6 +2241,30 @@ class DamengAgentMetadataTest {
         String dbmsMetadataDdl,
         List<List<Object>> columnRows,
         String dbmsMetadataError
+    ) {
+        return metadataConnection(
+            allColumnComment,
+            fallbackColumnComment,
+            includeMaterializedView,
+            independentIndexes,
+            sqls,
+            dbmsMetadataDdl,
+            columnRows,
+            dbmsMetadataError,
+            null
+        );
+    }
+
+    private static Connection metadataConnection(
+        String allColumnComment,
+        String fallbackColumnComment,
+        boolean includeMaterializedView,
+        List<List<Object>> independentIndexes,
+        List<String> sqls,
+        String dbmsMetadataDdl,
+        List<List<Object>> columnRows,
+        String dbmsMetadataError,
+        SQLException systemIndexError
     ) {
         boolean[] dbmsMetadataResultOpen = {false};
         return proxy(Connection.class, (method, args) -> {
@@ -2190,6 +2286,12 @@ class DamengAgentMetadataTest {
                 if (sql.startsWith("SELECT NAME FROM SYS.SYSOBJECTS WHERE TYPE$ = 'SCH'")) {
                     return metadataStatement(List.of(List.of("APP"), List.of("EMPTY_SCHEMA"), List.of("SYSDBA")));
                 }
+                if (sql.contains("SYS.SYSINDEXES")) {
+                    if (systemIndexError != null) {
+                        return failingMetadataStatement(systemIndexError);
+                    }
+                    return metadataStatement(independentIndexes);
+                }
                 if (sql.contains("ALL_CONS_COLUMNS")) {
                     return metadataStatement(List.of(List.of("ID")));
                 }
@@ -2233,7 +2335,7 @@ class DamengAgentMetadataTest {
                     return metadataStatement(List.of());
                 }
                 if (sql.contains("ALL_INDEXES")) {
-                    return metadataStatement(independentIndexes);
+                    return metadataStatement(dictionaryIndexRows(independentIndexes));
                 }
                 if (sql.contains("ALL_TAB_COMMENTS")) {
                     List<List<Object>> rows = new ArrayList<>();
@@ -2261,7 +2363,9 @@ class DamengAgentMetadataTest {
                     return metadataStatement(List.of());
                 }
                 if (sql.contains("ALL_COL_COMMENTS") && !sql.contains("ALL_TAB_COLUMNS")) {
-                    return metadataStatement(List.of());
+                    return metadataStatement(
+                        allColumnComment == null ? List.of() : List.of(List.of("ID", allColumnComment))
+                    );
                 }
                 if (sql.contains("ALL_TAB_COLUMNS")) {
                     return metadataStatement(columnRows);
@@ -2309,8 +2413,26 @@ class DamengAgentMetadataTest {
         });
     }
 
-    private static List<Object> indexRow(String name, String columns, String uniqueness, String indexType) {
-        return List.of(name, columns, uniqueness, indexType);
+    private static List<Object> indexRow(String name, String columns, String uniqueness, String indexType, int flags) {
+        return List.of(name, columns, uniqueness, indexType, flags);
+    }
+
+    private static List<List<Object>> dictionaryIndexRows(List<List<Object>> systemRows) {
+        return systemRows.stream().map(row -> List.of(
+            row.get(0),
+            row.get(1),
+            "Y".equals(row.get(2)) ? "UNIQUE" : "NONUNIQUE",
+            "ST".equals(row.get(3)) ? "SPATIAL" : "NORMAL"
+        )).toList();
+    }
+
+    private static int indexOfSql(List<String> sqls, String fragment) {
+        for (int i = 0; i < sqls.size(); i++) {
+            if (sqls.get(i).contains(fragment)) {
+                return i;
+            }
+        }
+        return -1;
     }
 
     private static PreparedStatement dbmsMetadataStatement(String ddl, boolean[] resultOpen) {
@@ -2905,6 +3027,10 @@ class DamengAgentMetadataTest {
                     default -> null;
                 };
             }
+            if ("getInt".equals(name) && args[0] instanceof Integer columnIndex) {
+                Object value = rows.get(index[0]).get(columnIndex - 1);
+                return value instanceof Number number ? number.intValue() : Integer.parseInt(String.valueOf(value));
+            }
             if ("getCharacterStream".equals(name) && args[0] instanceof Integer columnIndex) {
                 Object value = rows.get(index[0]).get(columnIndex - 1);
                 if (value instanceof LongText longText) {