diff --git a/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java b/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java index 7f8b0672f2..946c8aaa64 100644 --- a/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java +++ b/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java @@ -61,6 +61,8 @@ public final class TDengineAgent extends AbstractJdbcAgent { Pattern.compile("(?i)^(binary|nchar|varchar|varbinary)\\((\\d+)\\)"); private static final Pattern COMPOSITE_KEY_PATTERN = Pattern.compile("(?i)\\bCOMPOSITE\\s+KEY\\b"); + private static final Pattern TDENGINE_VERSION_PATTERN = + Pattern.compile("^(\\d+)\\.(\\d+)\\.(\\d+)(?:\\.|$)"); private final Object tableCacheLock = new Object(); private String tableCacheSchema = ""; @@ -69,6 +71,7 @@ public final class TDengineAgent extends AbstractJdbcAgent { private Connection restfulStateConnection; private String restfulOriginalCatalog; private String restfulOriginalDatabase; + private boolean restInformationSchemaPagingEnabled; @Override protected String driverClass() { @@ -101,6 +104,7 @@ public final class TDengineAgent extends AbstractJdbcAgent { protected void afterConnect(ConnectParams params, Connection connection) { clearTableCache(); clearRestfulState(); + restInformationSchemaPagingEnabled = supportsRestInformationSchemaPaging(connection); } @Override @@ -119,6 +123,7 @@ public final class TDengineAgent extends AbstractJdbcAgent { protected void afterDisconnect() { clearTableCache(); clearRestfulState(); + restInformationSchemaPagingEnabled = false; } @Override @@ -154,9 +159,79 @@ public final class TDengineAgent extends AbstractJdbcAgent { if (!normalized.tableTypeAllowed("TABLE")) { return Collections.emptyList(); } + if (normalized.hasLimit() && !normalized.hasFilter()) { + if (restInformationSchemaPagingEnabled) { + try { + return listTablesPageFromRestInformationSchema(schema, normalized); + } catch (RuntimeException ignored) { + } + } + return listTablesPageFromShow(schema, normalized); + } return normalized.filterTables(listTablesFromShow(schema)); } + private List listTablesPageFromRestInformationSchema( + String schema, + MetadataListConstraints constraints + ) { + return unchecked(() -> { + int limit = constraints.getLimit(); + int offset = constraints.getOffset() == null ? 0 : constraints.getOffset(); + List result = new ArrayList<>(limit); + + TablePageScan stablePage = queryTablesPage( + "SHOW " + quoteQualifiedPrefix(schema) + "STABLES", + "STABLE", + false, + offset, + limit + ); + result.addAll(stablePage.tables()); + if (result.size() >= limit) { + return result; + } + + int tableOffset = Math.max(0, offset - stablePage.scannedRows()); + result.addAll(queryInformationSchemaTablePage( + schema, + tableOffset, + limit - result.size() + )); + return result; + }); + } + + private List listTablesPageFromShow(String schema, MetadataListConstraints constraints) { + return unchecked(() -> { + int limit = constraints.getLimit(); + int offset = constraints.getOffset() == null ? 0 : constraints.getOffset(); + List result = new ArrayList<>(limit); + + TablePageScan stablePage = queryTablesPage( + "SHOW " + quoteQualifiedPrefix(schema) + "STABLES", + "STABLE", + false, + offset, + limit + ); + result.addAll(stablePage.tables()); + if (result.size() >= limit) { + return result; + } + + int tableOffset = Math.max(0, offset - stablePage.scannedRows()); + result.addAll(queryTablesPage( + "SHOW " + quoteQualifiedPrefix(schema) + "TABLES", + "TABLE", + true, + tableOffset, + limit - result.size() + ).tables()); + return result; + }); + } + private List listTablesFromShow(String schema) { List cached = cachedTables(schema); if (cached != null) { @@ -164,9 +239,9 @@ public final class TDengineAgent extends AbstractJdbcAgent { } return unchecked(() -> { List result = new ArrayList<>(); - // Connector/J 3.6.3 ignores Statement#setMaxRows. Read the SHOW - // results once and page locally from a short-lived cache instead of - // issuing the same full scan for every sidebar page. + // Connector/J 3.6.3 ignores Statement#setMaxRows. Filtered and + // unbounded callers still need the complete result, so cache that + // scan briefly instead of repeating it for adjacent requests. result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "STABLES", "STABLE", false)); result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "TABLES", "TABLE", true)); @@ -342,20 +417,95 @@ public final class TDengineAgent extends AbstractJdbcAgent { try (java.sql.Statement stmt = requireConnected().createStatement()) { try (ResultSet rs = stmt.executeQuery(sql)) { while (rs.next()) { - // SHOW TABLES returns the owning STABLE as its fourth column. It - // is absent for ordinary tables and older servers, where this - // best-effort read simply leaves the table at the root level. - String parentName = includesStableName ? optionalString(rs, 4) : null; - if (parentName != null && parentName.trim().isEmpty()) { - parentName = null; - } - result.add(new TableInfo(rs.getString(1), tableType, null, null, parentName)); + result.add(readShowTable(rs, tableType, includesStableName)); } } } return result; } + private TablePageScan queryTablesPage( + String sql, + String tableType, + boolean includesStableName, + int offset, + int limit + ) throws Exception { + List result = new ArrayList<>(limit); + int scannedRows = 0; + try (java.sql.Statement stmt = requireConnected().createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + while (result.size() < limit && rs.next()) { + scannedRows += 1; + if (scannedRows <= offset) { + continue; + } + result.add(readShowTable(rs, tableType, includesStableName)); + } + } + return new TablePageScan(result, scannedRows); + } + + private List queryInformationSchemaTablePage(String database, int offset, int limit) throws Exception { + StringBuilder sql = new StringBuilder( + "SELECT table_name, stable_name FROM information_schema.ins_tables WHERE db_name = ? LIMIT " + ).append(limit); + if (offset > 0) { + sql.append(" OFFSET ").append(offset); + } + + List result = new ArrayList<>(limit); + try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) { + stmt.setString(1, database); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + String parentName = optionalString(rs, 2); + if (parentName != null && parentName.trim().isEmpty()) { + parentName = null; + } + result.add(new TableInfo(rs.getString(1), "TABLE", null, null, parentName)); + } + } + } + return result; + } + + private static TableInfo readShowTable(ResultSet rs, String tableType, boolean includesStableName) throws Exception { + // SHOW TABLES returns the owning STABLE as its fourth column. It + // is absent for ordinary tables and older servers, where this + // best-effort read simply leaves the table at the root level. + String parentName = includesStableName ? optionalString(rs, 4) : null; + if (parentName != null && parentName.trim().isEmpty()) { + parentName = null; + } + return new TableInfo(rs.getString(1), tableType, null, null, parentName); + } + + private record TablePageScan(List tables, int scannedRows) { + } + + private static boolean supportsRestInformationSchemaPaging(Connection connection) { + if (unwrapConnection(connection, RestfulConnection.class) == null) { + return false; + } + try { + return supportsRestInformationSchemaPagingVersion(connection.getMetaData().getDatabaseProductVersion()); + } catch (Exception ignored) { + return false; + } + } + + static boolean supportsRestInformationSchemaPagingVersion(String version) { + Matcher matcher = TDENGINE_VERSION_PATTERN.matcher(version == null ? "" : version.trim()); + if (!matcher.find()) { + return false; + } + int major = Integer.parseInt(matcher.group(1)); + int minor = Integer.parseInt(matcher.group(2)); + int patch = Integer.parseInt(matcher.group(3)); + return major > 3 || (major == 3 && (minor > 3 || (minor == 3 && patch >= 8))); + } + private List cachedTables(String schema) { String normalizedSchema = normalizedSchema(schema); synchronized (tableCacheLock) { diff --git a/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java b/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java index e1f17f56ec..8c6c489b5c 100644 --- a/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java +++ b/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java @@ -94,25 +94,50 @@ class TDengineAgentMetadataTest { } @Test - void showMetadataCachesFullResultsForLocalPaging() { + void showMetadataStopsReadingAtTheRequestedPageBoundary() { + List statements = new ArrayList<>(); + List maxRows = new ArrayList<>(); + List rowReads = new ArrayList<>(); + TDengineAgent agent = new TDengineAgent(); + TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows, rowReads)); + + List firstPage = agent.listTables( + "dbx_alpha", + new MetadataListConstraints(null, 3, null, List.of("TABLE")) + ); + List secondPage = agent.listTables( + "dbx_alpha", + new MetadataListConstraints(null, 2, 3, List.of("TABLE")) + ); + + Assertions.assertEquals(List.of("meters", "weather", "device_a"), firstPage.stream().map(TableInfo::getName).toList()); + Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList()); + Assertions.assertEquals("meters", firstPage.get(2).getParent_name()); + Assertions.assertEquals("meters", secondPage.get(0).getParent_name()); + Assertions.assertTrue(maxRows.isEmpty()); + Assertions.assertEquals( + List.of( + "SHOW `dbx_alpha`.STABLES", + "SHOW `dbx_alpha`.TABLES", + "SHOW `dbx_alpha`.STABLES", + "SHOW `dbx_alpha`.TABLES" + ), + statements + ); + Assertions.assertEquals(4, rowReads.stream().filter("STABLE"::equals).count()); + Assertions.assertEquals(4, rowReads.stream().filter("TABLE"::equals).count()); + } + + @Test + void showMetadataCachesOnlyUnboundedResults() { List statements = new ArrayList<>(); List maxRows = new ArrayList<>(); TDengineAgent agent = new TDengineAgent(); TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows)); - List firstPage = agent.listTables( - "dbx_alpha", - new MetadataListConstraints(null, 2, null, List.of("TABLE")) - ); - List secondPage = agent.listTables( - "dbx_alpha", - new MetadataListConstraints(null, 2, 2, List.of("TABLE")) - ); + agent.listTables("dbx_alpha"); + agent.listTables("dbx_alpha"); - Assertions.assertEquals(List.of("meters", "device_a"), firstPage.stream().map(TableInfo::getName).toList()); - Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList()); - Assertions.assertEquals("meters", secondPage.get(0).getParent_name()); - Assertions.assertTrue(maxRows.isEmpty()); Assertions.assertEquals( List.of( "SHOW `dbx_alpha`.STABLES", @@ -250,10 +275,18 @@ class TDengineAgentMetadataTest { } private static Connection showMetadataConnection(List statements, List maxRows) { + return showMetadataConnection(statements, maxRows, null); + } + + private static Connection showMetadataConnection( + List statements, + List maxRows, + List rowReads + ) { return proxy(Connection.class, (proxy, method, args) -> { String name = method.getName(); if ("createStatement".equals(name)) { - return showMetadataStatement(statements, maxRows); + return showMetadataStatement(statements, maxRows, rowReads); } if ("isClosed".equals(name)) return false; if ("close".equals(name)) return null; @@ -261,7 +294,11 @@ class TDengineAgentMetadataTest { }); } - private static java.sql.Statement showMetadataStatement(List statements, List maxRows) { + private static java.sql.Statement showMetadataStatement( + List statements, + List maxRows, + List rowReads + ) { int[] activeMaxRows = {0}; return proxy(java.sql.Statement.class, (proxy, method, args) -> { String name = method.getName(); @@ -274,7 +311,12 @@ class TDengineAgentMetadataTest { String sql = (String) args[0]; statements.add(sql); if (sql.endsWith("STABLES")) { - return showTableResultSet(List.of(new String[] {"meters"}, new String[] {"weather"}), activeMaxRows[0]); + return showTableResultSet( + List.of(new String[] {"meters"}, new String[] {"weather"}), + activeMaxRows[0], + "STABLE", + rowReads + ); } if (sql.endsWith("TABLES")) { return showTableResultSet( @@ -283,7 +325,9 @@ class TDengineAgentMetadataTest { new String[] {"device_b", "", "", "meters"}, new String[] {"standalone", "", "", ""} ), - activeMaxRows[0] + activeMaxRows[0], + "TABLE", + rowReads ); } } @@ -292,14 +336,23 @@ class TDengineAgentMetadataTest { }); } - private static ResultSet showTableResultSet(List rows, int maxRows) { + private static ResultSet showTableResultSet( + List rows, + int maxRows, + String tableType, + List rowReads + ) { List limitedRows = maxRows > 0 ? rows.subList(0, Math.min(rows.size(), maxRows)) : rows; int[] index = {-1}; return proxy(ResultSet.class, (proxy, method, args) -> { String name = method.getName(); if ("next".equals(name)) { index[0] += 1; - return index[0] < limitedRows.size(); + boolean hasNext = index[0] < limitedRows.size(); + if (hasNext && rowReads != null) { + rowReads.add(tableType); + } + return hasNext; } if ("getString".equals(name)) { int column = (Integer) args[0]; @@ -465,6 +518,15 @@ class TDengineAgentMetadataTest { Assertions.assertEquals("jdbc:TAOS-RS://127.0.0.1:6041/db?timezone=UTC#anchor", sanitized); } + @Test + void enablesRestInformationSchemaPagingForTdengine338AndNewer() { + Assertions.assertFalse(TDengineAgent.supportsRestInformationSchemaPagingVersion(null)); + Assertions.assertFalse(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.7.9")); + Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.8.0")); + Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.8.1")); + Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("4.0.0.0")); + } + private static ResultSet describeResultSet(String[][] rows) { int[] rowIndex = {-1}; return (ResultSet) Proxy.newProxyInstance(