fix(tdengine): avoid full scans for table pages

This commit is contained in:
t8y2 committed 2026-08-01 19:41:41 +08:00
1 parent 165ad02375
commit 1bd223074c
2 files changed
+242 -30

No files matched your search

@@ -61,6 +61,8 @@ public final class TDengineAgent extends AbstractJdbcAgent {
Pattern.compile("(?i)^(binary|nchar|varchar|varbinary)\\((\\d+)\\)");
private static final Pattern COMPOSITE_KEY_PATTERN =
Pattern.compile("(?i)\\bCOMPOSITE\\s+KEY\\b");
private static final Pattern TDENGINE_VERSION_PATTERN =
Pattern.compile("^(\\d+)\\.(\\d+)\\.(\\d+)(?:\\.|$)");
private final Object tableCacheLock = new Object();
private String tableCacheSchema = "";
@@ -69,6 +71,7 @@ public final class TDengineAgent extends AbstractJdbcAgent {
private Connection restfulStateConnection;
private String restfulOriginalCatalog;
private String restfulOriginalDatabase;
private boolean restInformationSchemaPagingEnabled;
@Override
protected String driverClass() {
@@ -101,6 +104,7 @@ public final class TDengineAgent extends AbstractJdbcAgent {
protected void afterConnect(ConnectParams params, Connection connection) {
clearTableCache();
clearRestfulState();
restInformationSchemaPagingEnabled = supportsRestInformationSchemaPaging(connection);
}
@Override
@@ -119,6 +123,7 @@ public final class TDengineAgent extends AbstractJdbcAgent {
protected void afterDisconnect() {
clearTableCache();
clearRestfulState();
restInformationSchemaPagingEnabled = false;
}
@Override
@@ -154,9 +159,79 @@ public final class TDengineAgent extends AbstractJdbcAgent {
if (!normalized.tableTypeAllowed("TABLE")) {
return Collections.emptyList();
}
if (normalized.hasLimit() && !normalized.hasFilter()) {
if (restInformationSchemaPagingEnabled) {
try {
return listTablesPageFromRestInformationSchema(schema, normalized);
} catch (RuntimeException ignored) {
}
}
return listTablesPageFromShow(schema, normalized);
}
return normalized.filterTables(listTablesFromShow(schema));
}
private List<TableInfo> listTablesPageFromRestInformationSchema(
String schema,
MetadataListConstraints constraints
) {
return unchecked(() -> {
int limit = constraints.getLimit();
int offset = constraints.getOffset() == null ? 0 : constraints.getOffset();
List<TableInfo> result = new ArrayList<>(limit);
TablePageScan stablePage = queryTablesPage(
"SHOW " + quoteQualifiedPrefix(schema) + "STABLES",
"STABLE",
false,
offset,
limit
);
result.addAll(stablePage.tables());
if (result.size() >= limit) {
return result;
}
int tableOffset = Math.max(0, offset - stablePage.scannedRows());
result.addAll(queryInformationSchemaTablePage(
schema,
tableOffset,
limit - result.size()
));
return result;
});
}
private List<TableInfo> listTablesPageFromShow(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
int limit = constraints.getLimit();
int offset = constraints.getOffset() == null ? 0 : constraints.getOffset();
List<TableInfo> result = new ArrayList<>(limit);
TablePageScan stablePage = queryTablesPage(
"SHOW " + quoteQualifiedPrefix(schema) + "STABLES",
"STABLE",
false,
offset,
limit
);
result.addAll(stablePage.tables());
if (result.size() >= limit) {
return result;
}
int tableOffset = Math.max(0, offset - stablePage.scannedRows());
result.addAll(queryTablesPage(
"SHOW " + quoteQualifiedPrefix(schema) + "TABLES",
"TABLE",
true,
tableOffset,
limit - result.size()
).tables());
return result;
});
}
private List<TableInfo> listTablesFromShow(String schema) {
List<TableInfo> cached = cachedTables(schema);
if (cached != null) {
@@ -164,9 +239,9 @@ public final class TDengineAgent extends AbstractJdbcAgent {
}
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
// Connector/J 3.6.3 ignores Statement#setMaxRows. Read the SHOW
// results once and page locally from a short-lived cache instead of
// issuing the same full scan for every sidebar page.
// Connector/J 3.6.3 ignores Statement#setMaxRows. Filtered and
// unbounded callers still need the complete result, so cache that
// scan briefly instead of repeating it for adjacent requests.
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "STABLES", "STABLE", false));
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "TABLES", "TABLE", true));
@@ -342,20 +417,95 @@ public final class TDengineAgent extends AbstractJdbcAgent {
try (java.sql.Statement stmt = requireConnected().createStatement()) {
try (ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) {
// SHOW TABLES returns the owning STABLE as its fourth column. It
// is absent for ordinary tables and older servers, where this
// best-effort read simply leaves the table at the root level.
String parentName = includesStableName ? optionalString(rs, 4) : null;
if (parentName != null && parentName.trim().isEmpty()) {
parentName = null;
}
result.add(new TableInfo(rs.getString(1), tableType, null, null, parentName));
result.add(readShowTable(rs, tableType, includesStableName));
}
}
}
return result;
}
private TablePageScan queryTablesPage(
String sql,
String tableType,
boolean includesStableName,
int offset,
int limit
) throws Exception {
List<TableInfo> result = new ArrayList<>(limit);
int scannedRows = 0;
try (java.sql.Statement stmt = requireConnected().createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
while (result.size() < limit && rs.next()) {
scannedRows += 1;
if (scannedRows <= offset) {
continue;
}
result.add(readShowTable(rs, tableType, includesStableName));
}
}
return new TablePageScan(result, scannedRows);
}
private List<TableInfo> queryInformationSchemaTablePage(String database, int offset, int limit) throws Exception {
StringBuilder sql = new StringBuilder(
"SELECT table_name, stable_name FROM information_schema.ins_tables WHERE db_name = ? LIMIT "
).append(limit);
if (offset > 0) {
sql.append(" OFFSET ").append(offset);
}
List<TableInfo> result = new ArrayList<>(limit);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
stmt.setString(1, database);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
String parentName = optionalString(rs, 2);
if (parentName != null && parentName.trim().isEmpty()) {
parentName = null;
}
result.add(new TableInfo(rs.getString(1), "TABLE", null, null, parentName));
}
}
}
return result;
}
private static TableInfo readShowTable(ResultSet rs, String tableType, boolean includesStableName) throws Exception {
// SHOW TABLES returns the owning STABLE as its fourth column. It
// is absent for ordinary tables and older servers, where this
// best-effort read simply leaves the table at the root level.
String parentName = includesStableName ? optionalString(rs, 4) : null;
if (parentName != null && parentName.trim().isEmpty()) {
parentName = null;
}
return new TableInfo(rs.getString(1), tableType, null, null, parentName);
}
private record TablePageScan(List<TableInfo> tables, int scannedRows) {
}
private static boolean supportsRestInformationSchemaPaging(Connection connection) {
if (unwrapConnection(connection, RestfulConnection.class) == null) {
return false;
}
try {
return supportsRestInformationSchemaPagingVersion(connection.getMetaData().getDatabaseProductVersion());
} catch (Exception ignored) {
return false;
}
}
static boolean supportsRestInformationSchemaPagingVersion(String version) {
Matcher matcher = TDENGINE_VERSION_PATTERN.matcher(version == null ? "" : version.trim());
if (!matcher.find()) {
return false;
}
int major = Integer.parseInt(matcher.group(1));
int minor = Integer.parseInt(matcher.group(2));
int patch = Integer.parseInt(matcher.group(3));
return major > 3 || (major == 3 && (minor > 3 || (minor == 3 && patch >= 8)));
}
private List<TableInfo> cachedTables(String schema) {
String normalizedSchema = normalizedSchema(schema);
synchronized (tableCacheLock) {
@@ -94,25 +94,50 @@ class TDengineAgentMetadataTest {
}
@Test
void showMetadataCachesFullResultsForLocalPaging() {
void showMetadataStopsReadingAtTheRequestedPageBoundary() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
List<String> rowReads = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows, rowReads));
List<TableInfo> firstPage = agent.listTables(
"dbx_alpha",
new MetadataListConstraints(null, 3, null, List.of("TABLE"))
);
List<TableInfo> secondPage = agent.listTables(
"dbx_alpha",
new MetadataListConstraints(null, 2, 3, List.of("TABLE"))
);
Assertions.assertEquals(List.of("meters", "weather", "device_a"), firstPage.stream().map(TableInfo::getName).toList());
Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList());
Assertions.assertEquals("meters", firstPage.get(2).getParent_name());
Assertions.assertEquals("meters", secondPage.get(0).getParent_name());
Assertions.assertTrue(maxRows.isEmpty());
Assertions.assertEquals(
List.of(
"SHOW `dbx_alpha`.STABLES",
"SHOW `dbx_alpha`.TABLES",
"SHOW `dbx_alpha`.STABLES",
"SHOW `dbx_alpha`.TABLES"
),
statements
);
Assertions.assertEquals(4, rowReads.stream().filter("STABLE"::equals).count());
Assertions.assertEquals(4, rowReads.stream().filter("TABLE"::equals).count());
}
@Test
void showMetadataCachesOnlyUnboundedResults() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
List<TableInfo> firstPage = agent.listTables(
"dbx_alpha",
new MetadataListConstraints(null, 2, null, List.of("TABLE"))
);
List<TableInfo> secondPage = agent.listTables(
"dbx_alpha",
new MetadataListConstraints(null, 2, 2, List.of("TABLE"))
);
agent.listTables("dbx_alpha");
agent.listTables("dbx_alpha");
Assertions.assertEquals(List.of("meters", "device_a"), firstPage.stream().map(TableInfo::getName).toList());
Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList());
Assertions.assertEquals("meters", secondPage.get(0).getParent_name());
Assertions.assertTrue(maxRows.isEmpty());
Assertions.assertEquals(
List.of(
"SHOW `dbx_alpha`.STABLES",
@@ -250,10 +275,18 @@ class TDengineAgentMetadataTest {
}
private static Connection showMetadataConnection(List<String> statements, List<Integer> maxRows) {
return showMetadataConnection(statements, maxRows, null);
}
private static Connection showMetadataConnection(
List<String> statements,
List<Integer> maxRows,
List<String> rowReads
) {
return proxy(Connection.class, (proxy, method, args) -> {
String name = method.getName();
if ("createStatement".equals(name)) {
return showMetadataStatement(statements, maxRows);
return showMetadataStatement(statements, maxRows, rowReads);
}
if ("isClosed".equals(name)) return false;
if ("close".equals(name)) return null;
@@ -261,7 +294,11 @@ class TDengineAgentMetadataTest {
});
}
private static java.sql.Statement showMetadataStatement(List<String> statements, List<Integer> maxRows) {
private static java.sql.Statement showMetadataStatement(
List<String> statements,
List<Integer> maxRows,
List<String> rowReads
) {
int[] activeMaxRows = {0};
return proxy(java.sql.Statement.class, (proxy, method, args) -> {
String name = method.getName();
@@ -274,7 +311,12 @@ class TDengineAgentMetadataTest {
String sql = (String) args[0];
statements.add(sql);
if (sql.endsWith("STABLES")) {
return showTableResultSet(List.of(new String[] {"meters"}, new String[] {"weather"}), activeMaxRows[0]);
return showTableResultSet(
List.of(new String[] {"meters"}, new String[] {"weather"}),
activeMaxRows[0],
"STABLE",
rowReads
);
}
if (sql.endsWith("TABLES")) {
return showTableResultSet(
@@ -283,7 +325,9 @@ class TDengineAgentMetadataTest {
new String[] {"device_b", "", "", "meters"},
new String[] {"standalone", "", "", ""}
),
activeMaxRows[0]
activeMaxRows[0],
"TABLE",
rowReads
);
}
}
@@ -292,14 +336,23 @@ class TDengineAgentMetadataTest {
});
}
private static ResultSet showTableResultSet(List<String[]> rows, int maxRows) {
private static ResultSet showTableResultSet(
List<String[]> rows,
int maxRows,
String tableType,
List<String> rowReads
) {
List<String[]> limitedRows = maxRows > 0 ? rows.subList(0, Math.min(rows.size(), maxRows)) : rows;
int[] index = {-1};
return proxy(ResultSet.class, (proxy, method, args) -> {
String name = method.getName();
if ("next".equals(name)) {
index[0] += 1;
return index[0] < limitedRows.size();
boolean hasNext = index[0] < limitedRows.size();
if (hasNext && rowReads != null) {
rowReads.add(tableType);
}
return hasNext;
}
if ("getString".equals(name)) {
int column = (Integer) args[0];
@@ -465,6 +518,15 @@ class TDengineAgentMetadataTest {
Assertions.assertEquals("jdbc:TAOS-RS://127.0.0.1:6041/db?timezone=UTC#anchor", sanitized);
}
@Test
void enablesRestInformationSchemaPagingForTdengine338AndNewer() {
Assertions.assertFalse(TDengineAgent.supportsRestInformationSchemaPagingVersion(null));
Assertions.assertFalse(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.7.9"));
Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.8.0"));
Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.8.1"));
Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("4.0.0.0"));
}
private static ResultSet describeResultSet(String[][] rows) {
int[] rowIndex = {-1};
return (ResultSet) Proxy.newProxyInstance(