feat(tdengine): optimize metadata comment search

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-07-29 01:44:28 +08:00
1 parent 2cbbaaef4c
commit 4a5eb3f04a
4 files changed
+802 -195

No files matched your search

@@ -10,7 +10,6 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.MultiSessionJsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource;
import com.dbx.agent.QueryPageOptions;
@@ -48,6 +47,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public final class TDengineAgent extends BaseDatabaseAgent {
private static final long TABLE_CACHE_TTL_MILLIS = 10_000L;
private static final DateTimeFormatter TDENGINE_TIMESTAMP_FORMAT =
new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd HH:mm:ss")
@@ -63,6 +63,10 @@ public final class TDengineAgent extends BaseDatabaseAgent {
Pattern.compile("(?i)\\bCOMPOSITE\\s+KEY\\b");
private Connection connection;
private final Object tableCacheLock = new Object();
private String tableCacheSchema = "";
private long tableCacheTimeMillis;
private List<TableInfo> tableCache = Collections.emptyList();
@Override
public Connection getConnection() {
@@ -73,6 +77,7 @@ public final class TDengineAgent extends BaseDatabaseAgent {
public void connect(ConnectParams params) {
uncheckedVoid(() -> {
connection = TDengineConnectionFactory.open(params);
clearTableCache();
});
}
@@ -118,19 +123,21 @@ public final class TDengineAgent extends BaseDatabaseAgent {
if (!normalized.tableTypeAllowed("TABLE")) {
return Collections.emptyList();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException ignored) {
// TDengine 2.x does not expose information_schema.ins_stables/ins_tables.
return normalized.filterTables(listTablesLegacy(schema));
}
return normalized.filterTables(listTablesFromShow(schema));
}
private List<TableInfo> listTablesLegacy(String schema) {
private List<TableInfo> listTablesFromShow(String schema) {
List<TableInfo> cached = cachedTables(schema);
if (cached != null) {
return cached;
}
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "STABLES", "STABLE"));
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "TABLES", "TABLE"));
// Connector/J 3.6.3 ignores Statement#setMaxRows. Read the SHOW
// results once and page locally from a short-lived cache instead of
// issuing the same full scan for every sidebar page.
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "STABLES", "STABLE", false));
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "TABLES", "TABLE", true));
Map<String, TableInfo> distinct = new LinkedHashMap<>();
for (TableInfo table : result) {
@@ -138,7 +145,8 @@ public final class TDengineAgent extends BaseDatabaseAgent {
}
List<TableInfo> sorted = new ArrayList<>(distinct.values());
sortTablesForHierarchy(sorted);
return sorted;
cacheTables(schema, sorted);
return copyTables(sorted);
});
}
@@ -199,7 +207,7 @@ public final class TDengineAgent extends BaseDatabaseAgent {
@Override
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
return JdbcExecutor.current().execute(
QueryResult result = JdbcExecutor.current().execute(
requireConnected(),
sql,
prepareExecutionSchema(schema),
@@ -209,6 +217,10 @@ public final class TDengineAgent extends BaseDatabaseAgent {
options.getTimeoutSecs(),
this::tdengineResultValue
);
if (mayChangeMetadata(sql)) {
clearTableCache();
}
return result;
}
@Override
@@ -242,12 +254,20 @@ public final class TDengineAgent extends BaseDatabaseAgent {
@Override
public QueryResult executeTransaction(List<String> statements, String schema) {
return super.executeTransaction(statements, prepareExecutionSchema(schema));
QueryResult result = super.executeTransaction(statements, prepareExecutionSchema(schema));
if (statements.stream().anyMatch(TDengineAgent::mayChangeMetadata)) {
clearTableCache();
}
return result;
}
@Override
public QueryResult executeBatch(List<String> statements, String schema) {
return super.executeBatch(statements, prepareExecutionSchema(schema));
QueryResult result = super.executeBatch(statements, prepareExecutionSchema(schema));
if (statements.stream().anyMatch(TDengineAgent::mayChangeMetadata)) {
clearTableCache();
}
return result;
}
private String prepareExecutionSchema(String schema) {
@@ -273,98 +293,84 @@ public final class TDengineAgent extends BaseDatabaseAgent {
connection.close();
}
connection = null;
clearTableCache();
});
}
private List<TableInfo> queryTables(String sql, String tableType) throws Exception {
private List<TableInfo> queryTables(String sql, String tableType, boolean includesStableName) throws Exception {
List<TableInfo> result = new ArrayList<>();
try (java.sql.Statement stmt = requireConnected().createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) {
result.add(new TableInfo(rs.getString(1), tableType, null));
try (java.sql.Statement stmt = requireConnected().createStatement()) {
try (ResultSet rs = stmt.executeQuery(sql)) {
while (rs.next()) {
// SHOW TABLES returns the owning STABLE as its fourth column. It
// is absent for ordinary tables and older servers, where this
// best-effort read simply leaves the table at the root level.
String parentName = includesStableName ? optionalString(rs, 4) : null;
if (parentName != null && parentName.trim().isEmpty()) {
parentName = null;
}
result.add(new TableInfo(rs.getString(1), tableType, null, null, parentName));
}
}
}
return result;
}
private List<TableInfo> queryConstrainedTables(String database, MetadataListConstraints constraints) {
return unchecked(() -> {
TableMetadataQuery query = buildTableMetadataQuery(database, constraints);
List<TableInfo> result = new ArrayList<>();
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(query.sql())) {
MetadataSqlSupport.bind(stmt, query.args());
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString(1),
rs.getString(2),
optionalString(rs, 3),
null,
optionalString(rs, 4)
));
}
}
private List<TableInfo> cachedTables(String schema) {
String normalizedSchema = normalizedSchema(schema);
synchronized (tableCacheLock) {
if (cacheFresh(tableCacheTimeMillis) && tableCacheSchema.equals(normalizedSchema)) {
return copyTables(tableCache);
}
MetadataListConstraints postFilter = constraints.hasLimit() ? constraints.withoutPaging() : constraints;
return postFilter.filterTables(result);
});
}
return null;
}
static TableMetadataQuery buildTableMetadataQuery(String database, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("""
SELECT table_name, table_type, table_comment, parent_name
FROM (
SELECT stable_name AS table_name,
'STABLE' AS table_type,
table_comment,
CAST(NULL AS VARCHAR(192)) AS parent_name,
stable_name AS hierarchy_name,
0 AS hierarchy_rank
FROM information_schema.ins_stables
WHERE db_name = ?
UNION ALL
SELECT table_name,
'TABLE' AS table_type,
table_comment,
stable_name AS parent_name,
CASE WHEN stable_name IS NULL THEN table_name ELSE stable_name END AS hierarchy_name,
1 AS hierarchy_rank
FROM information_schema.ins_tables
WHERE db_name = ?
) metadata
WHERE 1 = 1
""".stripIndent().trim());
args.add(database);
args.add(database);
if (normalized.hasFilter()) {
String pattern = normalized.fuzzyLikePattern();
sql.append(" AND (table_name LIKE ? OR table_comment LIKE ?)");
args.add(pattern);
args.add(pattern);
private void cacheTables(String schema, List<TableInfo> tables) {
synchronized (tableCacheLock) {
tableCacheSchema = normalizedSchema(schema);
tableCache = copyTables(tables);
tableCacheTimeMillis = System.currentTimeMillis();
}
sql.append(" ORDER BY hierarchy_name, hierarchy_rank, table_name");
MetadataSqlSupport.appendLiteralLimitOffset(sql, normalized);
return new TableMetadataQuery(sql.toString(), args);
}
static final class TableMetadataQuery {
private final String sql;
private final List<Object> args;
TableMetadataQuery(String sql, List<Object> args) {
this.sql = sql;
this.args = args;
private void clearTableCache() {
synchronized (tableCacheLock) {
tableCacheSchema = "";
tableCacheTimeMillis = 0L;
tableCache = Collections.emptyList();
}
}
String sql() {
return sql;
}
private static boolean cacheFresh(long cachedAtMillis) {
return cachedAtMillis > 0L && System.currentTimeMillis() - cachedAtMillis <= TABLE_CACHE_TTL_MILLIS;
}
List<Object> args() {
return args;
private static String normalizedSchema(String schema) {
return schema == null ? "" : schema.trim();
}
private static List<TableInfo> copyTables(List<TableInfo> tables) {
List<TableInfo> copies = new ArrayList<>(tables.size());
for (TableInfo table : tables) {
copies.add(new TableInfo(
table.getName(),
table.getTable_type(),
table.getComment(),
table.getParent_schema(),
table.getParent_name()
));
}
return copies;
}
private static boolean mayChangeMetadata(String sql) {
String normalized = sql == null ? "" : sql.trim().toLowerCase(Locale.ROOT);
return normalized.startsWith("create ")
|| normalized.startsWith("drop ")
|| normalized.startsWith("alter ")
|| normalized.startsWith("rename ")
|| normalized.startsWith("truncate ");
}
static void sortTablesForHierarchy(List<TableInfo> tables) {
@@ -16,17 +16,12 @@ import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -80,7 +75,7 @@ class TDengineAgentMetadataTest {
}
@Test
void usesTdengineMetadataStatements() {
void usesTdengineShowStatementsForMetadata() {
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
@@ -89,66 +84,119 @@ class TDengineAgentMetadataTest {
agent.getColumns("power", "meters");
Assertions.assertEquals("SHOW DATABASES", JdbcMetadataSqlFake.statements.get(0));
Assertions.assertTrue(JdbcMetadataSqlFake.statements.get(1).contains("FROM information_schema.ins_stables"));
Assertions.assertTrue(JdbcMetadataSqlFake.statements.get(1).contains("FROM information_schema.ins_tables"));
Assertions.assertEquals("param:1=power", JdbcMetadataSqlFake.statements.get(2));
Assertions.assertEquals("param:2=power", JdbcMetadataSqlFake.statements.get(3));
Assertions.assertEquals("DESCRIBE `power`.`meters`", JdbcMetadataSqlFake.statements.get(4));
Assertions.assertEquals("SHOW `power`.STABLES", JdbcMetadataSqlFake.statements.get(1));
Assertions.assertEquals("SHOW `power`.TABLES", JdbcMetadataSqlFake.statements.get(2));
Assertions.assertFalse(JdbcMetadataSqlFake.statements.stream().anyMatch(sql -> sql.contains("information_schema")));
Assertions.assertTrue(JdbcMetadataSqlFake.statements.contains("DESCRIBE `power`.`meters`"));
}
@Test
void constrainedMetadataUsesRequestedDatabaseAndPushesFilterAndPaging() {
void showMetadataCachesFullResultsForLocalPaging() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
List<TableInfo> firstPage = agent.listTables(
"dbx_alpha",
new MetadataListConstraints(null, 2, null, List.of("TABLE"))
);
List<TableInfo> secondPage = agent.listTables(
"dbx_alpha",
new MetadataListConstraints(null, 2, 2, List.of("TABLE"))
);
Assertions.assertEquals(List.of("meters", "device_a"), firstPage.stream().map(TableInfo::getName).toList());
Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList());
Assertions.assertEquals("meters", secondPage.get(0).getParent_name());
Assertions.assertTrue(maxRows.isEmpty());
Assertions.assertEquals(
List.of(
"SHOW `dbx_alpha`.STABLES",
"SHOW `dbx_alpha`.TABLES"
),
statements
);
}
@Test
void showMetadataFiltersLocallyBeforeApplyingRowLimit() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
List<TableInfo> tables = agent.listTables(
"dbx_alpha",
new MetadataListConstraints("stand", 1, null, List.of("TABLE"))
);
Assertions.assertEquals(List.of("standalone"), tables.stream().map(TableInfo::getName).toList());
Assertions.assertTrue(maxRows.isEmpty());
Assertions.assertEquals(
List.of(
"SHOW `dbx_alpha`.STABLES",
"SHOW `dbx_alpha`.TABLES"
),
statements
);
}
@Test
void showMetadataFallsBackToLocalFilteringForLongLikePatterns() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
agent.listTables(
"dbx_alpha",
new MetadataListConstraints("ord", 25, 50, List.of("TABLE"))
new MetadataListConstraints("a".repeat(50), 1, null, List.of("TABLE"))
);
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertFalse(sql.contains("DATABASE()"), sql);
Assertions.assertTrue(sql.contains("FROM information_schema.ins_stables"), sql);
Assertions.assertTrue(sql.contains("FROM information_schema.ins_tables"), sql);
Assertions.assertTrue(sql.contains("WHERE db_name = ?"), sql);
Assertions.assertTrue(sql.contains("table_name LIKE ? OR table_comment LIKE ?"), sql);
Assertions.assertTrue(sql.contains("ORDER BY hierarchy_name, hierarchy_rank, table_name"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 25 OFFSET 50"), sql);
Assertions.assertTrue(maxRows.isEmpty());
Assertions.assertEquals(
List.of(
"param:1=dbx_alpha",
"param:2=dbx_alpha",
"param:3=%o%r%d%",
"param:4=%o%r%d%"
"SHOW `dbx_alpha`.STABLES",
"SHOW `dbx_alpha`.TABLES"
),
JdbcMetadataSqlFake.statements.subList(1, 5)
statements
);
}
@Test
void constrainedMetadataPagesDoNotRepeatOrSkipTables() {
List<TableInfo> metadata = List.of(
new TableInfo("meters", "STABLE", null),
new TableInfo("device_a", "TABLE", null, null, "meters"),
new TableInfo("device_b", "TABLE", null, null, "meters"),
new TableInfo("standalone", "TABLE", null),
new TableInfo("weather", "STABLE", null)
);
void showMetadataDoesNotLoadTableComments() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, pagedMetadataConnection(metadata, statements));
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
List<TableInfo> combined = new ArrayList<>();
combined.addAll(agent.listTables("dbx_alpha", new MetadataListConstraints(null, 2, null, List.of("TABLE"))));
combined.addAll(agent.listTables("dbx_alpha", new MetadataListConstraints(null, 2, 2, List.of("TABLE"))));
combined.addAll(agent.listTables("dbx_alpha", new MetadataListConstraints(null, 2, 4, List.of("TABLE"))));
List<TableInfo> tables = agent.listTables("dbx_alpha");
Assertions.assertEquals(metadata.stream().map(TableInfo::getName).toList(), combined.stream().map(TableInfo::getName).toList());
Set<String> distinctNames = new HashSet<>(combined.stream().map(TableInfo::getName).toList());
Assertions.assertEquals(metadata.size(), distinctNames.size());
Assertions.assertTrue(statements.get(0).endsWith("LIMIT 2"), statements.get(0));
Assertions.assertTrue(statements.get(1).endsWith("LIMIT 2 OFFSET 2"), statements.get(1));
Assertions.assertTrue(statements.get(2).endsWith("LIMIT 2 OFFSET 4"), statements.get(2));
Assertions.assertNull(tables.get(0).getComment());
Assertions.assertEquals(
List.of(
"SHOW `dbx_alpha`.STABLES",
"SHOW `dbx_alpha`.TABLES"
),
statements
);
}
@Test
void showMetadataFiltersByNameWithoutInformationSchema() {
List<String> statements = new ArrayList<>();
List<Integer> maxRows = new ArrayList<>();
TDengineAgent agent = new TDengineAgent();
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
List<TableInfo> tables = agent.listTables(
"dbx_alpha",
new MetadataListConstraints("device_b", 1, null, List.of("TABLE"))
);
Assertions.assertEquals(List.of("device_b"), tables.stream().map(TableInfo::getName).toList());
Assertions.assertTrue(maxRows.isEmpty());
}
@Test
@@ -198,13 +246,11 @@ class TDengineAgentMetadataTest {
Assertions.assertTrue(JdbcMetadataSqlFake.statements.isEmpty());
}
private static Connection pagedMetadataConnection(List<TableInfo> metadata, List<String> statements) {
private static Connection showMetadataConnection(List<String> statements, List<Integer> maxRows) {
return proxy(Connection.class, (proxy, method, args) -> {
String name = method.getName();
if ("prepareStatement".equals(name)) {
String sql = (String) args[0];
statements.add(sql);
return pagedMetadataStatement(sql, metadata);
if ("createStatement".equals(name)) {
return showMetadataStatement(statements, maxRows);
}
if ("isClosed".equals(name)) return false;
if ("close".equals(name)) return null;
@@ -212,45 +258,55 @@ class TDengineAgentMetadataTest {
});
}
private static PreparedStatement pagedMetadataStatement(String sql, List<TableInfo> metadata) {
return proxy(PreparedStatement.class, (proxy, method, args) -> {
private static java.sql.Statement showMetadataStatement(List<String> statements, List<Integer> maxRows) {
int[] activeMaxRows = {0};
return proxy(java.sql.Statement.class, (proxy, method, args) -> {
String name = method.getName();
if ("setMaxRows".equals(name)) {
activeMaxRows[0] = (Integer) args[0];
maxRows.add(activeMaxRows[0]);
return null;
}
if ("executeQuery".equals(name)) {
int limit = sqlClauseValue(sql, "LIMIT", metadata.size());
int offset = sqlClauseValue(sql, "OFFSET", 0);
int from = Math.min(offset, metadata.size());
int to = Math.min(from + limit, metadata.size());
return tableInfoResultSet(metadata.subList(from, to));
}
if ("setString".equals(name) || "setInt".equals(name) || "setObject".equals(name) || "close".equals(name)) return null;
return defaultValue(method.getReturnType());
});
}
private static ResultSet tableInfoResultSet(List<TableInfo> tables) {
int[] index = {-1};
return proxy(ResultSet.class, (proxy, method, args) -> {
String name = method.getName();
if ("next".equals(name)) {
index[0] += 1;
return index[0] < tables.size();
}
if ("getString".equals(name)) {
TableInfo table = tables.get(index[0]);
int column = (Integer) args[0];
if (column == 1) return table.getName();
if (column == 2) return table.getTable_type();
if (column == 3) return table.getComment();
if (column == 4) return table.getParent_name();
String sql = (String) args[0];
statements.add(sql);
if (sql.endsWith("STABLES")) {
return showTableResultSet(List.of(new String[] {"meters"}, new String[] {"weather"}), activeMaxRows[0]);
}
if (sql.endsWith("TABLES")) {
return showTableResultSet(
List.of(
new String[] {"device_a", "", "", "meters"},
new String[] {"device_b", "", "", "meters"},
new String[] {"standalone", "", "", ""}
),
activeMaxRows[0]
);
}
}
if ("close".equals(name)) return null;
return defaultValue(method.getReturnType());
});
}
private static int sqlClauseValue(String sql, String clause, int fallback) {
Matcher matcher = Pattern.compile("\\b" + clause + "\\s+(\\d+)").matcher(sql);
return matcher.find() ? Integer.parseInt(matcher.group(1)) : fallback;
private static ResultSet showTableResultSet(List<String[]> rows, int maxRows) {
List<String[]> limitedRows = maxRows > 0 ? rows.subList(0, Math.min(rows.size(), maxRows)) : rows;
int[] index = {-1};
return proxy(ResultSet.class, (proxy, method, args) -> {
String name = method.getName();
if ("next".equals(name)) {
index[0] += 1;
return index[0] < limitedRows.size();
}
if ("getString".equals(name)) {
int column = (Integer) args[0];
String[] row = limitedRows.get(index[0]);
return column <= row.length ? row[column - 1] : null;
}
if ("isClosed".equals(name)) return false;
if ("close".equals(name)) return null;
return defaultValue(method.getReturnType());
});
}
private static <T> T proxy(Class<T> type, InvocationHandler handler) {