fix(tdengine): avoid full scans for table pages
This commit is contained in:
2 files changed
+242
-30
No files matched your search
@@ -61,6 +61,8 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
Pattern.compile("(?i)^(binary|nchar|varchar|varbinary)\\((\\d+)\\)");
|
||||
private static final Pattern COMPOSITE_KEY_PATTERN =
|
||||
Pattern.compile("(?i)\\bCOMPOSITE\\s+KEY\\b");
|
||||
private static final Pattern TDENGINE_VERSION_PATTERN =
|
||||
Pattern.compile("^(\\d+)\\.(\\d+)\\.(\\d+)(?:\\.|$)");
|
||||
|
||||
private final Object tableCacheLock = new Object();
|
||||
private String tableCacheSchema = "";
|
||||
@@ -69,6 +71,7 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
private Connection restfulStateConnection;
|
||||
private String restfulOriginalCatalog;
|
||||
private String restfulOriginalDatabase;
|
||||
private boolean restInformationSchemaPagingEnabled;
|
||||
|
||||
@Override
|
||||
protected String driverClass() {
|
||||
@@ -101,6 +104,7 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
protected void afterConnect(ConnectParams params, Connection connection) {
|
||||
clearTableCache();
|
||||
clearRestfulState();
|
||||
restInformationSchemaPagingEnabled = supportsRestInformationSchemaPaging(connection);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -119,6 +123,7 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
protected void afterDisconnect() {
|
||||
clearTableCache();
|
||||
clearRestfulState();
|
||||
restInformationSchemaPagingEnabled = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -154,9 +159,79 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
if (!normalized.tableTypeAllowed("TABLE")) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
if (normalized.hasLimit() && !normalized.hasFilter()) {
|
||||
if (restInformationSchemaPagingEnabled) {
|
||||
try {
|
||||
return listTablesPageFromRestInformationSchema(schema, normalized);
|
||||
} catch (RuntimeException ignored) {
|
||||
}
|
||||
}
|
||||
return listTablesPageFromShow(schema, normalized);
|
||||
}
|
||||
return normalized.filterTables(listTablesFromShow(schema));
|
||||
}
|
||||
|
||||
private List<TableInfo> listTablesPageFromRestInformationSchema(
|
||||
String schema,
|
||||
MetadataListConstraints constraints
|
||||
) {
|
||||
return unchecked(() -> {
|
||||
int limit = constraints.getLimit();
|
||||
int offset = constraints.getOffset() == null ? 0 : constraints.getOffset();
|
||||
List<TableInfo> result = new ArrayList<>(limit);
|
||||
|
||||
TablePageScan stablePage = queryTablesPage(
|
||||
"SHOW " + quoteQualifiedPrefix(schema) + "STABLES",
|
||||
"STABLE",
|
||||
false,
|
||||
offset,
|
||||
limit
|
||||
);
|
||||
result.addAll(stablePage.tables());
|
||||
if (result.size() >= limit) {
|
||||
return result;
|
||||
}
|
||||
|
||||
int tableOffset = Math.max(0, offset - stablePage.scannedRows());
|
||||
result.addAll(queryInformationSchemaTablePage(
|
||||
schema,
|
||||
tableOffset,
|
||||
limit - result.size()
|
||||
));
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
private List<TableInfo> listTablesPageFromShow(String schema, MetadataListConstraints constraints) {
|
||||
return unchecked(() -> {
|
||||
int limit = constraints.getLimit();
|
||||
int offset = constraints.getOffset() == null ? 0 : constraints.getOffset();
|
||||
List<TableInfo> result = new ArrayList<>(limit);
|
||||
|
||||
TablePageScan stablePage = queryTablesPage(
|
||||
"SHOW " + quoteQualifiedPrefix(schema) + "STABLES",
|
||||
"STABLE",
|
||||
false,
|
||||
offset,
|
||||
limit
|
||||
);
|
||||
result.addAll(stablePage.tables());
|
||||
if (result.size() >= limit) {
|
||||
return result;
|
||||
}
|
||||
|
||||
int tableOffset = Math.max(0, offset - stablePage.scannedRows());
|
||||
result.addAll(queryTablesPage(
|
||||
"SHOW " + quoteQualifiedPrefix(schema) + "TABLES",
|
||||
"TABLE",
|
||||
true,
|
||||
tableOffset,
|
||||
limit - result.size()
|
||||
).tables());
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
private List<TableInfo> listTablesFromShow(String schema) {
|
||||
List<TableInfo> cached = cachedTables(schema);
|
||||
if (cached != null) {
|
||||
@@ -164,9 +239,9 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
}
|
||||
return unchecked(() -> {
|
||||
List<TableInfo> result = new ArrayList<>();
|
||||
// Connector/J 3.6.3 ignores Statement#setMaxRows. Read the SHOW
|
||||
// results once and page locally from a short-lived cache instead of
|
||||
// issuing the same full scan for every sidebar page.
|
||||
// Connector/J 3.6.3 ignores Statement#setMaxRows. Filtered and
|
||||
// unbounded callers still need the complete result, so cache that
|
||||
// scan briefly instead of repeating it for adjacent requests.
|
||||
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "STABLES", "STABLE", false));
|
||||
result.addAll(queryTables("SHOW " + quoteQualifiedPrefix(schema) + "TABLES", "TABLE", true));
|
||||
|
||||
@@ -342,20 +417,95 @@ public final class TDengineAgent extends AbstractJdbcAgent {
|
||||
try (java.sql.Statement stmt = requireConnected().createStatement()) {
|
||||
try (ResultSet rs = stmt.executeQuery(sql)) {
|
||||
while (rs.next()) {
|
||||
// SHOW TABLES returns the owning STABLE as its fourth column. It
|
||||
// is absent for ordinary tables and older servers, where this
|
||||
// best-effort read simply leaves the table at the root level.
|
||||
String parentName = includesStableName ? optionalString(rs, 4) : null;
|
||||
if (parentName != null && parentName.trim().isEmpty()) {
|
||||
parentName = null;
|
||||
}
|
||||
result.add(new TableInfo(rs.getString(1), tableType, null, null, parentName));
|
||||
result.add(readShowTable(rs, tableType, includesStableName));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private TablePageScan queryTablesPage(
|
||||
String sql,
|
||||
String tableType,
|
||||
boolean includesStableName,
|
||||
int offset,
|
||||
int limit
|
||||
) throws Exception {
|
||||
List<TableInfo> result = new ArrayList<>(limit);
|
||||
int scannedRows = 0;
|
||||
try (java.sql.Statement stmt = requireConnected().createStatement();
|
||||
ResultSet rs = stmt.executeQuery(sql)) {
|
||||
while (result.size() < limit && rs.next()) {
|
||||
scannedRows += 1;
|
||||
if (scannedRows <= offset) {
|
||||
continue;
|
||||
}
|
||||
result.add(readShowTable(rs, tableType, includesStableName));
|
||||
}
|
||||
}
|
||||
return new TablePageScan(result, scannedRows);
|
||||
}
|
||||
|
||||
private List<TableInfo> queryInformationSchemaTablePage(String database, int offset, int limit) throws Exception {
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT table_name, stable_name FROM information_schema.ins_tables WHERE db_name = ? LIMIT "
|
||||
).append(limit);
|
||||
if (offset > 0) {
|
||||
sql.append(" OFFSET ").append(offset);
|
||||
}
|
||||
|
||||
List<TableInfo> result = new ArrayList<>(limit);
|
||||
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
|
||||
stmt.setString(1, database);
|
||||
try (ResultSet rs = stmt.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
String parentName = optionalString(rs, 2);
|
||||
if (parentName != null && parentName.trim().isEmpty()) {
|
||||
parentName = null;
|
||||
}
|
||||
result.add(new TableInfo(rs.getString(1), "TABLE", null, null, parentName));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static TableInfo readShowTable(ResultSet rs, String tableType, boolean includesStableName) throws Exception {
|
||||
// SHOW TABLES returns the owning STABLE as its fourth column. It
|
||||
// is absent for ordinary tables and older servers, where this
|
||||
// best-effort read simply leaves the table at the root level.
|
||||
String parentName = includesStableName ? optionalString(rs, 4) : null;
|
||||
if (parentName != null && parentName.trim().isEmpty()) {
|
||||
parentName = null;
|
||||
}
|
||||
return new TableInfo(rs.getString(1), tableType, null, null, parentName);
|
||||
}
|
||||
|
||||
private record TablePageScan(List<TableInfo> tables, int scannedRows) {
|
||||
}
|
||||
|
||||
private static boolean supportsRestInformationSchemaPaging(Connection connection) {
|
||||
if (unwrapConnection(connection, RestfulConnection.class) == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return supportsRestInformationSchemaPagingVersion(connection.getMetaData().getDatabaseProductVersion());
|
||||
} catch (Exception ignored) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static boolean supportsRestInformationSchemaPagingVersion(String version) {
|
||||
Matcher matcher = TDENGINE_VERSION_PATTERN.matcher(version == null ? "" : version.trim());
|
||||
if (!matcher.find()) {
|
||||
return false;
|
||||
}
|
||||
int major = Integer.parseInt(matcher.group(1));
|
||||
int minor = Integer.parseInt(matcher.group(2));
|
||||
int patch = Integer.parseInt(matcher.group(3));
|
||||
return major > 3 || (major == 3 && (minor > 3 || (minor == 3 && patch >= 8)));
|
||||
}
|
||||
|
||||
private List<TableInfo> cachedTables(String schema) {
|
||||
String normalizedSchema = normalizedSchema(schema);
|
||||
synchronized (tableCacheLock) {
|
||||
|
||||
+81
-19
@@ -94,25 +94,50 @@ class TDengineAgentMetadataTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void showMetadataCachesFullResultsForLocalPaging() {
|
||||
void showMetadataStopsReadingAtTheRequestedPageBoundary() {
|
||||
List<String> statements = new ArrayList<>();
|
||||
List<Integer> maxRows = new ArrayList<>();
|
||||
List<String> rowReads = new ArrayList<>();
|
||||
TDengineAgent agent = new TDengineAgent();
|
||||
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows, rowReads));
|
||||
|
||||
List<TableInfo> firstPage = agent.listTables(
|
||||
"dbx_alpha",
|
||||
new MetadataListConstraints(null, 3, null, List.of("TABLE"))
|
||||
);
|
||||
List<TableInfo> secondPage = agent.listTables(
|
||||
"dbx_alpha",
|
||||
new MetadataListConstraints(null, 2, 3, List.of("TABLE"))
|
||||
);
|
||||
|
||||
Assertions.assertEquals(List.of("meters", "weather", "device_a"), firstPage.stream().map(TableInfo::getName).toList());
|
||||
Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList());
|
||||
Assertions.assertEquals("meters", firstPage.get(2).getParent_name());
|
||||
Assertions.assertEquals("meters", secondPage.get(0).getParent_name());
|
||||
Assertions.assertTrue(maxRows.isEmpty());
|
||||
Assertions.assertEquals(
|
||||
List.of(
|
||||
"SHOW `dbx_alpha`.STABLES",
|
||||
"SHOW `dbx_alpha`.TABLES",
|
||||
"SHOW `dbx_alpha`.STABLES",
|
||||
"SHOW `dbx_alpha`.TABLES"
|
||||
),
|
||||
statements
|
||||
);
|
||||
Assertions.assertEquals(4, rowReads.stream().filter("STABLE"::equals).count());
|
||||
Assertions.assertEquals(4, rowReads.stream().filter("TABLE"::equals).count());
|
||||
}
|
||||
|
||||
@Test
|
||||
void showMetadataCachesOnlyUnboundedResults() {
|
||||
List<String> statements = new ArrayList<>();
|
||||
List<Integer> maxRows = new ArrayList<>();
|
||||
TDengineAgent agent = new TDengineAgent();
|
||||
TestSupport.setPrivateConnection(agent, showMetadataConnection(statements, maxRows));
|
||||
|
||||
List<TableInfo> firstPage = agent.listTables(
|
||||
"dbx_alpha",
|
||||
new MetadataListConstraints(null, 2, null, List.of("TABLE"))
|
||||
);
|
||||
List<TableInfo> secondPage = agent.listTables(
|
||||
"dbx_alpha",
|
||||
new MetadataListConstraints(null, 2, 2, List.of("TABLE"))
|
||||
);
|
||||
agent.listTables("dbx_alpha");
|
||||
agent.listTables("dbx_alpha");
|
||||
|
||||
Assertions.assertEquals(List.of("meters", "device_a"), firstPage.stream().map(TableInfo::getName).toList());
|
||||
Assertions.assertEquals(List.of("device_b", "standalone"), secondPage.stream().map(TableInfo::getName).toList());
|
||||
Assertions.assertEquals("meters", secondPage.get(0).getParent_name());
|
||||
Assertions.assertTrue(maxRows.isEmpty());
|
||||
Assertions.assertEquals(
|
||||
List.of(
|
||||
"SHOW `dbx_alpha`.STABLES",
|
||||
@@ -250,10 +275,18 @@ class TDengineAgentMetadataTest {
|
||||
}
|
||||
|
||||
private static Connection showMetadataConnection(List<String> statements, List<Integer> maxRows) {
|
||||
return showMetadataConnection(statements, maxRows, null);
|
||||
}
|
||||
|
||||
private static Connection showMetadataConnection(
|
||||
List<String> statements,
|
||||
List<Integer> maxRows,
|
||||
List<String> rowReads
|
||||
) {
|
||||
return proxy(Connection.class, (proxy, method, args) -> {
|
||||
String name = method.getName();
|
||||
if ("createStatement".equals(name)) {
|
||||
return showMetadataStatement(statements, maxRows);
|
||||
return showMetadataStatement(statements, maxRows, rowReads);
|
||||
}
|
||||
if ("isClosed".equals(name)) return false;
|
||||
if ("close".equals(name)) return null;
|
||||
@@ -261,7 +294,11 @@ class TDengineAgentMetadataTest {
|
||||
});
|
||||
}
|
||||
|
||||
private static java.sql.Statement showMetadataStatement(List<String> statements, List<Integer> maxRows) {
|
||||
private static java.sql.Statement showMetadataStatement(
|
||||
List<String> statements,
|
||||
List<Integer> maxRows,
|
||||
List<String> rowReads
|
||||
) {
|
||||
int[] activeMaxRows = {0};
|
||||
return proxy(java.sql.Statement.class, (proxy, method, args) -> {
|
||||
String name = method.getName();
|
||||
@@ -274,7 +311,12 @@ class TDengineAgentMetadataTest {
|
||||
String sql = (String) args[0];
|
||||
statements.add(sql);
|
||||
if (sql.endsWith("STABLES")) {
|
||||
return showTableResultSet(List.of(new String[] {"meters"}, new String[] {"weather"}), activeMaxRows[0]);
|
||||
return showTableResultSet(
|
||||
List.of(new String[] {"meters"}, new String[] {"weather"}),
|
||||
activeMaxRows[0],
|
||||
"STABLE",
|
||||
rowReads
|
||||
);
|
||||
}
|
||||
if (sql.endsWith("TABLES")) {
|
||||
return showTableResultSet(
|
||||
@@ -283,7 +325,9 @@ class TDengineAgentMetadataTest {
|
||||
new String[] {"device_b", "", "", "meters"},
|
||||
new String[] {"standalone", "", "", ""}
|
||||
),
|
||||
activeMaxRows[0]
|
||||
activeMaxRows[0],
|
||||
"TABLE",
|
||||
rowReads
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -292,14 +336,23 @@ class TDengineAgentMetadataTest {
|
||||
});
|
||||
}
|
||||
|
||||
private static ResultSet showTableResultSet(List<String[]> rows, int maxRows) {
|
||||
private static ResultSet showTableResultSet(
|
||||
List<String[]> rows,
|
||||
int maxRows,
|
||||
String tableType,
|
||||
List<String> rowReads
|
||||
) {
|
||||
List<String[]> limitedRows = maxRows > 0 ? rows.subList(0, Math.min(rows.size(), maxRows)) : rows;
|
||||
int[] index = {-1};
|
||||
return proxy(ResultSet.class, (proxy, method, args) -> {
|
||||
String name = method.getName();
|
||||
if ("next".equals(name)) {
|
||||
index[0] += 1;
|
||||
return index[0] < limitedRows.size();
|
||||
boolean hasNext = index[0] < limitedRows.size();
|
||||
if (hasNext && rowReads != null) {
|
||||
rowReads.add(tableType);
|
||||
}
|
||||
return hasNext;
|
||||
}
|
||||
if ("getString".equals(name)) {
|
||||
int column = (Integer) args[0];
|
||||
@@ -465,6 +518,15 @@ class TDengineAgentMetadataTest {
|
||||
Assertions.assertEquals("jdbc:TAOS-RS://127.0.0.1:6041/db?timezone=UTC#anchor", sanitized);
|
||||
}
|
||||
|
||||
@Test
|
||||
void enablesRestInformationSchemaPagingForTdengine338AndNewer() {
|
||||
Assertions.assertFalse(TDengineAgent.supportsRestInformationSchemaPagingVersion(null));
|
||||
Assertions.assertFalse(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.7.9"));
|
||||
Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.8.0"));
|
||||
Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("3.3.8.1"));
|
||||
Assertions.assertTrue(TDengineAgent.supportsRestInformationSchemaPagingVersion("4.0.0.0"));
|
||||
}
|
||||
|
||||
private static ResultSet describeResultSet(String[][] rows) {
|
||||
int[] rowIndex = {-1};
|
||||
return (ResultSet) Proxy.newProxyInstance(
|
||||
|
||||
Reference in new issue
Block a user