github/dbx
4
0 Fork 0
mirror of https://github.com/t8y2/dbx.git synced 2026-11-15 14:52:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(metadata): optimize agent metadata loading

Browse Source
This commit is contained in:
t8y2 committed 2026-07-05 12:09:24 +08:00
1 parent 8883a6b3b7
commit e74782f651
65 files changed
+6192 -571
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

agents/common/src/main/java/com/dbx/agent/ConfiguredJdbcAgent.java
+24
View File
@@ -55,11 +55,35 @@ public abstract class ConfiguredJdbcAgent extends AbstractJdbcAgent {
return StandardJdbcMetadata.INSTANCE.listTables(requireConnection(), profile, configuredDatabase, schema);
}
@Override
public List<TableInfo> listTables(String schema, List<String> objectTypes) {
return listTables(schema, new MetadataListConstraints(null, null, null, objectTypes));
}
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
return StandardJdbcMetadata.INSTANCE.listTables(
requireConnection(),
profile,
configuredDatabase,
schema,
constraints
);
}
@Override
public List<ObjectInfo> listObjects(String schema) {
return StandardJdbcMetadata.INSTANCE.listObjects(listTables(schema), schema);
}
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
MetadataListConstraints tableConstraints =
new MetadataListConstraints(normalized.getFilter(), null, null, normalized.getObjectTypes());
return StandardJdbcMetadata.INSTANCE.listObjects(listTables(schema, tableConstraints), schema, normalized);
}
@Override
public List<String> listDataTypes() {
return StandardJdbcMetadata.INSTANCE.listDataTypes(requireConnection());
agents/common/src/main/java/com/dbx/agent/DatabaseAgent.java
+10
View File
@@ -31,6 +31,12 @@ public interface DatabaseAgent {
return listTables(schema);
}
default List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
// Keep old driver overrides for object type filtering, then apply the new optional constraints locally.
return normalized.filterTables(listTables(schema, normalized.getObjectTypes()));
}
default List<ObjectInfo> listObjects(String schema) {
List<ObjectInfo> result = new ArrayList<>();
for (TableInfo table : listTables(schema)) {
@@ -39,6 +45,10 @@ public interface DatabaseAgent {
return result;
}
default List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
return MetadataListConstraints.orNone(constraints).filterObjects(listObjects(schema));
}
default List<String> listDataTypes() {
return Collections.emptyList();
}
agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java
+11 -2
View File
@@ -124,11 +124,11 @@ public final class JsonRpcServer {
}
if (AgentProtocol.METHOD_LIST_TABLES.equals(method)) {
switchCatalog(params);
return agent.listTables(params.get("schema").getAsString(), stringListOrNull(params, "object_types"));
return agent.listTables(params.get("schema").getAsString(), metadataListConstraints(params));
}
if (AgentProtocol.METHOD_LIST_OBJECTS.equals(method)) {
switchCatalog(params);
return agent.listObjects(params.get("schema").getAsString());
return agent.listObjects(params.get("schema").getAsString(), metadataListConstraints(params));
}
if (AgentProtocol.METHOD_LIST_DATA_TYPES.equals(method)) {
switchCatalog(params);
@@ -356,6 +356,15 @@ public final class JsonRpcServer {
return element.getAsInt();
}
private MetadataListConstraints metadataListConstraints(JsonObject params) {
return new MetadataListConstraints(
stringOrNull(params, "filter"),
intOrNull(params, "limit"),
intOrNull(params, "offset"),
stringListOrNull(params, "object_types")
);
}
private static int intOrDefault(JsonObject object, String key, int defaultValue) {
Integer value = intOrNull(object, key);
return value == null ? defaultValue : value;
agents/common/src/main/java/com/dbx/agent/MetadataListConstraints.java
+211
View File
@@ -0,0 +1,211 @@
package com.dbx.agent;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public final class MetadataListConstraints {
public static final MetadataListConstraints NONE = new MetadataListConstraints(null, null, null, null);
private final String filter;
private final Integer limit;
private final Integer offset;
private final List<String> objectTypes;
public MetadataListConstraints(String filter, Integer limit, Integer offset, List<String> objectTypes) {
this.filter = normalizeFilter(filter);
this.limit = limit == null || limit <= 0 ? null : limit;
this.offset = offset == null || offset <= 0 ? null : offset;
this.objectTypes = normalizeObjectTypes(objectTypes);
}
public static MetadataListConstraints orNone(MetadataListConstraints constraints) {
return constraints == null ? NONE : constraints;
}
public String getFilter() {
return filter;
}
public Integer getLimit() {
return limit;
}
public Integer getOffset() {
return offset;
}
public List<String> getObjectTypes() {
return objectTypes.isEmpty() ? null : objectTypes;
}
public boolean hasFilter() {
return !filter.isEmpty();
}
public boolean hasLimit() {
return limit != null;
}
public boolean hasOffset() {
return offset != null;
}
public boolean hasObjectTypes() {
return !objectTypes.isEmpty();
}
public MetadataListConstraints withoutPaging() {
return new MetadataListConstraints(filter, null, null, objectTypes);
}
public String fuzzyLikePattern() {
if (filter.isEmpty()) {
return "%%";
}
StringBuilder builder = new StringBuilder(filter.length() * 2 + 2);
builder.append('%');
for (int i = 0; i < filter.length(); i++) {
char ch = filter.charAt(i);
if (ch == '\\' || ch == '%' || ch == '_') {
builder.append('\\');
}
builder.append(ch);
builder.append('%');
}
return builder.toString();
}
public boolean includesTableLikeTypes() {
if (objectTypes.isEmpty()) {
return true;
}
return objectTypes.contains("TABLE")
|| objectTypes.contains("VIEW")
|| objectTypes.contains("MATERIALIZED_VIEW");
}
public boolean tableTypeAllowed(String tableType) {
if (objectTypes.isEmpty()) {
return true;
}
return objectTypes.contains(normalizeTableType(tableType));
}
public boolean objectTypeAllowed(String objectType) {
if (objectTypes.isEmpty()) {
return true;
}
String normalized = normalizeObjectType(objectType);
return !normalized.isEmpty() && objectTypes.contains(normalized);
}
public boolean nameMatches(String name) {
if (filter.isEmpty()) {
return true;
}
if (name == null) {
return false;
}
String candidate = name.toLowerCase(Locale.ROOT);
return candidate.contains(filter) || (filter.length() >= 2 && fuzzySubsequenceMatches(candidate, filter));
}
public List<TableInfo> filterTables(List<TableInfo> tables) {
List<TableInfo> result = new ArrayList<>();
int skipped = 0;
int max = limit == null ? Integer.MAX_VALUE : limit;
int start = offset == null ? 0 : offset;
for (TableInfo table : tables) {
if (!nameMatches(table.getName()) || !tableTypeAllowed(table.getTable_type())) {
continue;
}
if (skipped++ < start) {
continue;
}
if (result.size() >= max) {
break;
}
result.add(table);
}
return result;
}
public List<ObjectInfo> filterObjects(List<ObjectInfo> objects) {
List<ObjectInfo> result = new ArrayList<>();
int skipped = 0;
int max = limit == null ? Integer.MAX_VALUE : limit;
int start = offset == null ? 0 : offset;
for (ObjectInfo object : objects) {
if (!nameMatches(object.getName()) || !objectTypeAllowed(object.getObject_type())) {
continue;
}
if (skipped++ < start) {
continue;
}
if (result.size() >= max) {
break;
}
result.add(object);
}
return result;
}
private static String normalizeFilter(String value) {
if (value == null) {
return "";
}
return value.trim().toLowerCase(Locale.ROOT);
}
private static List<String> normalizeObjectTypes(List<String> values) {
if (values == null || values.isEmpty()) {
return Collections.emptyList();
}
List<String> result = new ArrayList<>();
for (String value : values) {
String normalized = normalizeObjectType(value);
if (!normalized.isEmpty() && !result.contains(normalized)) {
result.add(normalized);
}
}
Collections.sort(result);
return Collections.unmodifiableList(result);
}
private static String normalizeTableType(String value) {
String normalized = normalizeObjectType(value);
return normalized.isEmpty() ? "TABLE" : normalized;
}
private static String normalizeObjectType(String value) {
if (value == null || value.trim().isEmpty()) {
return "";
}
String upper = value.trim().toUpperCase(Locale.ROOT).replace(' ', '_');
if (upper.contains("MATERIALIZED") && upper.contains("VIEW")) {
return "MATERIALIZED_VIEW";
}
if (upper.equals("BASE_TABLE") || upper.contains("TABLE")) {
return "TABLE";
}
if (upper.contains("VIEW")) {
return "VIEW";
}
return upper;
}
private static boolean fuzzySubsequenceMatches(String candidate, String expected) {
int cursor = 0;
for (int i = 0; i < expected.length(); i++) {
char ch = expected.charAt(i);
cursor = candidate.indexOf(ch, cursor);
if (cursor < 0) {
return false;
}
cursor++;
}
return true;
}
}
agents/common/src/main/java/com/dbx/agent/MetadataSqlSupport.java
+95
View File
@@ -0,0 +1,95 @@
package com.dbx.agent;
import java.sql.PreparedStatement;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public final class MetadataSqlSupport {
private MetadataSqlSupport() {
}
public static void appendNameFilter(StringBuilder sql, List<Object> args, String column, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasFilter()) {
return;
}
sql.append(" AND UPPER(").append(column).append(") LIKE ? ESCAPE '\\\\'");
args.add(normalized.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
public static void appendLimitOffset(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasLimit()) {
return;
}
sql.append(" LIMIT ?");
args.add(normalized.getLimit());
if (normalized.hasOffset()) {
sql.append(" OFFSET ?");
args.add(normalized.getOffset());
}
}
public static void appendOffsetFetch(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (normalized.hasOffset()) {
sql.append(" OFFSET ? ROWS");
args.add(normalized.getOffset());
}
if (normalized.hasLimit()) {
sql.append(normalized.hasOffset() ? " FETCH NEXT ? ROWS ONLY" : " FETCH FIRST ? ROWS ONLY");
args.add(normalized.getLimit());
}
}
public static void appendLiteralLimitOffset(StringBuilder sql, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasLimit()) {
return;
}
sql.append(" LIMIT ").append(normalized.getLimit());
if (normalized.hasOffset()) {
sql.append(" OFFSET ").append(normalized.getOffset());
}
}
public static void appendLiteralOffsetFetch(StringBuilder sql, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (normalized.hasOffset()) {
sql.append(" OFFSET ").append(normalized.getOffset()).append(" ROWS");
}
if (normalized.hasLimit()) {
sql.append(normalized.hasOffset() ? " FETCH NEXT " : " FETCH FIRST ")
.append(normalized.getLimit())
.append(" ROWS ONLY");
}
}
public static void appendLiteralSkipFirst(StringBuilder sql, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (normalized.hasOffset()) {
sql.append("SKIP ").append(normalized.getOffset()).append(' ');
}
if (normalized.hasLimit()) {
sql.append("FIRST ").append(normalized.getLimit()).append(' ');
}
}
public static String placeholders(int count) {
return String.join(", ", Collections.nCopies(count, "?"));
}
public static void bind(PreparedStatement stmt, List<Object> args) throws Exception {
for (int index = 0; index < args.size(); index += 1) {
Object arg = args.get(index);
if (arg instanceof Integer) {
stmt.setInt(index + 1, (Integer) arg);
} else if (arg == null) {
stmt.setObject(index + 1, null);
} else {
stmt.setString(index + 1, String.valueOf(arg));
}
}
}
}
agents/common/src/main/java/com/dbx/agent/StandardJdbcMetadata.java
+49 -6
View File
@@ -70,24 +70,42 @@ public final class StandardJdbcMetadata {
}
public List<TableInfo> listTables(Connection conn, JdbcAgentProfile profile, String configuredDatabase, String schema) {
return listTables(conn, profile, configuredDatabase, schema, MetadataListConstraints.NONE);
}
public List<TableInfo> listTables(
Connection conn,
JdbcAgentProfile profile,
String configuredDatabase,
String schema,
MetadataListConstraints constraints
) {
return unchecked(() -> {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.includesTableLikeTypes()) {
return Collections.emptyList();
}
DatabaseMetaData meta = conn.getMetaData();
List<TableInfo> result = new ArrayList<>();
appendTables(result, meta, profile, null, schema);
appendTables(result, meta, profile, null, schema, normalized);
if (result.isEmpty() && profile.getCatalogFallbackEnabled() && !configuredDatabase.trim().isEmpty()) {
appendTables(result, meta, profile, configuredDatabase, schema);
appendTables(result, meta, profile, configuredDatabase, schema, normalized);
}
result.sort(Comparator.comparing(TableInfo::getName));
return result;
return normalized.filterTables(result);
});
}
public List<ObjectInfo> listObjects(List<TableInfo> tables, String schema) {
return listObjects(tables, schema, MetadataListConstraints.NONE);
}
public List<ObjectInfo> listObjects(List<TableInfo> tables, String schema, MetadataListConstraints constraints) {
List<ObjectInfo> result = new ArrayList<>();
for (TableInfo table : tables) {
result.add(new ObjectInfo(table.getName(), table.getTable_type(), schema, table.getComment()));
}
return result;
return MetadataListConstraints.orNone(constraints).filterObjects(result);
}
public List<String> listDataTypes(Connection conn) {
@@ -270,8 +288,19 @@ public final class StandardJdbcMetadata {
return profile.schemaSwitchSql(schema, quote);
}
private void appendTables(List<TableInfo> result, DatabaseMetaData meta, JdbcAgentProfile profile, String catalog, String schema) throws Exception {
String[] tableTypes = getDriverTableTypes(meta, profile);
private void appendTables(
List<TableInfo> result,
DatabaseMetaData meta,
JdbcAgentProfile profile,
String catalog,
String schema,
MetadataListConstraints constraints
) throws Exception {
String[] tableTypes = constrainedTableTypes(getDriverTableTypes(meta, profile), constraints);
if (tableTypes.length == 0) {
return;
}
// JDBC has no portable metadata limit/offset. Use table types for safe pushdown and filter/page locally.
try (ResultSet rs = meta.getTables(catalog, blankToNull(schema), "%", tableTypes)) {
while (rs.next()) {
result.add(new TableInfo(
@@ -283,6 +312,20 @@ public final class StandardJdbcMetadata {
}
}
private static String[] constrainedTableTypes(String[] tableTypes, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasObjectTypes()) {
return tableTypes;
}
List<String> result = new ArrayList<>();
for (String tableType : tableTypes) {
if (normalized.tableTypeAllowed(tableType)) {
result.add(tableType);
}
}
return result.toArray(new String[0]);
}
private void appendCompletionSchemas(
List<CompletionAssistantCandidate> result,
DatabaseMetaData meta,