feat(metadata): optimize agent metadata loading

This commit is contained in:
t8y2 committed 2026-07-05 12:09:24 +08:00
1 parent 8883a6b3b7
commit e74782f651
65 files changed
+6192 -571

No files matched your search

@@ -55,11 +55,35 @@ public abstract class ConfiguredJdbcAgent extends AbstractJdbcAgent {
return StandardJdbcMetadata.INSTANCE.listTables(requireConnection(), profile, configuredDatabase, schema); return StandardJdbcMetadata.INSTANCE.listTables(requireConnection(), profile, configuredDatabase, schema);
} }
@Override
public List<TableInfo> listTables(String schema, List<String> objectTypes) {
return listTables(schema, new MetadataListConstraints(null, null, null, objectTypes));
}
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
return StandardJdbcMetadata.INSTANCE.listTables(
requireConnection(),
profile,
configuredDatabase,
schema,
constraints
);
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return StandardJdbcMetadata.INSTANCE.listObjects(listTables(schema), schema); return StandardJdbcMetadata.INSTANCE.listObjects(listTables(schema), schema);
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
MetadataListConstraints tableConstraints =
new MetadataListConstraints(normalized.getFilter(), null, null, normalized.getObjectTypes());
return StandardJdbcMetadata.INSTANCE.listObjects(listTables(schema, tableConstraints), schema, normalized);
}
@Override @Override
public List<String> listDataTypes() { public List<String> listDataTypes() {
return StandardJdbcMetadata.INSTANCE.listDataTypes(requireConnection()); return StandardJdbcMetadata.INSTANCE.listDataTypes(requireConnection());
@@ -31,6 +31,12 @@ public interface DatabaseAgent {
return listTables(schema); return listTables(schema);
} }
default List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
// Keep old driver overrides for object type filtering, then apply the new optional constraints locally.
return normalized.filterTables(listTables(schema, normalized.getObjectTypes()));
}
default List<ObjectInfo> listObjects(String schema) { default List<ObjectInfo> listObjects(String schema) {
List<ObjectInfo> result = new ArrayList<>(); List<ObjectInfo> result = new ArrayList<>();
for (TableInfo table : listTables(schema)) { for (TableInfo table : listTables(schema)) {
@@ -39,6 +45,10 @@ public interface DatabaseAgent {
return result; return result;
} }
default List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
return MetadataListConstraints.orNone(constraints).filterObjects(listObjects(schema));
}
default List<String> listDataTypes() { default List<String> listDataTypes() {
return Collections.emptyList(); return Collections.emptyList();
} }
@@ -124,11 +124,11 @@ public final class JsonRpcServer {
} }
if (AgentProtocol.METHOD_LIST_TABLES.equals(method)) { if (AgentProtocol.METHOD_LIST_TABLES.equals(method)) {
switchCatalog(params); switchCatalog(params);
return agent.listTables(params.get("schema").getAsString(), stringListOrNull(params, "object_types")); return agent.listTables(params.get("schema").getAsString(), metadataListConstraints(params));
} }
if (AgentProtocol.METHOD_LIST_OBJECTS.equals(method)) { if (AgentProtocol.METHOD_LIST_OBJECTS.equals(method)) {
switchCatalog(params); switchCatalog(params);
return agent.listObjects(params.get("schema").getAsString()); return agent.listObjects(params.get("schema").getAsString(), metadataListConstraints(params));
} }
if (AgentProtocol.METHOD_LIST_DATA_TYPES.equals(method)) { if (AgentProtocol.METHOD_LIST_DATA_TYPES.equals(method)) {
switchCatalog(params); switchCatalog(params);
@@ -356,6 +356,15 @@ public final class JsonRpcServer {
return element.getAsInt(); return element.getAsInt();
} }
private MetadataListConstraints metadataListConstraints(JsonObject params) {
return new MetadataListConstraints(
stringOrNull(params, "filter"),
intOrNull(params, "limit"),
intOrNull(params, "offset"),
stringListOrNull(params, "object_types")
);
}
private static int intOrDefault(JsonObject object, String key, int defaultValue) { private static int intOrDefault(JsonObject object, String key, int defaultValue) {
Integer value = intOrNull(object, key); Integer value = intOrNull(object, key);
return value == null ? defaultValue : value; return value == null ? defaultValue : value;
@@ -0,0 +1,211 @@
package com.dbx.agent;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public final class MetadataListConstraints {
public static final MetadataListConstraints NONE = new MetadataListConstraints(null, null, null, null);
private final String filter;
private final Integer limit;
private final Integer offset;
private final List<String> objectTypes;
public MetadataListConstraints(String filter, Integer limit, Integer offset, List<String> objectTypes) {
this.filter = normalizeFilter(filter);
this.limit = limit == null || limit <= 0 ? null : limit;
this.offset = offset == null || offset <= 0 ? null : offset;
this.objectTypes = normalizeObjectTypes(objectTypes);
}
public static MetadataListConstraints orNone(MetadataListConstraints constraints) {
return constraints == null ? NONE : constraints;
}
public String getFilter() {
return filter;
}
public Integer getLimit() {
return limit;
}
public Integer getOffset() {
return offset;
}
public List<String> getObjectTypes() {
return objectTypes.isEmpty() ? null : objectTypes;
}
public boolean hasFilter() {
return !filter.isEmpty();
}
public boolean hasLimit() {
return limit != null;
}
public boolean hasOffset() {
return offset != null;
}
public boolean hasObjectTypes() {
return !objectTypes.isEmpty();
}
public MetadataListConstraints withoutPaging() {
return new MetadataListConstraints(filter, null, null, objectTypes);
}
public String fuzzyLikePattern() {
if (filter.isEmpty()) {
return "%%";
}
StringBuilder builder = new StringBuilder(filter.length() * 2 + 2);
builder.append('%');
for (int i = 0; i < filter.length(); i++) {
char ch = filter.charAt(i);
if (ch == '\\' || ch == '%' || ch == '_') {
builder.append('\\');
}
builder.append(ch);
builder.append('%');
}
return builder.toString();
}
public boolean includesTableLikeTypes() {
if (objectTypes.isEmpty()) {
return true;
}
return objectTypes.contains("TABLE")
|| objectTypes.contains("VIEW")
|| objectTypes.contains("MATERIALIZED_VIEW");
}
public boolean tableTypeAllowed(String tableType) {
if (objectTypes.isEmpty()) {
return true;
}
return objectTypes.contains(normalizeTableType(tableType));
}
public boolean objectTypeAllowed(String objectType) {
if (objectTypes.isEmpty()) {
return true;
}
String normalized = normalizeObjectType(objectType);
return !normalized.isEmpty() && objectTypes.contains(normalized);
}
public boolean nameMatches(String name) {
if (filter.isEmpty()) {
return true;
}
if (name == null) {
return false;
}
String candidate = name.toLowerCase(Locale.ROOT);
return candidate.contains(filter) || (filter.length() >= 2 && fuzzySubsequenceMatches(candidate, filter));
}
public List<TableInfo> filterTables(List<TableInfo> tables) {
List<TableInfo> result = new ArrayList<>();
int skipped = 0;
int max = limit == null ? Integer.MAX_VALUE : limit;
int start = offset == null ? 0 : offset;
for (TableInfo table : tables) {
if (!nameMatches(table.getName()) || !tableTypeAllowed(table.getTable_type())) {
continue;
}
if (skipped++ < start) {
continue;
}
if (result.size() >= max) {
break;
}
result.add(table);
}
return result;
}
public List<ObjectInfo> filterObjects(List<ObjectInfo> objects) {
List<ObjectInfo> result = new ArrayList<>();
int skipped = 0;
int max = limit == null ? Integer.MAX_VALUE : limit;
int start = offset == null ? 0 : offset;
for (ObjectInfo object : objects) {
if (!nameMatches(object.getName()) || !objectTypeAllowed(object.getObject_type())) {
continue;
}
if (skipped++ < start) {
continue;
}
if (result.size() >= max) {
break;
}
result.add(object);
}
return result;
}
private static String normalizeFilter(String value) {
if (value == null) {
return "";
}
return value.trim().toLowerCase(Locale.ROOT);
}
private static List<String> normalizeObjectTypes(List<String> values) {
if (values == null || values.isEmpty()) {
return Collections.emptyList();
}
List<String> result = new ArrayList<>();
for (String value : values) {
String normalized = normalizeObjectType(value);
if (!normalized.isEmpty() && !result.contains(normalized)) {
result.add(normalized);
}
}
Collections.sort(result);
return Collections.unmodifiableList(result);
}
private static String normalizeTableType(String value) {
String normalized = normalizeObjectType(value);
return normalized.isEmpty() ? "TABLE" : normalized;
}
private static String normalizeObjectType(String value) {
if (value == null || value.trim().isEmpty()) {
return "";
}
String upper = value.trim().toUpperCase(Locale.ROOT).replace(' ', '_');
if (upper.contains("MATERIALIZED") && upper.contains("VIEW")) {
return "MATERIALIZED_VIEW";
}
if (upper.equals("BASE_TABLE") || upper.contains("TABLE")) {
return "TABLE";
}
if (upper.contains("VIEW")) {
return "VIEW";
}
return upper;
}
private static boolean fuzzySubsequenceMatches(String candidate, String expected) {
int cursor = 0;
for (int i = 0; i < expected.length(); i++) {
char ch = expected.charAt(i);
cursor = candidate.indexOf(ch, cursor);
if (cursor < 0) {
return false;
}
cursor++;
}
return true;
}
}
@@ -0,0 +1,95 @@
package com.dbx.agent;
import java.sql.PreparedStatement;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public final class MetadataSqlSupport {
private MetadataSqlSupport() {
}
public static void appendNameFilter(StringBuilder sql, List<Object> args, String column, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasFilter()) {
return;
}
sql.append(" AND UPPER(").append(column).append(") LIKE ? ESCAPE '\\\\'");
args.add(normalized.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
public static void appendLimitOffset(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasLimit()) {
return;
}
sql.append(" LIMIT ?");
args.add(normalized.getLimit());
if (normalized.hasOffset()) {
sql.append(" OFFSET ?");
args.add(normalized.getOffset());
}
}
public static void appendOffsetFetch(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (normalized.hasOffset()) {
sql.append(" OFFSET ? ROWS");
args.add(normalized.getOffset());
}
if (normalized.hasLimit()) {
sql.append(normalized.hasOffset() ? " FETCH NEXT ? ROWS ONLY" : " FETCH FIRST ? ROWS ONLY");
args.add(normalized.getLimit());
}
}
public static void appendLiteralLimitOffset(StringBuilder sql, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasLimit()) {
return;
}
sql.append(" LIMIT ").append(normalized.getLimit());
if (normalized.hasOffset()) {
sql.append(" OFFSET ").append(normalized.getOffset());
}
}
public static void appendLiteralOffsetFetch(StringBuilder sql, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (normalized.hasOffset()) {
sql.append(" OFFSET ").append(normalized.getOffset()).append(" ROWS");
}
if (normalized.hasLimit()) {
sql.append(normalized.hasOffset() ? " FETCH NEXT " : " FETCH FIRST ")
.append(normalized.getLimit())
.append(" ROWS ONLY");
}
}
public static void appendLiteralSkipFirst(StringBuilder sql, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (normalized.hasOffset()) {
sql.append("SKIP ").append(normalized.getOffset()).append(' ');
}
if (normalized.hasLimit()) {
sql.append("FIRST ").append(normalized.getLimit()).append(' ');
}
}
public static String placeholders(int count) {
return String.join(", ", Collections.nCopies(count, "?"));
}
public static void bind(PreparedStatement stmt, List<Object> args) throws Exception {
for (int index = 0; index < args.size(); index += 1) {
Object arg = args.get(index);
if (arg instanceof Integer) {
stmt.setInt(index + 1, (Integer) arg);
} else if (arg == null) {
stmt.setObject(index + 1, null);
} else {
stmt.setString(index + 1, String.valueOf(arg));
}
}
}
}
@@ -70,24 +70,42 @@ public final class StandardJdbcMetadata {
} }
public List<TableInfo> listTables(Connection conn, JdbcAgentProfile profile, String configuredDatabase, String schema) { public List<TableInfo> listTables(Connection conn, JdbcAgentProfile profile, String configuredDatabase, String schema) {
return listTables(conn, profile, configuredDatabase, schema, MetadataListConstraints.NONE);
}
public List<TableInfo> listTables(
Connection conn,
JdbcAgentProfile profile,
String configuredDatabase,
String schema,
MetadataListConstraints constraints
) {
return unchecked(() -> { return unchecked(() -> {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.includesTableLikeTypes()) {
return Collections.emptyList();
}
DatabaseMetaData meta = conn.getMetaData(); DatabaseMetaData meta = conn.getMetaData();
List<TableInfo> result = new ArrayList<>(); List<TableInfo> result = new ArrayList<>();
appendTables(result, meta, profile, null, schema); appendTables(result, meta, profile, null, schema, normalized);
if (result.isEmpty() && profile.getCatalogFallbackEnabled() && !configuredDatabase.trim().isEmpty()) { if (result.isEmpty() && profile.getCatalogFallbackEnabled() && !configuredDatabase.trim().isEmpty()) {
appendTables(result, meta, profile, configuredDatabase, schema); appendTables(result, meta, profile, configuredDatabase, schema, normalized);
} }
result.sort(Comparator.comparing(TableInfo::getName)); result.sort(Comparator.comparing(TableInfo::getName));
return result; return normalized.filterTables(result);
}); });
} }
public List<ObjectInfo> listObjects(List<TableInfo> tables, String schema) { public List<ObjectInfo> listObjects(List<TableInfo> tables, String schema) {
return listObjects(tables, schema, MetadataListConstraints.NONE);
}
public List<ObjectInfo> listObjects(List<TableInfo> tables, String schema, MetadataListConstraints constraints) {
List<ObjectInfo> result = new ArrayList<>(); List<ObjectInfo> result = new ArrayList<>();
for (TableInfo table : tables) { for (TableInfo table : tables) {
result.add(new ObjectInfo(table.getName(), table.getTable_type(), schema, table.getComment())); result.add(new ObjectInfo(table.getName(), table.getTable_type(), schema, table.getComment()));
} }
return result; return MetadataListConstraints.orNone(constraints).filterObjects(result);
} }
public List<String> listDataTypes(Connection conn) { public List<String> listDataTypes(Connection conn) {
@@ -270,8 +288,19 @@ public final class StandardJdbcMetadata {
return profile.schemaSwitchSql(schema, quote); return profile.schemaSwitchSql(schema, quote);
} }
private void appendTables(List<TableInfo> result, DatabaseMetaData meta, JdbcAgentProfile profile, String catalog, String schema) throws Exception { private void appendTables(
String[] tableTypes = getDriverTableTypes(meta, profile); List<TableInfo> result,
DatabaseMetaData meta,
JdbcAgentProfile profile,
String catalog,
String schema,
MetadataListConstraints constraints
) throws Exception {
String[] tableTypes = constrainedTableTypes(getDriverTableTypes(meta, profile), constraints);
if (tableTypes.length == 0) {
return;
}
// JDBC has no portable metadata limit/offset. Use table types for safe pushdown and filter/page locally.
try (ResultSet rs = meta.getTables(catalog, blankToNull(schema), "%", tableTypes)) { try (ResultSet rs = meta.getTables(catalog, blankToNull(schema), "%", tableTypes)) {
while (rs.next()) { while (rs.next()) {
result.add(new TableInfo( result.add(new TableInfo(
@@ -283,6 +312,20 @@ public final class StandardJdbcMetadata {
} }
} }
private static String[] constrainedTableTypes(String[] tableTypes, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.hasObjectTypes()) {
return tableTypes;
}
List<String> result = new ArrayList<>();
for (String tableType : tableTypes) {
if (normalized.tableTypeAllowed(tableType)) {
result.add(tableType);
}
}
return result.toArray(new String[0]);
}
private void appendCompletionSchemas( private void appendCompletionSchemas(
List<CompletionAssistantCandidate> result, List<CompletionAssistantCandidate> result,
DatabaseMetaData meta, DatabaseMetaData meta,
@@ -146,6 +146,36 @@ class CommonJavaCompatibilityTest {
assertEquals("app", agent.lastSchema); assertEquals("app", agent.lastSchema);
} }
@Test
void jsonRpcServerAppliesConstrainedTableMetadataRequests() {
MetadataConstraintAgent agent = new MetadataConstraintAgent();
JsonRpcServer server = new JsonRpcServer(agent);
String response = server.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":4,\"method\":\"" + AgentProtocol.METHOD_LIST_TABLES + "\",\"params\":{\"schema\":\"app\",\"filter\":\"us\",\"limit\":1,\"offset\":1,\"object_types\":[\"TABLE\"]}}"
);
JsonArray result = JsonParser.parseString(response).getAsJsonObject().getAsJsonArray("result");
assertEquals(1, result.size());
assertEquals("user_settings", result.get(0).getAsJsonObject().get("name").getAsString());
assertEquals("app", agent.lastSchema);
}
@Test
void jsonRpcServerAppliesConstrainedObjectMetadataRequests() {
MetadataConstraintAgent agent = new MetadataConstraintAgent();
JsonRpcServer server = new JsonRpcServer(agent);
String response = server.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":5,\"method\":\"" + AgentProtocol.METHOD_LIST_OBJECTS + "\",\"params\":{\"schema\":\"app\",\"filter\":\"fn\",\"limit\":1,\"offset\":1,\"object_types\":[\"FUNCTION\"]}}"
);
JsonArray result = JsonParser.parseString(response).getAsJsonObject().getAsJsonArray("result");
assertEquals(1, result.size());
assertEquals("fetch_name", result.get(0).getAsJsonObject().get("name").getAsString());
assertEquals("FUNCTION", result.get(0).getAsJsonObject().get("object_type").getAsString());
}
@Test @Test
void jsonRpcServerDispatchesTableReadSessionMethods() { void jsonRpcServerDispatchesTableReadSessionMethods() {
TableReadDispatchAgent agent = new TableReadDispatchAgent(); TableReadDispatchAgent agent = new TableReadDispatchAgent();
@@ -253,6 +283,25 @@ class CommonJavaCompatibilityTest {
); );
} }
@Test
void databaseAgentDefaultConstraintsFilterLegacyMetadataOverrides() {
DatabaseAgent agent = new LegacyObjectTypeAgent();
List<TableInfo> tables = agent.listTables(
"public",
new MetadataListConstraints("us", 1, 1, Collections.singletonList("TABLE"))
);
assertEquals(1, tables.size());
assertEquals("user_settings", tables.get(0).getName());
List<ObjectInfo> objects = agent.listObjects(
"public",
new MetadataListConstraints("us", 1, 0, Collections.singletonList("VIEW"))
);
assertEquals(1, objects.size());
assertEquals("usage_view", objects.get(0).getName());
}
@Test @Test
void executesTransactionsOneByOneWhenJdbcDriverDoesNotSupportTransactions() { void executesTransactionsOneByOneWhenJdbcDriverDoesNotSupportTransactions() {
List<String> calls = new ArrayList<>(); List<String> calls = new ArrayList<>();
@@ -394,6 +443,33 @@ class CommonJavaCompatibilityTest {
} }
} }
private static final class LegacyObjectTypeAgent extends MinimalAgent {
@Override
public List<TableInfo> listTables(String schema) {
return Arrays.asList(
new TableInfo("orders", "TABLE"),
new TableInfo("usage_view", "VIEW"),
new TableInfo("users", "TABLE"),
new TableInfo("user_settings", "TABLE")
);
}
@Override
public List<TableInfo> listTables(String schema, List<String> objectTypes) {
List<TableInfo> result = listTables(schema);
if (objectTypes == null || objectTypes.isEmpty()) {
return result;
}
List<TableInfo> filtered = new ArrayList<>();
for (TableInfo table : result) {
if (objectTypes.contains(table.getTable_type())) {
filtered.add(table);
}
}
return filtered;
}
}
private static final class PreciseNumberAgent extends MinimalAgent { private static final class PreciseNumberAgent extends MinimalAgent {
@Override @Override
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) { public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
@@ -472,6 +548,44 @@ class CommonJavaCompatibilityTest {
} }
} }
private static final class MetadataConstraintAgent extends MinimalAgent {
private String lastSchema = "";
@Override
public Connection getConnection() {
return proxy(Connection.class, (method, args) -> {
if ("isClosed".equals(method.getName())) {
return false;
}
if ("isValid".equals(method.getName())) {
return true;
}
return defaultValue(method.getReturnType());
});
}
@Override
public List<TableInfo> listTables(String schema) {
lastSchema = schema;
return Arrays.asList(
new TableInfo("orders", "TABLE"),
new TableInfo("users", "TABLE"),
new TableInfo("usage_view", "VIEW"),
new TableInfo("user_settings", "TABLE")
);
}
@Override
public List<ObjectInfo> listObjects(String schema) {
return Arrays.asList(
new ObjectInfo("orders", "TABLE", schema, null),
new ObjectInfo("find_user", "FUNCTION", schema, null),
new ObjectInfo("fetch_name", "FUNCTION", schema, null),
new ObjectInfo("cleanup_user", "PROCEDURE", schema, null)
);
}
}
private static final class TableReadDispatchAgent extends MinimalAgent { private static final class TableReadDispatchAgent extends MinimalAgent {
private String lastSql; private String lastSql;
private String lastSchema; private String lastSchema;
@@ -0,0 +1,114 @@
package com.dbx.agent;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class MetadataConstraintCoverageTest {
private static final Pattern JAVA_METADATA_OVERRIDE = Pattern.compile(
"public\\s+List\\s*<\\s*(?:TableInfo|ObjectInfo)\\s*>\\s+list(?:Tables|Objects)\\s*\\(\\s*String\\s+schema"
);
private static final Set<String> VALID_STRATEGIES = new HashSet<>(Arrays.asList(
"native-pushdown",
"shared-fallback",
"intentional-fallback"
));
@Test
void everyCustomMetadataDriverIsListedInCoverageMatrix() throws Exception {
Path agentsRoot = agentsRoot();
Map<String, String> coverage = readCoverageMatrix(agentsRoot.resolve("metadata-constraint-coverage.tsv"));
Set<String> discovered = discoverCustomMetadataDrivers(agentsRoot.resolve("drivers"));
assertEquals(discovered, coverage.keySet());
}
@Test
void coverageMatrixUsesKnownStrategiesAndReasons() throws Exception {
Path agentsRoot = agentsRoot();
Path matrix = agentsRoot.resolve("metadata-constraint-coverage.tsv");
for (String line : Files.readAllLines(matrix, StandardCharsets.UTF_8)) {
if (line.startsWith("driver\t") || line.trim().isEmpty()) {
continue;
}
String[] parts = line.split("\t", -1);
assertEquals(4, parts.length, line);
assertTrue(VALID_STRATEGIES.contains(parts[1]), line);
assertFalse(parts[3].trim().isEmpty(), line);
}
}
private static Set<String> discoverCustomMetadataDrivers(Path driversRoot) throws IOException {
Set<String> result = new HashSet<>();
try (Stream<Path> files = Files.walk(driversRoot)) {
files.filter(Files::isRegularFile).forEach(file -> {
String fileName = file.getFileName().toString();
if (fileName.endsWith(".java") && hasJavaMetadataOverride(file)) {
result.add(driversRoot.relativize(file).getName(0).toString());
} else if ("main.go".equals(fileName) && hasGoMetadataDispatcher(file)) {
result.add(driversRoot.relativize(file).getName(0).toString());
}
});
}
return result;
}
private static boolean hasJavaMetadataOverride(Path file) {
try {
return JAVA_METADATA_OVERRIDE.matcher(readUtf8(file)).find();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static boolean hasGoMetadataDispatcher(Path file) {
try {
String source = readUtf8(file);
return source.contains("\"list_tables\"") || source.contains("\"list_objects\"");
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static Map<String, String> readCoverageMatrix(Path matrix) throws IOException {
Map<String, String> result = new HashMap<>();
for (String line : Files.readAllLines(matrix, StandardCharsets.UTF_8)) {
if (line.startsWith("driver\t") || line.trim().isEmpty()) {
continue;
}
String[] parts = line.split("\t", -1);
result.put(parts[0], parts[1]);
}
return result;
}
private static String readUtf8(Path file) throws IOException {
return new String(Files.readAllBytes(file), StandardCharsets.UTF_8);
}
private static Path agentsRoot() {
Path current = Paths.get("").toAbsolutePath();
while (current != null) {
if (Files.isDirectory(current.resolve("drivers")) && Files.isDirectory(current.resolve("common"))) {
return current;
}
current = current.getParent();
}
throw new IllegalStateException("Unable to locate agents root");
}
}
@@ -8,12 +8,15 @@ import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo; import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo; import com.dbx.agent.TriggerInfo;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale;
public final class BigQueryAgent extends AbstractJdbcAgent { public final class BigQueryAgent extends AbstractJdbcAgent {
@@ -70,6 +73,56 @@ public final class BigQueryAgent extends AbstractJdbcAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
if (hasLikeWildcard(normalized.getFilter())) {
return normalized.filterTables(listTables(schema));
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
String quotedSchema = JdbcIdentifiers.INSTANCE.backtick(schema);
StringBuilder sql = new StringBuilder("SELECT table_name, table_type FROM ")
.append(quotedSchema)
.append(".INFORMATION_SCHEMA.TABLES WHERE 1 = 1");
appendBigQueryTableTypePredicate(sql, args, constraints);
if (constraints.hasFilter()) {
sql.append(" AND UPPER(table_name) LIKE ?");
args.add(constraints.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
sql.append(" ORDER BY table_name");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString("table_name"),
normalizeTableType(rs.getString("table_type")),
null
));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ColumnInfo> getColumns(String schema, String table) { public List<ColumnInfo> getColumns(String schema, String table) {
return unchecked(() -> { return unchecked(() -> {
@@ -136,7 +189,43 @@ public final class BigQueryAgent extends AbstractJdbcAgent {
} }
private static String normalizeTableType(String type) { private static String normalizeTableType(String type) {
return "BASE TABLE".equals(type) ? "TABLE" : type; if ("BASE TABLE".equals(type)) {
return "TABLE";
}
if ("MATERIALIZED VIEW".equals(type)) {
return "MATERIALIZED_VIEW";
}
return type;
}
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static void appendBigQueryTableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (constraints.tableTypeAllowed("MATERIALIZED_VIEW")) {
types.add("MATERIALIZED VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND table_type IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static boolean hasLikeWildcard(String value) {
return value != null && (value.contains("%") || value.contains("_") || value.contains("\\"));
} }
public static void main(String[] args) { public static void main(String[] args) {
@@ -2,9 +2,11 @@ package com.dbx.agent.bigquery;
import com.dbx.agent.test.TestSupport; import com.dbx.agent.test.TestSupport;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.JdbcAgentFake; import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.JdbcMetadataSqlFake; import com.dbx.agent.test.JdbcMetadataSqlFake;
import java.sql.Connection; import java.sql.Connection;
import java.util.List;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -46,4 +48,18 @@ class BigQueryAgentMetadataTest {
BigQueryAgent.buildUrl(params) BigQueryAgent.buildUrl(params)
); );
} }
@Test
void constrainedTableMetadataPushesFilterTypesAndPaging() {
BigQueryAgent agent = new BigQueryAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("demo`dataset", new MetadataListConstraints("ord", 25, 50, List.of("TABLE", "VIEW")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM `demo``dataset`.INFORMATION_SCHEMA.TABLES"), sql);
Assertions.assertTrue(sql.contains("table_type IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("UPPER(table_name) LIKE ?"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 25 OFFSET 50"), sql);
}
} }
@@ -10,6 +10,7 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor; import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.QueryPageOptions; import com.dbx.agent.QueryPageOptions;
@@ -140,7 +141,9 @@ public final class DamengAgent extends BaseDatabaseAgent {
@Override @Override
public List<TableInfo> listTables(String schema) { public List<TableInfo> listTables(String schema) {
return listTables(schema, null); // Keep this call on the legacy object-type overload; the common agent
// also exposes a constraints overload for paged metadata listing.
return listTables(schema, (List<String>) null);
} }
@Override @Override
@@ -162,6 +165,60 @@ public final class DamengAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
return queryConstrainedTables(schema, normalized);
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
if (!constraints.includesTableLikeTypes()) {
return List.of();
}
try {
return unchecked(() -> {
MetadataQuery query = buildConstrainedTablesQuery(schema, constraints);
List<TableInfo> result = new ArrayList<>();
try (PreparedStatement stmt = requireConnected().prepareStatement(query.sql())) {
bind(stmt, query.args());
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString("TABLE_NAME"), normalizeObjectType(rs.getString("TABLE_TYPE")), rs.getString("COMMENTS")));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
} catch (RuntimeException e) {
// Restricted Dameng catalog views vary by version; fall back to the
// legacy metadata path so browsing still works when pushdown fails.
return constraints.filterTables(listTables(schema));
}
}
static MetadataQuery buildConstrainedTablesQuery(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("""
SELECT o.OBJECT_NAME AS TABLE_NAME,
CASE WHEN o.OBJECT_TYPE = 'MATERIALIZED VIEW' THEN 'MATERIALIZED_VIEW' ELSE o.OBJECT_TYPE END AS TABLE_TYPE,
c.COMMENTS
FROM ALL_OBJECTS o
LEFT JOIN ALL_TAB_COMMENTS c ON c.OWNER = o.OWNER AND c.TABLE_NAME = o.OBJECT_NAME
WHERE o.OWNER = ?
""".stripIndent().trim());
args.add(schema);
appendDamengObjectTypePredicate(sql, args, normalized, true);
sql.append(" AND (o.OBJECT_TYPE <> 'TABLE' OR o.OBJECT_NAME NOT LIKE 'MTAB$_%')");
appendNameFilter(sql, args, "o.OBJECT_NAME", normalized);
sql.append(" ORDER BY o.OBJECT_NAME");
appendLimitOffset(sql, args, normalized);
return new MetadataQuery(sql.toString(), args);
}
private void loadTableOrView(String schema, String tableType, Map<String, TableInfo> tablesByName) { private void loadTableOrView(String schema, String tableType, Map<String, TableInfo> tablesByName) {
if (!loadTableOrViewFromAllObjects(schema, tableType, tablesByName)) { if (!loadTableOrViewFromAllObjects(schema, tableType, tablesByName)) {
loadTableOrViewFromComments(schema, tableType, tablesByName); loadTableOrViewFromComments(schema, tableType, tablesByName);
@@ -307,6 +364,111 @@ public final class DamengAgent extends BaseDatabaseAgent {
return false; return false;
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjectTypes(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes()
|| constraints.objectTypeAllowed("PROCEDURE")
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendDamengObjectTypePredicate(
StringBuilder sql,
List<Object> args,
MetadataListConstraints constraints,
boolean tableOnly
) {
List<String> objectTypes = tableOnly ? damengTableObjectTypes(constraints) : damengObjectTypes(constraints);
if (objectTypes.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND o.OBJECT_TYPE IN (").append(placeholders(objectTypes.size())).append(")");
args.addAll(objectTypes);
}
private static List<String> damengTableObjectTypes(MetadataListConstraints constraints) {
List<String> result = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
result.add("TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
result.add("VIEW");
}
if (constraints.tableTypeAllowed("MATERIALIZED_VIEW")) {
result.add("MATERIALIZED VIEW");
}
return result;
}
private static List<String> damengObjectTypes(MetadataListConstraints constraints) {
List<String> result = damengTableObjectTypes(constraints);
if (constraints.objectTypeAllowed("PROCEDURE")) {
result.add("PROCEDURE");
}
if (constraints.objectTypeAllowed("FUNCTION")) {
result.add("FUNCTION");
}
return result;
}
private static void appendNameFilter(StringBuilder sql, List<Object> args, String column, MetadataListConstraints constraints) {
if (!constraints.hasFilter()) {
return;
}
sql.append(" AND UPPER(").append(column).append(") LIKE ? ESCAPE '\\\\'");
args.add(constraints.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
private static void appendLimitOffset(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasLimit()) {
return;
}
sql.append(" LIMIT ?");
args.add(constraints.getLimit());
if (constraints.hasOffset()) {
sql.append(" OFFSET ?");
args.add(constraints.getOffset());
}
}
private static String placeholders(int count) {
return String.join(", ", java.util.Collections.nCopies(count, "?"));
}
private static void bind(PreparedStatement stmt, List<Object> args) throws Exception {
for (int index = 0; index < args.size(); index += 1) {
Object arg = args.get(index);
if (arg instanceof Integer) {
stmt.setInt(index + 1, (Integer) arg);
} else if (arg == null) {
stmt.setObject(index + 1, null);
} else {
stmt.setString(index + 1, String.valueOf(arg));
}
}
}
static final class MetadataQuery {
private final String sql;
private final List<Object> args;
MetadataQuery(String sql, List<Object> args) {
this.sql = sql;
this.args = args;
}
String sql() {
return sql;
}
List<Object> args() {
return args;
}
}
private static String normalizeObjectType(String value) { private static String normalizeObjectType(String value) {
String upper = value == null ? "" : value.trim().toUpperCase(Locale.ROOT).replace(' ', '_').replace('-', '_'); String upper = value == null ? "" : value.trim().toUpperCase(Locale.ROOT).replace(' ', '_').replace('-', '_');
if (upper.contains("MATERIALIZED") && upper.contains("VIEW")) { if (upper.contains("MATERIALIZED") && upper.contains("VIEW")) {
@@ -349,6 +511,71 @@ public final class DamengAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
return queryConstrainedObjects(schema, normalized);
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
if (!includesSupportedObjectTypes(constraints)) {
return List.of();
}
try {
return unchecked(() -> {
MetadataQuery query = buildConstrainedObjectsQuery(schema, constraints);
List<ObjectInfo> result = new ArrayList<>();
try (PreparedStatement stmt = requireConnected().prepareStatement(query.sql())) {
bind(stmt, query.args());
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(
rs.getString("OBJECT_NAME"),
normalizeObjectType(rs.getString("OBJECT_TYPE")),
schema,
rs.getString("COMMENTS")
));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
} catch (RuntimeException e) {
// Keep restricted/older Dameng catalogs usable even if SQL pushdown
// is unavailable for a specific connection.
return constraints.filterObjects(listObjects(schema));
}
}
static MetadataQuery buildConstrainedObjectsQuery(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("""
SELECT o.OBJECT_NAME,
CASE WHEN o.OBJECT_TYPE = 'MATERIALIZED VIEW' THEN 'MATERIALIZED_VIEW' ELSE o.OBJECT_TYPE END AS OBJECT_TYPE,
c.COMMENTS
FROM ALL_OBJECTS o
LEFT JOIN ALL_TAB_COMMENTS c ON c.OWNER = o.OWNER AND c.TABLE_NAME = o.OBJECT_NAME
WHERE o.OWNER = ?
""".stripIndent().trim());
args.add(schema);
appendDamengObjectTypePredicate(sql, args, normalized, false);
sql.append(" AND (o.OBJECT_TYPE <> 'TABLE' OR o.OBJECT_NAME NOT LIKE 'MTAB$_%')");
appendNameFilter(sql, args, "o.OBJECT_NAME", normalized);
sql.append(" ORDER BY CASE o.OBJECT_TYPE")
.append(" WHEN 'TABLE' THEN 0")
.append(" WHEN 'VIEW' THEN 1")
.append(" WHEN 'MATERIALIZED VIEW' THEN 2")
.append(" WHEN 'PROCEDURE' THEN 3")
.append(" WHEN 'FUNCTION' THEN 4")
.append(" ELSE 9 END, o.OBJECT_NAME");
appendLimitOffset(sql, args, normalized);
return new MetadataQuery(sql.toString(), args);
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -1,7 +1,14 @@
package com.dbx.agent.dameng; package com.dbx.agent.dameng;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DamengAgentTest extends JdbcFakeExecutionBehaviorTest { class DamengAgentTest extends JdbcFakeExecutionBehaviorTest {
@Override @Override
@@ -13,4 +20,42 @@ class DamengAgentTest extends JdbcFakeExecutionBehaviorTest {
protected String resultSetSql() { protected String resultSetSql() {
return "CALL SP_SAMPLE()"; return "CALL SP_SAMPLE()";
} }
@Test
void constrainedTableQueryPushesFilterTypeAndPagingToDameng() {
DamengAgent.MetadataQuery query = DamengAgent.buildConstrainedTablesQuery(
"APP",
new MetadataListConstraints("ord", 50, 100, List.of("TABLE", "VIEW"))
);
assertTrue(query.sql().contains("FROM ALL_OBJECTS o"));
assertTrue(query.sql().contains("o.OBJECT_TYPE IN (?, ?)"));
assertTrue(query.sql().contains("UPPER(o.OBJECT_NAME) LIKE ? ESCAPE '\\\\'"));
assertTrue(query.sql().endsWith("LIMIT ? OFFSET ?"));
assertEquals(List.of("APP", "TABLE", "VIEW", "%O%R%D%", 50, 100), query.args());
}
@Test
void constrainedTableQueryMapsMaterializedViewTypeToDamengCatalogName() {
DamengAgent.MetadataQuery query = DamengAgent.buildConstrainedTablesQuery(
"APP",
new MetadataListConstraints(null, 20, null, List.of("MATERIALIZED_VIEW"))
);
assertTrue(query.sql().contains("MATERIALIZED_VIEW"));
assertEquals(List.of("APP", "MATERIALIZED VIEW", 20), query.args());
}
@Test
void constrainedObjectQueryPushesRoutineOnlySearchToDameng() {
DamengAgent.MetadataQuery query = DamengAgent.buildConstrainedObjectsQuery(
"APP",
new MetadataListConstraints("sync", 20, null, List.of("PROCEDURE", "FUNCTION"))
);
assertTrue(query.sql().contains("o.OBJECT_TYPE IN (?, ?)"));
assertTrue(query.sql().contains("WHEN 'PROCEDURE' THEN 3"));
assertTrue(query.sql().endsWith("LIMIT ?"));
assertEquals(List.of("APP", "PROCEDURE", "FUNCTION", "%S%Y%N%C%", 20), query.args());
}
} }
@@ -3,11 +3,13 @@ package com.dbx.agent.databend;
import com.dbx.agent.ConfiguredJdbcAgent; import com.dbx.agent.ConfiguredJdbcAgent;
import com.dbx.agent.JdbcAgentProfile; import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.PreparedStatement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
@@ -54,6 +56,23 @@ public final class DatabendAgent extends ConfiguredJdbcAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
boolean includeTables = normalized.includesTableLikeTypes();
boolean includeProcedures = includesProcedures(normalized);
if (includeProcedures && !includeTables) {
return listProcedures(schema, normalized);
}
if (includeTables && !includeProcedures) {
return super.listObjects(schema, normalized);
}
if (!includeTables && !includeProcedures) {
return List.of();
}
return normalized.filterObjects(listObjects(schema));
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
if (!"PROCEDURE".equalsIgnoreCase(objectType)) { if (!"PROCEDURE".equalsIgnoreCase(objectType)) {
@@ -80,6 +99,53 @@ public final class DatabendAgent extends ConfiguredJdbcAgent {
} }
} }
private List<ObjectInfo> listProcedures(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
useSchema(schema);
List<ObjectInfo> result = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT name, comment FROM system.procedures");
List<Object> args = new ArrayList<>();
if (constraints.hasFilter()) {
sql.append(" WHERE UPPER(name) LIKE ? ESCAPE '\\\\'");
args.add(constraints.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
sql.append(" ORDER BY name");
if (constraints.hasLimit()) {
sql.append(" LIMIT ?");
args.add(constraints.getLimit());
if (constraints.hasOffset()) {
sql.append(" OFFSET ?");
args.add(constraints.getOffset());
}
}
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(rs.getString("name"), "PROCEDURE", schema, rs.getString("comment")));
}
}
}
MetadataListConstraints guard = constraints.hasLimit() ? constraints.withoutPaging() : constraints;
return guard.filterObjects(result);
});
}
private static boolean includesProcedures(MetadataListConstraints constraints) {
return !constraints.hasObjectTypes() || constraints.objectTypeAllowed("PROCEDURE");
}
private static void bind(PreparedStatement stmt, List<Object> args) throws SQLException {
for (int index = 0; index < args.size(); index += 1) {
Object arg = args.get(index);
if (arg instanceof Integer) {
stmt.setInt(index + 1, (Integer) arg);
} else {
stmt.setString(index + 1, String.valueOf(arg));
}
}
}
private ProcedureMetadata findProcedure(String name) throws SQLException { private ProcedureMetadata findProcedure(String name) throws SQLException {
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement( try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(
"SELECT arguments, comment FROM system.procedures WHERE name = ? ORDER BY procedure_id LIMIT 1" "SELECT arguments, comment FROM system.procedures WHERE name = ? ORDER BY procedure_id LIMIT 1"
@@ -2,10 +2,24 @@ package com.dbx.agent.databend;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
class DatabendAgentTest extends JdbcFakeExecutionBehaviorTest { class DatabendAgentTest extends JdbcFakeExecutionBehaviorTest {
@Override @Override
protected DatabaseAgent createAgent() { protected DatabaseAgent createAgent() {
@@ -51,4 +65,132 @@ class DatabendAgentTest extends JdbcFakeExecutionBehaviorTest {
DatabendAgent.DATABEND_PROFILE.schemaSwitchSql("analytics\"prod") DatabendAgent.DATABEND_PROFILE.schemaSwitchSql("analytics\"prod")
); );
} }
@Test
void constrainedListObjectsKeepsDatabendProcedures() {
List<String> sql = new ArrayList<>();
DatabendAgent agent = new DatabendAgent();
TestSupport.setPrivateConnection(agent, databendMetadataConnection(sql, resultSet(
new String[]{"name", "comment"},
new Object[][]{
{"refresh_orders", "proc comment"},
{"format_order", null}
}
)));
List<ObjectInfo> objects = agent.listObjects(
"default",
new MetadataListConstraints("format", 1, null, List.of("PROCEDURE"))
);
Assertions.assertEquals(1, objects.size());
Assertions.assertEquals("format_order", objects.get(0).getName());
Assertions.assertEquals("PROCEDURE", objects.get(0).getObject_type());
Assertions.assertTrue(sql.stream().anyMatch(value -> value.contains("FROM system.procedures")), sql.toString());
Assertions.assertTrue(sql.stream().anyMatch(value -> value.contains("UPPER(name) LIKE ?")), sql.toString());
Assertions.assertTrue(sql.stream().anyMatch(value -> value.contains("LIMIT ?")), sql.toString());
}
private static Connection databendMetadataConnection(List<String> sql, ResultSet procedures) {
DatabaseMetaData metadata = proxy(DatabaseMetaData.class, (method, args) -> {
if ("getTableTypes".equals(method.getName()) || "getTables".equals(method.getName())) {
return emptyResultSet();
}
return defaultValue(method.getReturnType());
});
Statement statement = proxy(Statement.class, (method, args) -> {
if ("execute".equals(method.getName()) || "close".equals(method.getName())) {
return "execute".equals(method.getName()) ? false : null;
}
if ("executeQuery".equals(method.getName())) {
return procedures;
}
return defaultValue(method.getReturnType());
});
PreparedStatement preparedStatement = proxy(PreparedStatement.class, (method, args) -> {
if ("executeQuery".equals(method.getName())) {
return procedures;
}
if ("setString".equals(method.getName()) || "setInt".equals(method.getName()) || "close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
});
return proxy(Connection.class, (method, args) -> {
if ("getMetaData".equals(method.getName())) {
return metadata;
}
if ("createStatement".equals(method.getName())) {
return statement;
}
if ("prepareStatement".equals(method.getName())) {
sql.add(String.valueOf(args[0]));
return preparedStatement;
}
if ("isClosed".equals(method.getName())) {
return false;
}
return defaultValue(method.getReturnType());
});
}
private static ResultSet emptyResultSet() {
return resultSet(new String[]{"VALUE"}, new Object[][]{});
}
private static ResultSet resultSet(String[] columns, Object[][] rows) {
int[] index = {-1};
return proxy(ResultSet.class, (method, args) -> {
switch (method.getName()) {
case "next":
index[0] += 1;
return index[0] < rows.length;
case "getString":
Object value = columnValue(columns, rows[index[0]], args[0]);
return value == null ? null : String.valueOf(value);
case "close":
return null;
default:
return defaultValue(method.getReturnType());
}
});
}
private static Object columnValue(String[] columns, Object[] row, Object key) {
if (key instanceof Number) {
return row[((Number) key).intValue() - 1];
}
for (int i = 0; i < columns.length; i++) {
if (columns[i].equalsIgnoreCase(String.valueOf(key))) {
return row[i];
}
}
return null;
}
private static <T> T proxy(Class<T> type, MethodHandler handler) {
InvocationHandler invocationHandler = new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
return handler.handle(method, args == null ? new Object[0] : args);
}
};
return type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[]{type}, invocationHandler));
}
private static Object defaultValue(Class<?> type) {
if (type == Boolean.TYPE) return false;
if (type == Byte.TYPE) return (byte) 0;
if (type == Short.TYPE) return (short) 0;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
if (type == Float.TYPE) return 0f;
if (type == Double.TYPE) return 0d;
if (type == Character.TYPE) return (char) 0;
return null;
}
private interface MethodHandler {
Object handle(Method method, Object[] args) throws Throwable;
}
} }
@@ -10,6 +10,8 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor; import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
@@ -114,6 +116,44 @@ public final class Db2Agent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT TABNAME, TYPE FROM SYSCAT.TABLES WHERE TABSCHEMA = ?");
args.add(schema);
appendDb2TableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "TABNAME", constraints);
sql.append(" ORDER BY TABNAME");
MetadataSqlSupport.appendLiteralOffsetFetch(sql, constraints);
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString(1).trim(), db2TableType(rs.getString(2)), null));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -135,6 +175,63 @@ public final class Db2Agent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return queryConstrainedObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder(
"SELECT TABNAME AS OBJECT_NAME, CASE TYPE WHEN 'T' THEN 'TABLE' WHEN 'V' THEN 'VIEW' ELSE TYPE END AS OBJECT_TYPE FROM SYSCAT.TABLES WHERE TABSCHEMA = ?"
);
args.add(schema);
appendDb2TableTypePredicate(tableSql, args, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "TABNAME", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE")) {
StringBuilder procedureSql = new StringBuilder(
"SELECT PROCNAME AS OBJECT_NAME, 'PROCEDURE' AS OBJECT_TYPE FROM SYSCAT.PROCEDURES WHERE PROCSCHEMA = ?"
);
args.add(schema);
MetadataSqlSupport.appendNameFilter(procedureSql, args, "PROCNAME", constraints);
branches.add(procedureSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT OBJECT_NAME, OBJECT_TYPE FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY CASE OBJECT_TYPE WHEN 'TABLE' THEN 0 WHEN 'VIEW' THEN 1 WHEN 'PROCEDURE' THEN 2 ELSE 9 END, OBJECT_NAME");
MetadataSqlSupport.appendLiteralOffsetFetch(sql, constraints);
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(rs.getString(1).trim(), rs.getString(2), schema, null));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -331,6 +428,39 @@ public final class Db2Agent extends BaseDatabaseAgent {
return url + ":" + extraParams + (extraParams.endsWith(";") ? "" : ";"); return url + ":" + extraParams + (extraParams.endsWith(";") ? "" : ";");
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes() || constraints.objectTypeAllowed("PROCEDURE");
}
private static void appendDb2TableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("T");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("V");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static String db2TableType(String value) {
String db2Type = value == null ? "" : value.trim();
return switch (db2Type) {
case "T" -> "TABLE";
case "V" -> "VIEW";
default -> db2Type;
};
}
private static String trimDb2UrlParams(String urlParams) { private static String trimDb2UrlParams(String urlParams) {
String value = urlParams == null ? "" : urlParams.trim(); String value = urlParams == null ? "" : urlParams.trim();
while (value.startsWith("?") || value.startsWith("&") || value.startsWith(":") || value.startsWith(";")) { while (value.startsWith("?") || value.startsWith("&") || value.startsWith(":") || value.startsWith(";")) {
@@ -2,7 +2,9 @@ package com.dbx.agent.db2;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import com.dbx.agent.test.JdbcMetadataSqlFake;
import com.dbx.agent.test.TestSupport; import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -20,6 +22,7 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class Db2AgentTest extends JdbcFakeExecutionBehaviorTest { class Db2AgentTest extends JdbcFakeExecutionBehaviorTest {
@Override @Override
@@ -59,6 +62,35 @@ class Db2AgentTest extends JdbcFakeExecutionBehaviorTest {
assertFalse(executedSql.get().contains("OWNERTYPE")); assertFalse(executedSql.get().contains("OWNERTYPE"));
} }
@Test
void constrainedTableMetadataUsesDb2CatalogPushdown() {
Db2Agent agent = new Db2Agent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("APP", new MetadataListConstraints("ord", 25, 50, List.of("TABLE")));
String sql = JdbcMetadataSqlFake.statements.get(0);
assertTrue(sql.contains("FROM SYSCAT.TABLES"));
assertTrue(sql.contains("TYPE IN (?)"));
assertTrue(sql.contains("UPPER(TABNAME) LIKE ? ESCAPE '\\\\'"));
assertTrue(sql.endsWith("OFFSET 50 ROWS FETCH NEXT 25 ROWS ONLY"));
assertEquals(Arrays.asList("param:1=APP", "param:2=T", "param:3=%O%R%D%"), JdbcMetadataSqlFake.statements.subList(1, 4));
}
@Test
void constrainedObjectMetadataUsesDb2CatalogPushdown() {
Db2Agent agent = new Db2Agent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listObjects("APP", new MetadataListConstraints("sync", 10, null, List.of("PROCEDURE")));
String sql = JdbcMetadataSqlFake.statements.get(0);
assertTrue(sql.contains("FROM SYSCAT.PROCEDURES"));
assertTrue(sql.contains("ORDER BY CASE OBJECT_TYPE"));
assertTrue(sql.endsWith("FETCH FIRST 10 ROWS ONLY"));
assertEquals(Arrays.asList("param:1=APP", "param:2=%S%Y%N%C%"), JdbcMetadataSqlFake.statements.subList(1, 3));
}
private static Connection connection(AtomicReference<String> executedSql, ResultSet resultSet) { private static Connection connection(AtomicReference<String> executedSql, ResultSet resultSet) {
Statement statement = proxy(Statement.class, new MethodHandler() { Statement statement = proxy(Statement.class, new MethodHandler() {
@Override @Override
@@ -4,6 +4,7 @@ import com.dbx.agent.ConfiguredJdbcAgent;
import com.dbx.agent.ExecuteQueryOptions; import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.JdbcAgentProfile; import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.QueryPageOptions; import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryPageResult; import com.dbx.agent.QueryPageResult;
@@ -55,18 +56,34 @@ public final class Gbase8aAgent extends ConfiguredJdbcAgent {
@Override @Override
public List<TableInfo> listTables(String schema) { public List<TableInfo> listTables(String schema) {
return queryTables(schema, MetadataListConstraints.NONE);
}
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
return queryTables(schema, MetadataListConstraints.orNone(constraints));
}
private List<TableInfo> queryTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> { return unchecked(() -> {
List<TableInfo> result = new ArrayList<>(); if (!constraints.includesTableLikeTypes()) {
String sql; return List.of();
if (schema != null && !schema.trim().isEmpty()) {
sql = "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME";
} else {
sql = "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA NOT IN ('information_schema', 'performance_schema', 'gclusterdb', 'gctmpdb') ORDER BY TABLE_SCHEMA, TABLE_NAME";
} }
try (PreparedStatement stmt = requireConnection().prepareStatement(sql)) { List<TableInfo> result = new ArrayList<>();
if (schema != null && !schema.trim().isEmpty()) { List<Object> args = new ArrayList<>();
stmt.setString(1, schema); StringBuilder sql = new StringBuilder("SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE ");
} if (hasSchema(schema)) {
sql.append("TABLE_SCHEMA = ?");
args.add(schema);
} else {
sql.append("TABLE_SCHEMA NOT IN ('information_schema', 'performance_schema', 'gclusterdb', 'gctmpdb')");
}
appendGbase8aTableTypePredicate(sql, args, constraints);
appendNameFilter(sql, args, "TABLE_NAME", constraints);
sql.append(hasSchema(schema) ? " ORDER BY TABLE_NAME" : " ORDER BY TABLE_SCHEMA, TABLE_NAME");
appendLimitOffset(sql, args, constraints);
try (PreparedStatement stmt = requireConnection().prepareStatement(sql.toString())) {
bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) { try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) { while (rs.next()) {
String tableType = rs.getString("TABLE_TYPE"); String tableType = rs.getString("TABLE_TYPE");
@@ -78,32 +95,63 @@ public final class Gbase8aAgent extends ConfiguredJdbcAgent {
} }
} }
result.sort(Comparator.comparing(TableInfo::getName)); result.sort(Comparator.comparing(TableInfo::getName));
return result; MetadataListConstraints guard = constraints.hasLimit() ? constraints.withoutPaging() : constraints;
return guard.filterTables(result);
}); });
} }
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return queryObjects(schema, MetadataListConstraints.NONE);
}
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
return queryObjects(schema, MetadataListConstraints.orNone(constraints));
}
private List<ObjectInfo> queryObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> { return unchecked(() -> {
List<ObjectInfo> result = StandardJdbcMetadata.INSTANCE.listObjects(listTables(schema), schema); boolean includeTables = constraints.includesTableLikeTypes();
// GBase 8a's table metadata does not surface routines, so sidebar objects must load them explicitly. boolean includeRoutines = includesRoutineTypes(constraints);
appendRoutines(result, schema); if (!includeTables && !includeRoutines) {
return result; return List.of();
}
List<ObjectInfo> result = new ArrayList<>();
if (includeTables) {
MetadataListConstraints tableConstraints = includeRoutines
? new MetadataListConstraints(constraints.getFilter(), null, null, tableLikeObjectTypes(constraints))
: constraints;
result.addAll(StandardJdbcMetadata.INSTANCE.listObjects(queryTables(schema, tableConstraints), schema));
}
if (includeRoutines) {
MetadataListConstraints routineConstraints = includeTables
? new MetadataListConstraints(constraints.getFilter(), null, null, routineObjectTypes(constraints))
: constraints;
appendRoutines(result, schema, routineConstraints);
}
boolean singlePagedSource = includeTables != includeRoutines && constraints.hasLimit();
MetadataListConstraints guard = singlePagedSource ? constraints.withoutPaging() : constraints;
return guard.filterObjects(result);
}); });
} }
private void appendRoutines(List<ObjectInfo> result, String schema) throws Exception { private void appendRoutines(List<ObjectInfo> result, String schema, MetadataListConstraints constraints) throws Exception {
String sql; StringBuilder sql = new StringBuilder();
boolean hasSchema = schema != null && !schema.trim().isEmpty(); List<Object> args = new ArrayList<>();
boolean hasSchema = hasSchema(schema);
if (hasSchema) { if (hasSchema) {
sql = "SELECT ROUTINE_NAME, ROUTINE_TYPE, ROUTINE_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = ? ORDER BY ROUTINE_NAME"; sql.append("SELECT ROUTINE_NAME, ROUTINE_TYPE, ROUTINE_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = ?");
args.add(schema);
} else { } else {
sql = "SELECT ROUTINE_SCHEMA, ROUTINE_NAME, ROUTINE_TYPE, ROUTINE_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA NOT IN ('information_schema', 'performance_schema', 'gclusterdb', 'gctmpdb') ORDER BY ROUTINE_SCHEMA, ROUTINE_NAME"; sql.append("SELECT ROUTINE_SCHEMA, ROUTINE_NAME, ROUTINE_TYPE, ROUTINE_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA NOT IN ('information_schema', 'performance_schema', 'gclusterdb', 'gctmpdb')");
} }
try (PreparedStatement stmt = requireConnection().prepareStatement(sql)) { appendGbase8aRoutineTypePredicate(sql, args, constraints);
if (hasSchema) { appendNameFilter(sql, args, "ROUTINE_NAME", constraints);
stmt.setString(1, schema); sql.append(hasSchema ? " ORDER BY ROUTINE_NAME" : " ORDER BY ROUTINE_SCHEMA, ROUTINE_NAME");
} appendLimitOffset(sql, args, constraints);
try (PreparedStatement stmt = requireConnection().prepareStatement(sql.toString())) {
bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) { try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) { while (rs.next()) {
String routineSchema = hasSchema ? schema : rs.getString("ROUTINE_SCHEMA"); String routineSchema = hasSchema ? schema : rs.getString("ROUTINE_SCHEMA");
@@ -118,6 +166,126 @@ public final class Gbase8aAgent extends ConfiguredJdbcAgent {
} }
} }
private static void appendGbase8aTableTypePredicate(
StringBuilder sql,
List<Object> args,
MetadataListConstraints constraints
) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> tableTypes = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
tableTypes.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
tableTypes.add("VIEW");
}
if (tableTypes.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND TABLE_TYPE IN (").append(placeholders(tableTypes.size())).append(")");
args.addAll(tableTypes);
}
private static void appendGbase8aRoutineTypePredicate(
StringBuilder sql,
List<Object> args,
MetadataListConstraints constraints
) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> routineTypes = new ArrayList<>();
if (constraints.objectTypeAllowed("PROCEDURE")) {
routineTypes.add("PROCEDURE");
}
if (constraints.objectTypeAllowed("FUNCTION")) {
routineTypes.add("FUNCTION");
}
if (routineTypes.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND ROUTINE_TYPE IN (").append(placeholders(routineTypes.size())).append(")");
args.addAll(routineTypes);
}
private static void appendNameFilter(StringBuilder sql, List<Object> args, String column, MetadataListConstraints constraints) {
if (!constraints.hasFilter()) {
return;
}
sql.append(" AND UPPER(").append(column).append(") LIKE ? ESCAPE '\\\\'");
args.add(constraints.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
private static void appendLimitOffset(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasLimit()) {
return;
}
sql.append(" LIMIT ?");
args.add(constraints.getLimit());
if (constraints.hasOffset()) {
sql.append(" OFFSET ?");
args.add(constraints.getOffset());
}
}
private static boolean hasSchema(String schema) {
return schema != null && !schema.trim().isEmpty();
}
private static boolean includesRoutineTypes(MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return true;
}
return constraints.objectTypeAllowed("PROCEDURE") || constraints.objectTypeAllowed("FUNCTION");
}
private static List<String> tableLikeObjectTypes(MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return null;
}
List<String> result = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
result.add("TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
result.add("VIEW");
}
return result;
}
private static List<String> routineObjectTypes(MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return null;
}
List<String> result = new ArrayList<>();
if (constraints.objectTypeAllowed("PROCEDURE")) {
result.add("PROCEDURE");
}
if (constraints.objectTypeAllowed("FUNCTION")) {
result.add("FUNCTION");
}
return result;
}
private static String placeholders(int count) {
return String.join(", ", java.util.Collections.nCopies(count, "?"));
}
private static void bind(PreparedStatement stmt, List<Object> args) throws Exception {
for (int index = 0; index < args.size(); index += 1) {
Object arg = args.get(index);
if (arg instanceof Integer) {
stmt.setInt(index + 1, (Integer) arg);
} else {
stmt.setString(index + 1, String.valueOf(arg));
}
}
}
private static String normalizeRoutineType(String routineType) { private static String normalizeRoutineType(String routineType) {
if (routineType == null || routineType.trim().isEmpty()) { if (routineType == null || routineType.trim().isEmpty()) {
return "PROCEDURE"; return "PROCEDURE";
@@ -1,6 +1,8 @@
package com.dbx.agent.gbase8a; package com.dbx.agent.gbase8a;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.TableInfo;
import com.dbx.agent.test.TestSupport; import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -46,6 +48,53 @@ class Gbase8aAgentTest {
Assertions.assertTrue(sql.get(1).contains("ROUTINE_SCHEMA = ?"), sql.get(1)); Assertions.assertTrue(sql.get(1).contains("ROUTINE_SCHEMA = ?"), sql.get(1));
} }
@Test
void constrainedListTablesPushesFilterTypeAndLimit() {
List<String> sql = new ArrayList<>();
Gbase8aAgent agent = new Gbase8aAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql,
resultSet(
new String[]{"TABLE_NAME", "TABLE_TYPE"},
new Object[][]{{"user_order", "BASE TABLE"}}
)
));
List<TableInfo> tables = agent.listTables(
"app",
new MetadataListConstraints("user", 1, 1, List.of("TABLE"))
);
Assertions.assertEquals(1, tables.size());
Assertions.assertEquals("user_order", tables.get(0).getName());
Assertions.assertTrue(sql.get(0).contains("TABLE_TYPE IN (?)"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("UPPER(TABLE_NAME) LIKE ?"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("LIMIT ?"), sql.get(0));
}
@Test
void constrainedListObjectsKeepsCustomRoutineMetadata() {
List<String> sql = new ArrayList<>();
Gbase8aAgent agent = new Gbase8aAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql,
resultSet(
new String[]{"ROUTINE_NAME", "ROUTINE_TYPE", "ROUTINE_COMMENT"},
new Object[][]{{"refresh_orders", "PROCEDURE", "proc comment"}, {"format_order", "FUNCTION", null}}
)
));
List<ObjectInfo> objects = agent.listObjects(
"app",
new MetadataListConstraints("format", 1, null, List.of("FUNCTION"))
);
Assertions.assertEquals(1, objects.size());
Assertions.assertEquals("format_order", objects.get(0).getName());
Assertions.assertEquals("FUNCTION", objects.get(0).getObject_type());
Assertions.assertTrue(sql.get(0).contains("ROUTINE_TYPE IN (?)"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("UPPER(ROUTINE_NAME) LIKE ?"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("LIMIT ?"), sql.get(0));
}
private static Connection preparedConnection(List<String> sql, ResultSet... resultSets) { private static Connection preparedConnection(List<String> sql, ResultSet... resultSets) {
int[] resultSetIndex = {0}; int[] resultSetIndex = {0};
PreparedStatement statement = proxy(PreparedStatement.class, (method, args) -> { PreparedStatement statement = proxy(PreparedStatement.class, (method, args) -> {
@@ -54,7 +103,7 @@ class Gbase8aAgentTest {
resultSetIndex[0] += 1; resultSetIndex[0] += 1;
return resultSets[current]; return resultSets[current];
} }
if ("setString".equals(method.getName()) || "close".equals(method.getName())) { if ("setString".equals(method.getName()) || "setInt".equals(method.getName()) || "close".equals(method.getName())) {
return null; return null;
} }
return defaultValue(method.getReturnType()); return defaultValue(method.getReturnType());
@@ -6,6 +6,7 @@ import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.ExecuteQueryOptions; import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.JdbcAgentProfile; import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
import java.sql.Connection; import java.sql.Connection;
@@ -185,6 +186,60 @@ public final class Gbase8sAgent extends ConfiguredJdbcAgent {
} }
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
return queryConstrainedTables(schema, normalized);
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
if (!constraints.includesTableLikeTypes()) {
return List.of();
}
try {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
String owner = trim(schema);
// GBase 8s follows Informix-style SKIP/FIRST pagination in the SELECT list.
StringBuilder sql = new StringBuilder("SELECT ");
if (constraints.hasOffset()) {
sql.append("SKIP ").append(constraints.getOffset()).append(' ');
}
if (constraints.hasLimit()) {
sql.append("FIRST ").append(constraints.getLimit()).append(' ');
}
sql.append("tabname, tabtype FROM systables WHERE tabid >= 100");
appendGbase8sTableTypePredicate(sql, constraints);
if (!owner.isEmpty()) {
sql.append(" AND owner = ?");
args.add(owner);
}
if (constraints.hasFilter()) {
sql.append(" AND UPPER(tabname) LIKE ? ESCAPE '\\\\'");
args.add(constraints.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
sql.append(" ORDER BY tabname");
try (PreparedStatement stmt = requireConnection().prepareStatement(sql.toString())) {
bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(
trim(rs.getString("tabname")),
tableType(rs.getString("tabtype"))
));
}
}
}
result.sort(Comparator.comparing(TableInfo::getName));
return constraints.withoutPaging().filterTables(result);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static void main(String[] args) { public static void main(String[] args) {
new JsonRpcServer(new Gbase8sAgent()).run(); new JsonRpcServer(new Gbase8sAgent()).run();
} }
@@ -281,6 +336,38 @@ public final class Gbase8sAgent extends ConfiguredJdbcAgent {
return "V".equalsIgnoreCase(trim(tabtype)) ? "VIEW" : "TABLE"; return "V".equalsIgnoreCase(trim(tabtype)) ? "VIEW" : "TABLE";
} }
private static void appendGbase8sTableTypePredicate(StringBuilder sql, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
sql.append(" AND tabtype IN ('T', 'V')");
return;
}
List<String> tabTypes = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
tabTypes.add("'T'");
}
if (constraints.tableTypeAllowed("VIEW")) {
tabTypes.add("'V'");
}
if (tabTypes.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND tabtype IN (").append(String.join(", ", tabTypes)).append(")");
}
private static void bind(PreparedStatement stmt, List<Object> args) throws Exception {
for (int index = 0; index < args.size(); index += 1) {
stmt.setString(index + 1, String.valueOf(args.get(index)));
}
}
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter()
&& !constraints.hasLimit()
&& !constraints.hasOffset()
&& !constraints.hasObjectTypes();
}
private static String trim(String value) { private static String trim(String value) {
return value == null ? "" : value.trim(); return value == null ? "" : value.trim();
} }
@@ -1,9 +1,21 @@
package com.dbx.agent.gbase8s; package com.dbx.agent.gbase8s;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.TableInfo;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
class Gbase8sAgentTest { class Gbase8sAgentTest {
@Test @Test
void declaresGbase8sProfile() { void declaresGbase8sProfile() {
@@ -95,4 +107,108 @@ class Gbase8sAgentTest {
Assertions.assertEquals("jdbc:gbasedbt-sqli://db.example.com:20013/app:GBASEDBTSERVER=gbase01", url); Assertions.assertEquals("jdbc:gbasedbt-sqli://db.example.com:20013/app:GBASEDBTSERVER=gbase01", url);
} }
@Test
void constrainedListTablesUsesGbase8sSystemTableQuery() {
List<String> sql = new ArrayList<>();
Gbase8sAgent agent = new Gbase8sAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql, resultSet(
new String[]{"tabname", "tabtype"},
new Object[][]{
{"user_order", "T"}
}
)));
List<TableInfo> tables = agent.listTables(
"app",
new MetadataListConstraints("user", 1, 1, List.of("TABLE"))
);
Assertions.assertEquals(1, tables.size());
Assertions.assertEquals("user_order", tables.get(0).getName());
Assertions.assertTrue(sql.get(0).contains("FROM systables"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("SELECT SKIP 1 FIRST 1"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("UPPER(tabname) LIKE ?"), sql.get(0));
}
private static Connection preparedConnection(List<String> sql, ResultSet resultSet) {
PreparedStatement statement = proxy(PreparedStatement.class, (method, args) -> {
if ("executeQuery".equals(method.getName())) {
return resultSet;
}
if ("setString".equals(method.getName()) || "close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
});
return proxy(Connection.class, (method, args) -> {
if ("getCatalog".equals(method.getName())) {
return "appdb";
}
if ("prepareStatement".equals(method.getName())) {
sql.add(String.valueOf(args[0]));
return statement;
}
if ("isClosed".equals(method.getName())) {
return false;
}
return defaultValue(method.getReturnType());
});
}
private static ResultSet resultSet(String[] columns, Object[][] rows) {
int[] index = {-1};
return proxy(ResultSet.class, (method, args) -> {
switch (method.getName()) {
case "next":
index[0] += 1;
return index[0] < rows.length;
case "getString":
Object value = columnValue(columns, rows[index[0]], args[0]);
return value == null ? null : String.valueOf(value);
case "close":
return null;
default:
return defaultValue(method.getReturnType());
}
});
}
private static Object columnValue(String[] columns, Object[] row, Object key) {
if (key instanceof Number) {
return row[((Number) key).intValue() - 1];
}
for (int i = 0; i < columns.length; i++) {
if (columns[i].equalsIgnoreCase(String.valueOf(key))) {
return row[i];
}
}
return null;
}
private static <T> T proxy(Class<T> type, MethodHandler handler) {
InvocationHandler invocationHandler = new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
return handler.handle(method, args == null ? new Object[0] : args);
}
};
return type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[]{type}, invocationHandler));
}
private static Object defaultValue(Class<?> type) {
if (type == Boolean.TYPE) return false;
if (type == Byte.TYPE) return (byte) 0;
if (type == Short.TYPE) return (short) 0;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
if (type == Float.TYPE) return 0f;
if (type == Double.TYPE) return 0d;
if (type == Character.TYPE) return (char) 0;
return null;
}
private interface MethodHandler {
Object handle(Method method, Object[] args) throws Throwable;
}
} }
@@ -10,6 +10,8 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor; import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
@@ -99,6 +101,44 @@ public final class GoldendbAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT TABLE_NAME, TABLE_TYPE, TABLE_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?");
args.add(schema);
appendInformationSchemaTableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "TABLE_NAME", constraints);
sql.append(" ORDER BY TABLE_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString("TABLE_NAME"), normalizeTableType(rs.getString("TABLE_TYPE")), emptyToNull(rs.getString("TABLE_COMMENT"))));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -121,6 +161,60 @@ public final class GoldendbAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return queryConstrainedObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT TABLE_NAME AS OBJECT_NAME, TABLE_TYPE AS OBJECT_TYPE, TABLE_COMMENT AS OBJECT_COMMENT FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?");
args.add(schema);
appendInformationSchemaTableTypePredicate(tableSql, args, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "TABLE_NAME", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE") || constraints.objectTypeAllowed("FUNCTION")) {
StringBuilder routineSql = new StringBuilder("SELECT ROUTINE_NAME AS OBJECT_NAME, ROUTINE_TYPE AS OBJECT_TYPE, NULL AS OBJECT_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = ?");
args.add(schema);
appendRoutineTypePredicate(routineSql, args, constraints);
MetadataSqlSupport.appendNameFilter(routineSql, args, "ROUTINE_NAME", constraints);
branches.add(routineSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT OBJECT_NAME, OBJECT_TYPE, OBJECT_COMMENT FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY CASE OBJECT_TYPE WHEN 'BASE TABLE' THEN 0 WHEN 'TABLE' THEN 0 WHEN 'VIEW' THEN 1 WHEN 'PROCEDURE' THEN 2 WHEN 'FUNCTION' THEN 3 ELSE 9 END, OBJECT_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(rs.getString("OBJECT_NAME"), normalizeTableType(rs.getString("OBJECT_TYPE")), schema, emptyToNull(rs.getString("OBJECT_COMMENT"))));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -379,6 +473,58 @@ public final class GoldendbAgent extends BaseDatabaseAgent {
+ "?useSSL=false&allowPublicKeyRetrieval=true"; + "?useSSL=false&allowPublicKeyRetrieval=true";
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes()
|| constraints.objectTypeAllowed("PROCEDURE")
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendInformationSchemaTableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND TABLE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static void appendRoutineTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.objectTypeAllowed("PROCEDURE")) {
types.add("PROCEDURE");
}
if (constraints.objectTypeAllowed("FUNCTION")) {
types.add("FUNCTION");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND ROUTINE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static String normalizeTableType(String tableType) {
return "BASE TABLE".equals(tableType) ? "TABLE" : tableType;
}
private static String emptyToNull(String value) { private static String emptyToNull(String value) {
return value == null || value.isEmpty() ? null : value; return value == null || value.isEmpty() ? null : value;
} }
@@ -1,5 +1,6 @@
package com.dbx.agent.goldendb; package com.dbx.agent.goldendb;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.TestSupport; import com.dbx.agent.test.TestSupport;
import com.dbx.agent.test.JdbcAgentFake; import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.JdbcMetadataSqlFake; import com.dbx.agent.test.JdbcMetadataSqlFake;
@@ -22,4 +23,32 @@ class GoldendbAgentMetadataTest {
JdbcMetadataSqlFake.statements JdbcMetadataSqlFake.statements
); );
} }
@Test
void constrainedTableMetadataPushesFilterTypesAndPaging() {
GoldendbAgent agent = new GoldendbAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("app", new MetadataListConstraints("ord", 25, 50, List.of("TABLE", "VIEW")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM information_schema.TABLES"), sql);
Assertions.assertTrue(sql.contains("TABLE_TYPE IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("UPPER(TABLE_NAME) LIKE ? ESCAPE '\\\\'"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 25 OFFSET 50"), sql);
}
@Test
void constrainedObjectMetadataPushesRoutineTypesAndPaging() {
GoldendbAgent agent = new GoldendbAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listObjects("app", new MetadataListConstraints("sync", 10, null, List.of("PROCEDURE", "FUNCTION")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM information_schema.ROUTINES"), sql);
Assertions.assertTrue(sql.contains("ROUTINE_TYPE IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("ORDER BY CASE OBJECT_TYPE"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 10"), sql);
}
} }
@@ -9,6 +9,8 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor; import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
@@ -93,6 +95,45 @@ public final class H2Agent extends AbstractJdbcAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
String effectiveSchema = resolveSchema(schema);
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?");
args.add(effectiveSchema);
appendH2TableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "TABLE_NAME", constraints);
sql.append(" ORDER BY TABLE_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (var stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString("TABLE_NAME"), normalizeTableType(rs.getString("TABLE_TYPE")), null));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -116,6 +157,61 @@ public final class H2Agent extends AbstractJdbcAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return queryConstrainedObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
String effectiveSchema = resolveSchema(schema);
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT TABLE_NAME AS OBJECT_NAME, TABLE_TYPE AS OBJECT_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?");
args.add(effectiveSchema);
appendH2TableTypePredicate(tableSql, args, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "TABLE_NAME", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE") || constraints.objectTypeAllowed("FUNCTION")) {
StringBuilder routineSql = new StringBuilder("SELECT ROUTINE_NAME AS OBJECT_NAME, ROUTINE_TYPE AS OBJECT_TYPE FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = ?");
args.add(effectiveSchema);
appendRoutineTypePredicate(routineSql, args, constraints);
MetadataSqlSupport.appendNameFilter(routineSql, args, "ROUTINE_NAME", constraints);
branches.add(routineSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT OBJECT_NAME, OBJECT_TYPE FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY CASE OBJECT_TYPE WHEN 'BASE TABLE' THEN 0 WHEN 'TABLE' THEN 0 WHEN 'VIEW' THEN 1 WHEN 'PROCEDURE' THEN 2 WHEN 'FUNCTION' THEN 3 ELSE 9 END, OBJECT_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (var stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(rs.getString(1), normalizeTableType(rs.getString(2)), schema, null));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -346,6 +442,58 @@ public final class H2Agent extends AbstractJdbcAgent {
return "jdbc:h2:tcp://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase(); return "jdbc:h2:tcp://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase();
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes()
|| constraints.objectTypeAllowed("PROCEDURE")
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendH2TableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND TABLE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static void appendRoutineTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.objectTypeAllowed("PROCEDURE")) {
types.add("PROCEDURE");
}
if (constraints.objectTypeAllowed("FUNCTION")) {
types.add("FUNCTION");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND ROUTINE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static String normalizeTableType(String tableType) {
return "BASE TABLE".equals(tableType) ? "TABLE" : tableType;
}
private static String resolveSchema(String schema) { private static String resolveSchema(String schema) {
if ("PUBLIC".equalsIgnoreCase(schema) || "INFORMATION_SCHEMA".equalsIgnoreCase(schema)) { if ("PUBLIC".equalsIgnoreCase(schema) || "INFORMATION_SCHEMA".equalsIgnoreCase(schema)) {
return schema.toUpperCase(Locale.ROOT); return schema.toUpperCase(Locale.ROOT);
@@ -4,7 +4,9 @@ import com.dbx.agent.BaseDatabaseAgent;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.ExecuteQueryOptions; import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
import com.dbx.agent.TableInfo;
import com.dbx.agent.test.JdbcExecutionBehaviorTest; import com.dbx.agent.test.JdbcExecutionBehaviorTest;
import com.dbx.agent.test.JdbcMetadataBehaviorTest; import com.dbx.agent.test.JdbcMetadataBehaviorTest;
import java.util.List; import java.util.List;
@@ -132,6 +134,23 @@ class H2MetadataBehaviorTest extends JdbcMetadataBehaviorTest {
protected List<String> expectedColumnsInOrder() { protected List<String> expectedColumnsInOrder() {
return List.of("ID", "NAME", "CREATED_AT"); return List.of("ID", "NAME", "CREATED_AT");
} }
@Test
void constrainedTableMetadataFiltersTypesAndPages() {
withAgent("dbx-agent-h2-constrained-metadata", agent -> {
for (String sql : metadataFixtureSql()) {
agent.executeQuery(sql, null, new ExecuteQueryOptions());
}
List<TableInfo> tables = agent.listTables(
metadataSchema(),
new MetadataListConstraints("table", 1, 1, List.of("TABLE"))
);
Assertions.assertEquals(1, tables.size());
Assertions.assertEquals("BETA_TABLE", tables.get(0).getName());
});
}
} }
final class H2AgentTestSupport { final class H2AgentTestSupport {
@@ -9,6 +9,8 @@ import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo; import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor; import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
@@ -16,6 +18,7 @@ import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo; import com.dbx.agent.TriggerInfo;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
@@ -205,6 +208,45 @@ public final class InformixAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT ");
MetadataSqlSupport.appendLiteralSkipFirst(sql, constraints);
sql.append("tabname, CASE tabtype WHEN 'T' THEN 'TABLE' WHEN 'V' THEN 'VIEW' ELSE tabtype END ")
.append("FROM systables WHERE tabid >= 100");
appendInformixTableTypePredicate(sql, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "tabname", constraints);
sql.append(" ORDER BY tabname");
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString(1).trim(), rs.getString(2).trim(), null));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -234,6 +276,63 @@ public final class InformixAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return queryConstrainedObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT tabname AS object_name, CASE tabtype WHEN 'T' THEN 'TABLE' WHEN 'V' THEN 'VIEW' ELSE tabtype END AS object_type, 0 AS object_order FROM systables WHERE tabid >= 100");
appendInformixTableTypePredicate(tableSql, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "tabname", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("FUNCTION")) {
StringBuilder functionSql = new StringBuilder("SELECT procname AS object_name, 'FUNCTION' AS object_type, 1 AS object_order FROM sysprocedures WHERE owner != 'informix' AND isproc = 'f'");
MetadataSqlSupport.appendNameFilter(functionSql, args, "procname", constraints);
branches.add(functionSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE")) {
StringBuilder procedureSql = new StringBuilder("SELECT procname AS object_name, 'PROCEDURE' AS object_type, 2 AS object_order FROM sysprocedures WHERE owner != 'informix' AND isproc = 't'");
MetadataSqlSupport.appendNameFilter(procedureSql, args, "procname", constraints);
branches.add(procedureSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT ");
MetadataSqlSupport.appendLiteralSkipFirst(sql, constraints);
sql.append("object_name, object_type FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY object_order, object_name");
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(rs.getString(1).trim(), rs.getString(2).trim(), schema, null));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -394,6 +493,34 @@ public final class InformixAgent extends BaseDatabaseAgent {
}); });
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes()
|| constraints.objectTypeAllowed("PROCEDURE")
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendInformixTableTypePredicate(StringBuilder sql, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("'T'");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("'V'");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND tabtype IN (").append(String.join(", ", types)).append(")");
}
private static String trimStart(String value, char... chars) { private static String trimStart(String value, char... chars) {
int index = 0; int index = 0;
while (index < value.length() && contains(chars, value.charAt(index))) { while (index < value.length() && contains(chars, value.charAt(index))) {
@@ -1,7 +1,11 @@
package com.dbx.agent.informix; package com.dbx.agent.informix;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.JdbcMetadataSqlFake;
import com.dbx.agent.test.TestSupport;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.Set; import java.util.Set;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -125,4 +129,33 @@ class InformixAgentTest {
void listsDatabasesFromSysmasterCatalog() { void listsDatabasesFromSysmasterCatalog() {
Assertions.assertEquals("SELECT name FROM sysmaster:sysdatabases ORDER BY name", InformixAgent.databaseCatalogSql()); Assertions.assertEquals("SELECT name FROM sysmaster:sysdatabases ORDER BY name", InformixAgent.databaseCatalogSql());
} }
@Test
void constrainedTableMetadataUsesInformixSkipFirstPushdown() {
InformixAgent agent = new InformixAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("stores", new MetadataListConstraints("ord", 25, 50, List.of("TABLE")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.startsWith("SELECT SKIP 50 FIRST 25 tabname"), sql);
Assertions.assertTrue(sql.contains("tabtype IN ('T')"), sql);
Assertions.assertTrue(sql.contains("UPPER(tabname) LIKE ? ESCAPE '\\\\'"), sql);
Assertions.assertTrue(sql.endsWith("ORDER BY tabname"), sql);
}
@Test
void constrainedObjectMetadataUsesInformixUnionPushdown() {
InformixAgent agent = new InformixAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listObjects("stores", new MetadataListConstraints("sync", 10, null, List.of("PROCEDURE", "FUNCTION")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.startsWith("SELECT FIRST 10 object_name, object_type FROM ("), sql);
Assertions.assertTrue(sql.contains("FROM sysprocedures"), sql);
Assertions.assertTrue(sql.contains("isproc = 'f'"), sql);
Assertions.assertTrue(sql.contains("isproc = 't'"), sql);
Assertions.assertTrue(sql.endsWith("ORDER BY object_order, object_name"), sql);
}
} }
@@ -7,6 +7,8 @@ import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo; import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.PostgresLikeAgent; import com.dbx.agent.PostgresLikeAgent;
@@ -125,6 +127,84 @@ public final class KingbaseAgent extends PostgresLikeAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return isMysqlCompatMode()
? queryMysqlCompatTables(schema, normalized)
: queryRegularTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryRegularTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT c.relname AS table_name, ")
.append("CASE c.relkind ")
.append("WHEN 'r' THEN 'TABLE' ")
.append("WHEN 'p' THEN 'TABLE' ")
.append("WHEN 'v' THEN 'VIEW' ")
.append("WHEN 'm' THEN 'MATERIALIZED_VIEW' ")
.append("WHEN 'f' THEN 'FOREIGN_TABLE' ")
.append("ELSE 'TABLE' END AS table_type, ")
.append("d.description AS table_comment ")
.append("FROM sys_catalog.sys_class c ")
.append("JOIN sys_catalog.sys_namespace n ON n.oid = c.relnamespace ")
.append("LEFT JOIN sys_catalog.sys_description d ON d.objoid = c.oid AND d.objsubid = 0 ")
.append("WHERE n.nspname = ? AND c.relkind IN ('r','p','v','m','f')");
args.add(effectiveSchema(schema));
appendRelkindPredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "c.relname", constraints);
sql.append(" ORDER BY c.relname");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString("table_name"),
normalizeTableType(rs.getString("table_type")),
rs.getString("table_comment")
));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
private List<TableInfo> queryMysqlCompatTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = ?");
args.add(effectiveSchema(schema));
appendMysqlCompatTableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "table_name", constraints);
sql.append(" ORDER BY table_name");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString(1), normalizeTableType(rs.getString(2))));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -162,6 +242,80 @@ public final class KingbaseAgent extends PostgresLikeAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return isMysqlCompatMode()
? normalized.filterObjects(toObjects(queryMysqlCompatTables(schema, normalized), effectiveSchema(schema)))
: queryRegularObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryRegularObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
String effectiveSchema = effectiveSchema(schema);
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT c.relname AS object_name, ")
.append("CASE c.relkind WHEN 'r' THEN 'TABLE' WHEN 'p' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED_VIEW' WHEN 'f' THEN 'FOREIGN_TABLE' ELSE 'TABLE' END AS object_type, ")
.append("d.description AS object_comment ")
.append("FROM sys_catalog.sys_class c ")
.append("JOIN sys_catalog.sys_namespace n ON n.oid = c.relnamespace ")
.append("LEFT JOIN sys_catalog.sys_description d ON d.objoid = c.oid AND d.objsubid = 0 ")
.append("WHERE n.nspname = ? AND c.relkind IN ('r','p','v','m','f')");
args.add(effectiveSchema);
appendRelkindPredicate(tableSql, args, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "c.relname", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE") || constraints.objectTypeAllowed("FUNCTION")) {
StringBuilder routineSql = new StringBuilder("SELECT p.proname AS object_name, ")
.append("CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS object_type, ")
.append("d.description AS object_comment ")
.append("FROM sys_catalog.sys_proc p ")
.append("JOIN sys_catalog.sys_namespace n ON n.oid = p.pronamespace ")
.append("LEFT JOIN sys_catalog.sys_description d ON d.objoid = p.oid AND d.objsubid = 0 ")
.append("WHERE n.nspname = ?");
args.add(effectiveSchema);
appendRoutineKindPredicate(routineSql, args, constraints);
MetadataSqlSupport.appendNameFilter(routineSql, args, "p.proname", constraints);
branches.add(routineSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT object_name, object_type, object_comment FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY CASE object_type WHEN 'TABLE' THEN 0 WHEN 'VIEW' THEN 1 WHEN 'MATERIALIZED_VIEW' THEN 2 WHEN 'FOREIGN_TABLE' THEN 3 WHEN 'PROCEDURE' THEN 4 WHEN 'FUNCTION' THEN 5 ELSE 9 END, object_name");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(
rs.getString("object_name"),
rs.getString("object_type"),
effectiveSchema,
rs.getString("object_comment")
));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
if ("FUNCTION".equalsIgnoreCase(objectType) || "PROCEDURE".equalsIgnoreCase(objectType)) { if ("FUNCTION".equalsIgnoreCase(objectType) || "PROCEDURE".equalsIgnoreCase(objectType)) {
@@ -465,6 +619,83 @@ public final class KingbaseAgent extends PostgresLikeAgent {
}); });
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes()
|| constraints.objectTypeAllowed("PROCEDURE")
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendRelkindPredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> kinds = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
kinds.add("r");
kinds.add("p");
}
if (constraints.tableTypeAllowed("VIEW")) {
kinds.add("v");
}
if (constraints.tableTypeAllowed("MATERIALIZED_VIEW")) {
kinds.add("m");
}
if (kinds.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND c.relkind IN (").append(MetadataSqlSupport.placeholders(kinds.size())).append(")");
args.addAll(kinds);
}
private static void appendMysqlCompatTableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
sql.append(" AND table_type IN ('BASE TABLE', 'VIEW')");
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND table_type IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static void appendRoutineKindPredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
List<String> kinds = new ArrayList<>();
if (!constraints.hasObjectTypes() || constraints.objectTypeAllowed("PROCEDURE")) {
kinds.add("p");
}
if (!constraints.hasObjectTypes() || constraints.objectTypeAllowed("FUNCTION")) {
kinds.add("f");
}
if (kinds.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND p.prokind IN (").append(MetadataSqlSupport.placeholders(kinds.size())).append(")");
args.addAll(kinds);
}
private static List<ObjectInfo> toObjects(List<TableInfo> tables, String schema) {
List<ObjectInfo> result = new ArrayList<>();
for (TableInfo table : tables) {
result.add(new ObjectInfo(table.getName(), table.getTable_type(), schema, table.getComment()));
}
return result;
}
private String effectiveSchema(String schema) { private String effectiveSchema(String schema) {
if (schema != null && !schema.trim().isEmpty()) { if (schema != null && !schema.trim().isEmpty()) {
return schema; return schema;
@@ -4,6 +4,7 @@ import com.dbx.agent.ColumnInfo;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.DatabaseInfo; import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.IndexInfo; import com.dbx.agent.IndexInfo;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
@@ -153,6 +154,58 @@ class KingbaseAgentTest extends JdbcFakeExecutionBehaviorTest {
Assertions.assertTrue(sql.get(1).contains("p.prokind IN ('p','f')"), sql.get(1)); Assertions.assertTrue(sql.get(1).contains("p.prokind IN ('p','f')"), sql.get(1));
} }
@Test
void constrainedRegularTableMetadataPushesFilterTypesAndPaging() {
List<String> sql = new ArrayList<>();
KingbaseAgent agent = new KingbaseAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql, resultSet(
new String[]{"table_name", "table_type", "table_comment"},
new Object[][]{}
)));
agent.listTables("public", new MetadataListConstraints("ord", 30, 60, List.of("TABLE", "VIEW")));
Assertions.assertTrue(sql.get(0).contains("FROM sys_catalog.sys_class"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("c.relkind IN (?, ?, ?)"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("UPPER(c.relname) LIKE ? ESCAPE '\\\\'"), sql.get(0));
Assertions.assertTrue(sql.get(0).endsWith("LIMIT 30 OFFSET 60"), sql.get(0));
}
@Test
void constrainedRegularObjectMetadataPushesRoutineTypesAndPaging() {
List<String> sql = new ArrayList<>();
KingbaseAgent agent = new KingbaseAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql, resultSet(
new String[]{"object_name", "object_type", "object_comment"},
new Object[][]{}
)));
agent.listObjects("public", new MetadataListConstraints("sync", 10, null, List.of("PROCEDURE", "FUNCTION")));
Assertions.assertTrue(sql.get(0).contains("FROM sys_catalog.sys_proc"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("p.prokind IN (?, ?)"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("ORDER BY CASE object_type"), sql.get(0));
Assertions.assertTrue(sql.get(0).endsWith("LIMIT 10"), sql.get(0));
}
@Test
void constrainedMysqlCompatTableMetadataPushesInformationSchemaPaging() {
List<String> sql = new ArrayList<>();
KingbaseAgent agent = new KingbaseAgent();
agent.setMysqlCompatMode(true);
TestSupport.setPrivateConnection(agent, preparedConnection(sql, resultSet(
new String[]{"table_name", "table_type"},
new Object[][]{}
)));
agent.listTables("PUBLIC", new MetadataListConstraints("ord", 20, 40, List.of("VIEW")));
Assertions.assertTrue(sql.get(0).contains("FROM information_schema.tables"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("table_type IN (?)"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("UPPER(table_name) LIKE ? ESCAPE '\\\\'"), sql.get(0));
Assertions.assertTrue(sql.get(0).endsWith("LIMIT 20 OFFSET 40"), sql.get(0));
}
@Test @Test
void regularRoutineSourceUsesKingbaseFunctionDefinition() { void regularRoutineSourceUsesKingbaseFunctionDefinition() {
List<String> sql = new ArrayList<>(); List<String> sql = new ArrayList<>();
@@ -9,6 +9,7 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcAgentProfile; import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo; import com.dbx.agent.TriggerInfo;
@@ -81,39 +82,79 @@ public final class OceanBaseOracleAgent extends ConfiguredJdbcAgent {
@Override @Override
public List<TableInfo> listTables(String schema) { public List<TableInfo> listTables(String schema) {
return queryTables(schema, MetadataListConstraints.NONE);
}
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
return queryTables(schema, MetadataListConstraints.orNone(constraints));
}
private List<TableInfo> queryTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> { return unchecked(() -> {
String owner = normalizeSchema(schema); String owner = normalizeSchema(schema);
String sql = """ List<String> objectTypes = oceanBaseTableTypes(constraints);
if (objectTypes.isEmpty()) {
return List.of();
}
String baseSql = """
SELECT o.OBJECT_NAME, SELECT o.OBJECT_NAME,
CASE o.OBJECT_TYPE WHEN 'VIEW' THEN 'VIEW' ELSE 'TABLE' END AS TABLE_TYPE, CASE o.OBJECT_TYPE WHEN 'VIEW' THEN 'VIEW' ELSE 'TABLE' END AS TABLE_TYPE,
c.COMMENTS c.COMMENTS
FROM ALL_OBJECTS o FROM ALL_OBJECTS o
LEFT JOIN ALL_TAB_COMMENTS c ON c.OWNER = o.OWNER AND c.TABLE_NAME = o.OBJECT_NAME LEFT JOIN ALL_TAB_COMMENTS c ON c.OWNER = o.OWNER AND c.TABLE_NAME = o.OBJECT_NAME
WHERE o.OWNER = ? AND o.OBJECT_TYPE IN ('TABLE', 'VIEW') WHERE o.OWNER = ? AND o.OBJECT_TYPE IN (%s)
ORDER BY o.OBJECT_NAME
""".stripIndent().trim(); """.stripIndent().trim();
MetadataSql query = oceanBaseMetadataSql(
String.format(baseSql, placeholders(objectTypes.size())),
"OBJECT_NAME, TABLE_TYPE, COMMENTS",
"o.OBJECT_NAME",
"ORDER BY OBJECT_NAME",
owner,
objectTypes,
constraints
);
List<TableInfo> result = new ArrayList<>(); List<TableInfo> result = new ArrayList<>();
try (var stmt = requireConnection().prepareStatement(sql)) { try (var stmt = requireConnection().prepareStatement(query.sql)) {
stmt.setString(1, owner); bind(stmt, query.args);
try (ResultSet rs = stmt.executeQuery()) { try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) { while (rs.next()) {
result.add(new TableInfo(rs.getString(1), rs.getString(2), rs.getString(3))); result.add(new TableInfo(rs.getString(1), rs.getString(2), rs.getString(3)));
} }
} }
} }
return result; return constraints.withoutPaging().filterTables(result);
}); });
} }
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return queryObjects(schema, MetadataListConstraints.NONE);
}
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
return queryObjects(schema, MetadataListConstraints.orNone(constraints));
}
private List<ObjectInfo> queryObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> { return unchecked(() -> {
String owner = normalizeSchema(schema); String owner = normalizeSchema(schema);
String sql = """ List<String> objectTypes = oceanBaseObjectTypes(constraints);
if (objectTypes.isEmpty()) {
return List.of();
}
String baseSql = """
SELECT OBJECT_NAME, OBJECT_TYPE SELECT OBJECT_NAME, OBJECT_TYPE
FROM ALL_OBJECTS FROM ALL_OBJECTS
WHERE OWNER = ? AND OBJECT_TYPE IN ('TABLE', 'VIEW', 'PROCEDURE', 'FUNCTION', 'PACKAGE', 'SEQUENCE', 'SYNONYM') WHERE OWNER = ? AND OBJECT_TYPE IN (%s)
""".stripIndent().trim();
MetadataSql query = oceanBaseMetadataSql(
String.format(baseSql, placeholders(objectTypes.size())),
"OBJECT_NAME, OBJECT_TYPE",
"OBJECT_NAME",
"""
ORDER BY CASE OBJECT_TYPE ORDER BY CASE OBJECT_TYPE
WHEN 'TABLE' THEN 0 WHEN 'TABLE' THEN 0
WHEN 'VIEW' THEN 1 WHEN 'VIEW' THEN 1
@@ -123,21 +164,113 @@ public final class OceanBaseOracleAgent extends ConfiguredJdbcAgent {
WHEN 'SEQUENCE' THEN 5 WHEN 'SEQUENCE' THEN 5
ELSE 6 ELSE 6
END, OBJECT_NAME END, OBJECT_NAME
""".stripIndent().trim(); """.stripIndent().trim(),
owner,
objectTypes,
constraints
);
List<ObjectInfo> result = new ArrayList<>(); List<ObjectInfo> result = new ArrayList<>();
try (var stmt = requireConnection().prepareStatement(sql)) { try (var stmt = requireConnection().prepareStatement(query.sql)) {
stmt.setString(1, owner); bind(stmt, query.args);
try (ResultSet rs = stmt.executeQuery()) { try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) { while (rs.next()) {
result.add(new ObjectInfo(rs.getString(1), rs.getString(2), owner, null)); result.add(new ObjectInfo(rs.getString(1), rs.getString(2), owner, null));
} }
} }
} }
return result; return constraints.withoutPaging().filterObjects(result);
}); });
} }
private static MetadataSql oceanBaseMetadataSql(
String baseSql,
String selectList,
String nameColumn,
String orderSql,
String owner,
List<String> objectTypes,
MetadataListConstraints constraints
) {
List<Object> args = new ArrayList<>();
args.add(owner);
args.addAll(objectTypes);
String sql = baseSql;
if (constraints.hasFilter()) {
sql += " AND UPPER(" + nameColumn + ") LIKE ? ESCAPE '\\'";
args.add(constraints.fuzzyLikePattern().toUpperCase(Locale.ROOT));
}
sql += "\n" + orderSql;
if (constraints.hasLimit()) {
// OceanBase Oracle mode is safest with the classic ordered ROWNUM wrapper for paged metadata.
int offset = constraints.getOffset() == null ? 0 : constraints.getOffset();
sql = "SELECT " + selectList + "\nFROM (\n SELECT DBX_Q.*, ROWNUM AS DBX_RN\n FROM (\n"
+ sql
+ "\n ) DBX_Q\n WHERE ROWNUM <= ?\n)\nWHERE DBX_RN > ?";
args.add(offset + constraints.getLimit());
args.add(offset);
} else if (constraints.hasOffset()) {
sql = "SELECT " + selectList + "\nFROM (\n SELECT DBX_Q.*, ROWNUM AS DBX_RN\n FROM (\n"
+ sql
+ "\n ) DBX_Q\n)\nWHERE DBX_RN > ?";
args.add(constraints.getOffset());
}
return new MetadataSql(sql, args);
}
private static List<String> oceanBaseTableTypes(MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return List.of("TABLE", "VIEW");
}
List<String> result = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
result.add("TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
result.add("VIEW");
}
return result;
}
private static List<String> oceanBaseObjectTypes(MetadataListConstraints constraints) {
List<String> supported = List.of("TABLE", "VIEW", "PROCEDURE", "FUNCTION", "PACKAGE", "SEQUENCE", "SYNONYM");
if (!constraints.hasObjectTypes()) {
return supported;
}
List<String> result = new ArrayList<>();
for (String objectType : supported) {
if (constraints.objectTypeAllowed(objectType)) {
result.add(objectType);
}
}
return result;
}
private static String placeholders(int count) {
return String.join(", ", java.util.Collections.nCopies(count, "?"));
}
private static void bind(java.sql.PreparedStatement stmt, List<Object> args) throws SQLException {
for (int index = 0; index < args.size(); index += 1) {
Object arg = args.get(index);
if (arg instanceof Integer) {
stmt.setInt(index + 1, (Integer) arg);
} else {
stmt.setString(index + 1, String.valueOf(arg));
}
}
}
private static final class MetadataSql {
private final String sql;
private final List<Object> args;
private MetadataSql(String sql, List<Object> args) {
this.sql = sql;
this.args = args;
}
}
@Override @Override
public List<ColumnInfo> getColumns(String schema, String table) { public List<ColumnInfo> getColumns(String schema, String table) {
return unchecked(() -> { return unchecked(() -> {
@@ -1,9 +1,22 @@
package com.dbx.agent.oceanbaseoracle; package com.dbx.agent.oceanbaseoracle;
import com.dbx.agent.ConnectParams; import com.dbx.agent.ConnectParams;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo;
import com.dbx.agent.TableInfo;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
class OceanBaseOracleAgentTest { class OceanBaseOracleAgentTest {
@Test @Test
void buildsOceanBaseJdbcUrl() { void buildsOceanBaseJdbcUrl() {
@@ -56,4 +69,131 @@ class OceanBaseOracleAgentTest {
OceanBaseOracleAgent.buildUrl(params) OceanBaseOracleAgent.buildUrl(params)
); );
} }
@Test
void constrainedListTablesUsesOceanBaseOracleMetadataSql() {
List<String> sql = new ArrayList<>();
OceanBaseOracleAgent agent = new OceanBaseOracleAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql, resultSet(
new String[]{"OBJECT_NAME", "TABLE_TYPE", "COMMENTS"},
new Object[][]{
{"USER_SETTINGS", "TABLE", null}
}
)));
List<TableInfo> tables = agent.listTables(
"APP",
new MetadataListConstraints("user", 1, 1, List.of("TABLE"))
);
Assertions.assertEquals(1, tables.size());
Assertions.assertEquals("USER_SETTINGS", tables.get(0).getName());
Assertions.assertTrue(sql.get(0).contains("ALL_OBJECTS"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("UPPER(o.OBJECT_NAME) LIKE ?"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("ROWNUM <= ?"), sql.get(0));
}
@Test
void constrainedListObjectsUsesOceanBaseOracleMetadataSql() {
List<String> sql = new ArrayList<>();
OceanBaseOracleAgent agent = new OceanBaseOracleAgent();
TestSupport.setPrivateConnection(agent, preparedConnection(sql, resultSet(
new String[]{"OBJECT_NAME", "OBJECT_TYPE"},
new Object[][]{
{"FORMAT_USER", "FUNCTION"}
}
)));
List<ObjectInfo> objects = agent.listObjects(
"APP",
new MetadataListConstraints("user", 1, 1, List.of("FUNCTION"))
);
Assertions.assertEquals(1, objects.size());
Assertions.assertEquals("FORMAT_USER", objects.get(0).getName());
Assertions.assertEquals("FUNCTION", objects.get(0).getObject_type());
Assertions.assertTrue(sql.get(0).contains("OBJECT_TYPE IN (?)"), sql.get(0));
Assertions.assertTrue(sql.get(0).contains("ROWNUM <= ?"), sql.get(0));
}
private static Connection preparedConnection(List<String> sql, ResultSet... resultSets) {
int[] resultSetIndex = {0};
PreparedStatement statement = proxy(PreparedStatement.class, (method, args) -> {
if ("executeQuery".equals(method.getName())) {
int current = Math.min(resultSetIndex[0], resultSets.length - 1);
resultSetIndex[0] += 1;
return resultSets[current];
}
if ("setString".equals(method.getName()) || "setInt".equals(method.getName()) || "close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
});
return proxy(Connection.class, (method, args) -> {
if ("prepareStatement".equals(method.getName())) {
sql.add(String.valueOf(args[0]));
return statement;
}
if ("isClosed".equals(method.getName())) {
return false;
}
return defaultValue(method.getReturnType());
});
}
private static ResultSet resultSet(String[] columns, Object[][] rows) {
int[] index = {-1};
return proxy(ResultSet.class, (method, args) -> {
switch (method.getName()) {
case "next":
index[0] += 1;
return index[0] < rows.length;
case "getString":
Object value = columnValue(columns, rows[index[0]], args[0]);
return value == null ? null : String.valueOf(value);
case "close":
return null;
default:
return defaultValue(method.getReturnType());
}
});
}
private static Object columnValue(String[] columns, Object[] row, Object key) {
if (key instanceof Number) {
return row[((Number) key).intValue() - 1];
}
for (int i = 0; i < columns.length; i++) {
if (columns[i].equalsIgnoreCase(String.valueOf(key))) {
return row[i];
}
}
return null;
}
private static <T> T proxy(Class<T> type, MethodHandler handler) {
InvocationHandler invocationHandler = new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
return handler.handle(method, args == null ? new Object[0] : args);
}
};
return type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[]{type}, invocationHandler));
}
private static Object defaultValue(Class<?> type) {
if (type == Boolean.TYPE) return false;
if (type == Byte.TYPE) return (byte) 0;
if (type == Short.TYPE) return (short) 0;
if (type == Integer.TYPE) return 0;
if (type == Long.TYPE) return 0L;
if (type == Float.TYPE) return 0f;
if (type == Double.TYPE) return 0d;
if (type == Character.TYPE) return (char) 0;
return null;
}
private interface MethodHandler {
Object handle(Method method, Object[] args) throws Throwable;
}
} }
+160 -15
View File
@@ -11,6 +11,7 @@ import (
"net/url" "net/url"
"os" "os"
"regexp" "regexp"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -42,7 +43,7 @@ ORDER BY CASE
WHEN username = SYS_CONTEXT('USERENV', 'SESSION_USER') THEN 1 WHEN username = SYS_CONTEXT('USERENV', 'SESSION_USER') THEN 1
ELSE 2 ELSE 2
END, username` END, username`
const oracleListTablesSQL = ` const oracleListTablesBaseSQL = `
SELECT OBJECT_NAME, TABLE_TYPE, COMMENTS SELECT OBJECT_NAME, TABLE_TYPE, COMMENTS
FROM ( FROM (
SELECT t.TABLE_NAME AS OBJECT_NAME, SELECT t.TABLE_NAME AS OBJECT_NAME,
@@ -58,9 +59,10 @@ SELECT o.OBJECT_NAME,
FROM ALL_OBJECTS o FROM ALL_OBJECTS o
WHERE o.OWNER = :2 WHERE o.OWNER = :2
AND o.OBJECT_TYPE = 'VIEW' AND o.OBJECT_TYPE = 'VIEW'
) )`
ORDER BY OBJECT_NAME` const oracleListTablesOrderSQL = `ORDER BY OBJECT_NAME`
const oracleListObjectsSQL = ` const oracleListTablesSQL = oracleListTablesBaseSQL + "\n" + oracleListTablesOrderSQL
const oracleListObjectsBaseSQL = `
SELECT OBJECT_NAME, OBJECT_TYPE, COMMENTS SELECT OBJECT_NAME, OBJECT_TYPE, COMMENTS
FROM ( FROM (
SELECT t.TABLE_NAME AS OBJECT_NAME, SELECT t.TABLE_NAME AS OBJECT_NAME,
@@ -71,19 +73,21 @@ WHERE t.OWNER = :1
AND t.NESTED = 'NO' AND t.NESTED = 'NO'
UNION ALL UNION ALL
SELECT o.OBJECT_NAME, SELECT o.OBJECT_NAME,
o.OBJECT_TYPE, CASE o.OBJECT_TYPE WHEN 'PACKAGE BODY' THEN 'PACKAGE_BODY' ELSE o.OBJECT_TYPE END AS OBJECT_TYPE,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM ALL_OBJECTS o FROM ALL_OBJECTS o
WHERE o.OWNER = :2 WHERE o.OWNER = :2
AND o.OBJECT_TYPE IN ('VIEW', 'PROCEDURE', 'FUNCTION', 'PACKAGE') AND o.OBJECT_TYPE IN ('VIEW', 'PROCEDURE', 'FUNCTION', 'PACKAGE', 'PACKAGE BODY')
) )`
ORDER BY CASE OBJECT_TYPE const oracleListObjectsOrderSQL = `ORDER BY CASE OBJECT_TYPE
WHEN 'TABLE' THEN 0 WHEN 'TABLE' THEN 0
WHEN 'VIEW' THEN 1 WHEN 'VIEW' THEN 1
WHEN 'PROCEDURE' THEN 2 WHEN 'PROCEDURE' THEN 2
WHEN 'FUNCTION' THEN 3 WHEN 'FUNCTION' THEN 3
ELSE 4 WHEN 'PACKAGE' THEN 4
ELSE 5
END, OBJECT_NAME` END, OBJECT_NAME`
const oracleListObjectsSQL = oracleListObjectsBaseSQL + "\n" + oracleListObjectsOrderSQL
type request struct { type request struct {
ID json.RawMessage `json:"id"` ID json.RawMessage `json:"id"`
@@ -188,6 +192,13 @@ type tableInfo struct {
Comment *string `json:"comment"` Comment *string `json:"comment"`
} }
type metadataListConstraints struct {
Filter string
Limit int
Offset int
ObjectTypes []string
}
type objectInfo struct { type objectInfo struct {
Name string `json:"name"` Name string `json:"name"`
ObjectType string `json:"object_type"` ObjectType string `json:"object_type"`
@@ -336,11 +347,11 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
return result, false, err return result, false, err
case "list_tables": case "list_tables":
schema := stringParam(params, "schema") schema := stringParam(params, "schema")
result, err := s.listTables(schema) result, err := s.listTables(schema, metadataListConstraintsFromParams(params))
return result, false, err return result, false, err
case "list_objects": case "list_objects":
schema := stringParam(params, "schema") schema := stringParam(params, "schema")
result, err := s.listObjects(schema) result, err := s.listObjects(schema, metadataListConstraintsFromParams(params))
return result, false, err return result, false, err
case "get_columns": case "get_columns":
schema := stringParam(params, "schema") schema := stringParam(params, "schema")
@@ -675,12 +686,145 @@ func (s *server) normalizeSchema(schema string) (string, error) {
return strings.ToUpper(schema), nil return strings.ToUpper(schema), nil
} }
func (s *server) listTables(schema string) ([]tableInfo, error) { type oracleMetadataListQuery struct {
SQL string
Args []any
}
func metadataListConstraintsFromParams(params map[string]json.RawMessage) metadataListConstraints {
objectTypes := stringSliceParam(params, "object_types")
if len(objectTypes) == 0 {
objectTypes = stringSliceParam(params, "objectTypes")
}
limit := intParam(params, "limit")
offset := intParam(params, "offset")
if limit < 0 {
limit = 0
}
if offset < 0 {
offset = 0
}
return metadataListConstraints{
Filter: stringParam(params, "filter"),
Limit: limit,
Offset: offset,
ObjectTypes: objectTypes,
}
}
func oracleListTablesQuery(schema string, constraints metadataListConstraints) oracleMetadataListQuery {
return oracleConstrainedMetadataListQuery(
oracleListTablesBaseSQL,
"OBJECT_NAME, TABLE_TYPE, COMMENTS",
"TABLE_TYPE",
oracleListTablesOrderSQL,
[]any{schema, schema},
constraints,
)
}
func oracleListObjectsQuery(schema string, constraints metadataListConstraints) oracleMetadataListQuery {
return oracleConstrainedMetadataListQuery(
oracleListObjectsBaseSQL,
"OBJECT_NAME, OBJECT_TYPE, COMMENTS",
"OBJECT_TYPE",
oracleListObjectsOrderSQL,
[]any{schema, schema},
constraints,
)
}
func oracleConstrainedMetadataListQuery(baseSQL, selectList, typeColumn, orderSQL string, baseArgs []any, constraints metadataListConstraints) oracleMetadataListQuery {
args := append([]any{}, baseArgs...)
where := make([]string, 0, 2)
if filter := strings.TrimSpace(constraints.Filter); filter != "" {
args = append(args, strings.ToUpper(oracleFuzzyLikePattern(filter)))
where = append(where, fmt.Sprintf("UPPER(OBJECT_NAME) LIKE :%d ESCAPE '\\'", len(args)))
}
if objectTypes := normalizedMetadataObjectTypes(constraints.ObjectTypes); len(objectTypes) > 0 {
placeholders := make([]string, 0, len(objectTypes))
for _, objectType := range objectTypes {
args = append(args, objectType)
placeholders = append(placeholders, fmt.Sprintf(":%d", len(args)))
}
where = append(where, fmt.Sprintf("%s IN (%s)", typeColumn, strings.Join(placeholders, ",")))
}
sqlText := fmt.Sprintf("SELECT %s\nFROM (\n%s\n)", selectList, baseSQL)
if len(where) > 0 {
sqlText += "\nWHERE " + strings.Join(where, " AND ")
}
sqlText += "\n" + orderSQL
if constraints.Limit > 0 {
args = append(args, constraints.Offset+constraints.Limit)
maxRowParam := len(args)
args = append(args, constraints.Offset)
offsetParam := len(args)
sqlText = fmt.Sprintf(
"SELECT %s\nFROM (\n SELECT DBX_Q.*, ROWNUM AS DBX_RN\n FROM (\n%s\n ) DBX_Q\n WHERE ROWNUM <= :%d\n)\nWHERE DBX_RN > :%d",
selectList,
sqlText,
maxRowParam,
offsetParam,
)
} else if constraints.Offset > 0 {
args = append(args, constraints.Offset)
offsetParam := len(args)
sqlText = fmt.Sprintf(
"SELECT %s\nFROM (\n SELECT DBX_Q.*, ROWNUM AS DBX_RN\n FROM (\n%s\n ) DBX_Q\n)\nWHERE DBX_RN > :%d",
selectList,
sqlText,
offsetParam,
)
}
return oracleMetadataListQuery{SQL: sqlText, Args: args}
}
func normalizedMetadataObjectTypes(values []string) []string {
seen := map[string]bool{}
result := make([]string, 0, len(values))
for _, value := range values {
normalized := strings.ToUpper(strings.TrimSpace(value))
normalized = strings.ReplaceAll(normalized, "-", "_")
normalized = strings.ReplaceAll(normalized, " ", "_")
if normalized == "" || seen[normalized] {
continue
}
seen[normalized] = true
result = append(result, normalized)
}
sort.Strings(result)
return result
}
func oracleFuzzyLikePattern(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "%%"
}
var builder strings.Builder
builder.Grow(len(value)*2 + 2)
builder.WriteByte('%')
for _, ch := range value {
switch ch {
case '\\', '%', '_':
builder.WriteByte('\\')
}
builder.WriteRune(ch)
builder.WriteByte('%')
}
return builder.String()
}
func (s *server) listTables(schema string, constraints metadataListConstraints) ([]tableInfo, error) {
schema, err := s.normalizeSchema(schema) schema, err := s.normalizeSchema(schema)
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := s.queryRows(oracleListTablesSQL, []any{schema, schema}) query := oracleListTablesQuery(schema, constraints)
rows, err := s.queryRows(query.SQL, query.Args)
if err != nil { if err != nil {
if isOraclePGALimitError(err) { if isOraclePGALimitError(err) {
return []tableInfo{}, nil return []tableInfo{}, nil
@@ -699,12 +843,13 @@ func (s *server) listTables(schema string) ([]tableInfo, error) {
return emptyIfNil(result), rows.Err() return emptyIfNil(result), rows.Err()
} }
func (s *server) listObjects(schema string) ([]objectInfo, error) { func (s *server) listObjects(schema string, constraints metadataListConstraints) ([]objectInfo, error) {
schema, err := s.normalizeSchema(schema) schema, err := s.normalizeSchema(schema)
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := s.queryRows(oracleListObjectsSQL, []any{schema, schema}) query := oracleListObjectsQuery(schema, constraints)
rows, err := s.queryRows(query.SQL, query.Args)
if err != nil { if err != nil {
if isOraclePGALimitError(err) { if isOraclePGALimitError(err) {
return []objectInfo{}, nil return []objectInfo{}, nil
+62
View File
@@ -358,6 +358,32 @@ func TestListTablesSQLUsesSplitDictionaryQuery(t *testing.T) {
} }
} }
func TestListTablesQueryAppliesMetadataConstraints(t *testing.T) {
query := oracleListTablesQuery("APP", metadataListConstraints{
Filter: "u_r",
Limit: 501,
Offset: 10,
ObjectTypes: []string{"view", "TABLE", "TABLE"},
})
sqlText := strings.ToUpper(query.SQL)
if !strings.Contains(sqlText, "UPPER(OBJECT_NAME) LIKE :3 ESCAPE '\\'") {
t.Fatalf("table listing should push filter predicate, got: %s", query.SQL)
}
if !strings.Contains(sqlText, "TABLE_TYPE IN (:4,:5)") {
t.Fatalf("table listing should push table type predicate, got: %s", query.SQL)
}
if !strings.Contains(sqlText, "ROWNUM <= :6") || !strings.Contains(sqlText, "DBX_RN > :7") {
t.Fatalf("table listing should use rownum pagination, got: %s", query.SQL)
}
if len(query.Args) != 7 {
t.Fatalf("unexpected args: %#v", query.Args)
}
if query.Args[0] != "APP" || query.Args[1] != "APP" || query.Args[2] != "%U%\\_%R%" || query.Args[3] != "TABLE" || query.Args[4] != "VIEW" || query.Args[5] != 511 || query.Args[6] != 10 {
t.Fatalf("constraints args were not normalized: %#v", query.Args)
}
}
func TestListObjectsSQLUsesSplitDictionaryQuery(t *testing.T) { func TestListObjectsSQLUsesSplitDictionaryQuery(t *testing.T) {
sqlText := strings.ToUpper(oracleListObjectsSQL) sqlText := strings.ToUpper(oracleListObjectsSQL)
@@ -370,6 +396,42 @@ func TestListObjectsSQLUsesSplitDictionaryQuery(t *testing.T) {
if strings.Contains(sqlText, "ALL_TAB_COMMENTS") { if strings.Contains(sqlText, "ALL_TAB_COMMENTS") {
t.Fatalf("object listing should not load comments during refresh, got: %s", oracleListObjectsSQL) t.Fatalf("object listing should not load comments during refresh, got: %s", oracleListObjectsSQL)
} }
if !strings.Contains(sqlText, "'PACKAGE BODY'") || !strings.Contains(sqlText, "PACKAGE_BODY") {
t.Fatalf("object listing should include package bodies with normalized type, got: %s", oracleListObjectsSQL)
}
}
func TestListObjectsQueryAppliesMetadataConstraints(t *testing.T) {
query := oracleListObjectsQuery("APP", metadataListConstraints{
Filter: "pkg%",
Limit: 25,
ObjectTypes: []string{"FUNCTION", "package"},
})
sqlText := strings.ToUpper(query.SQL)
if !strings.Contains(sqlText, "UPPER(OBJECT_NAME) LIKE :3 ESCAPE '\\'") {
t.Fatalf("object listing should push filter predicate, got: %s", query.SQL)
}
if !strings.Contains(sqlText, "OBJECT_TYPE IN (:4,:5)") {
t.Fatalf("object listing should push object type predicate, got: %s", query.SQL)
}
if !strings.Contains(sqlText, "ROWNUM <= :6") || !strings.Contains(sqlText, "DBX_RN > :7") {
t.Fatalf("object listing should use rownum pagination, got: %s", query.SQL)
}
if len(query.Args) != 7 {
t.Fatalf("unexpected args: %#v", query.Args)
}
if query.Args[2] != "%P%K%G%\\%%" || query.Args[3] != "FUNCTION" || query.Args[4] != "PACKAGE" || query.Args[5] != 25 || query.Args[6] != 0 {
t.Fatalf("object constraints args were not normalized: %#v", query.Args)
}
}
func TestOracleFuzzyLikePatternEscapesSpecialCharacters(t *testing.T) {
got := oracleFuzzyLikePattern(`a_%\b`)
want := `%a%\_%\%%\\%b%`
if got != want {
t.Fatalf("oracleFuzzyLikePattern() = %q, want %q", got, want)
}
} }
func TestIsOraclePGALimitError(t *testing.T) { func TestIsOraclePGALimitError(t *testing.T) {
@@ -8,6 +8,8 @@ import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo; import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
@@ -86,6 +88,48 @@ public final class SnowflakeAgent extends AbstractJdbcAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?");
args.add(schema);
appendSnowflakeTableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "TABLE_NAME", constraints);
sql.append(" ORDER BY TABLE_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString("TABLE_NAME"),
normalizeTableType(rs.getString("TABLE_TYPE")),
null
));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -110,6 +154,64 @@ public final class SnowflakeAgent extends AbstractJdbcAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return queryConstrainedObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT TABLE_NAME AS OBJECT_NAME, TABLE_TYPE AS OBJECT_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?");
args.add(schema);
appendSnowflakeTableTypePredicate(tableSql, args, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "TABLE_NAME", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE")) {
StringBuilder procedureSql = new StringBuilder("SELECT PROCEDURE_NAME AS OBJECT_NAME, 'PROCEDURE' AS OBJECT_TYPE FROM INFORMATION_SCHEMA.PROCEDURES WHERE PROCEDURE_SCHEMA = ?");
args.add(schema);
MetadataSqlSupport.appendNameFilter(procedureSql, args, "PROCEDURE_NAME", constraints);
branches.add(procedureSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT OBJECT_NAME, OBJECT_TYPE FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY CASE OBJECT_TYPE WHEN 'BASE TABLE' THEN 0 WHEN 'TABLE' THEN 0 WHEN 'VIEW' THEN 1 WHEN 'MATERIALIZED VIEW' THEN 2 WHEN 'PROCEDURE' THEN 3 ELSE 9 END, OBJECT_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(
rs.getString("OBJECT_NAME"),
normalizeTableType(rs.getString("OBJECT_TYPE")),
schema,
null
));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -224,8 +326,44 @@ public final class SnowflakeAgent extends AbstractJdbcAgent {
return "jdbc:snowflake://" + params.getHost() + ":" + params.getPort() + "/?db=" + params.getDatabase(); return "jdbc:snowflake://" + params.getHost() + ":" + params.getPort() + "/?db=" + params.getDatabase();
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes() || constraints.objectTypeAllowed("PROCEDURE");
}
private static void appendSnowflakeTableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (constraints.tableTypeAllowed("MATERIALIZED_VIEW")) {
types.add("MATERIALIZED VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND TABLE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static String normalizeTableType(String tableType) { private static String normalizeTableType(String tableType) {
return "BASE TABLE".equals(tableType) ? "TABLE" : tableType; if ("BASE TABLE".equals(tableType)) {
return "TABLE";
}
if ("MATERIALIZED VIEW".equals(tableType)) {
return "MATERIALIZED_VIEW";
}
return tableType;
} }
private static String blankToNull(String value) { private static String blankToNull(String value) {
@@ -1,8 +1,10 @@
package com.dbx.agent.snowflake; package com.dbx.agent.snowflake;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.TestSupport; import com.dbx.agent.test.TestSupport;
import com.dbx.agent.test.JdbcAgentFake; import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.JdbcMetadataSqlFake; import com.dbx.agent.test.JdbcMetadataSqlFake;
import java.util.List;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -26,4 +28,32 @@ class SnowflakeAgentMetadataTest {
) )
); );
} }
@Test
void constrainedTableMetadataPushesFilterTypesAndPaging() {
SnowflakeAgent agent = new SnowflakeAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("PUBLIC", new MetadataListConstraints("ord", 25, 50, List.of("TABLE", "VIEW")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM INFORMATION_SCHEMA.TABLES"), sql);
Assertions.assertTrue(sql.contains("TABLE_TYPE IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("UPPER(TABLE_NAME) LIKE ? ESCAPE '\\\\'"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 25 OFFSET 50"), sql);
}
@Test
void constrainedObjectMetadataPushesProcedureSearchAndPaging() {
SnowflakeAgent agent = new SnowflakeAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listObjects("PUBLIC", new MetadataListConstraints("sync", 10, null, List.of("PROCEDURE")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM INFORMATION_SCHEMA.PROCEDURES"), sql);
Assertions.assertTrue(sql.contains("UPPER(PROCEDURE_NAME) LIKE ? ESCAPE '\\\\'"), sql);
Assertions.assertTrue(sql.contains("ORDER BY CASE OBJECT_TYPE"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 10"), sql);
}
} }
@@ -10,6 +10,8 @@ import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcExecutor; import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.ObjectInfo; import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource; import com.dbx.agent.ObjectSource;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
@@ -103,6 +105,44 @@ public final class SundbAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?");
args.add(schema);
appendInformationSchemaTableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "TABLE_NAME", constraints);
sql.append(" ORDER BY TABLE_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (var stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(rs.getString("TABLE_NAME"), normalizeTableType(rs.getString("TABLE_TYPE")), null));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ObjectInfo> listObjects(String schema) { public List<ObjectInfo> listObjects(String schema) {
return unchecked(() -> { return unchecked(() -> {
@@ -125,6 +165,60 @@ public final class SundbAgent extends BaseDatabaseAgent {
}); });
} }
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listObjects(schema);
}
if (!includesSupportedObjects(normalized)) {
return List.of();
}
try {
return queryConstrainedObjects(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterObjects(listObjects(schema));
}
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<ObjectInfo> result = new ArrayList<>();
List<String> branches = new ArrayList<>();
List<Object> args = new ArrayList<>();
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT TABLE_NAME AS OBJECT_NAME, TABLE_TYPE AS OBJECT_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?");
args.add(schema);
appendInformationSchemaTableTypePredicate(tableSql, args, constraints);
MetadataSqlSupport.appendNameFilter(tableSql, args, "TABLE_NAME", constraints);
branches.add(tableSql.toString());
}
if (constraints.objectTypeAllowed("PROCEDURE") || constraints.objectTypeAllowed("FUNCTION")) {
StringBuilder routineSql = new StringBuilder("SELECT ROUTINE_NAME AS OBJECT_NAME, ROUTINE_TYPE AS OBJECT_TYPE FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = ?");
args.add(schema);
appendRoutineTypePredicate(routineSql, args, constraints);
MetadataSqlSupport.appendNameFilter(routineSql, args, "ROUTINE_NAME", constraints);
branches.add(routineSql.toString());
}
if (branches.isEmpty()) {
return List.of();
}
StringBuilder sql = new StringBuilder("SELECT OBJECT_NAME, OBJECT_TYPE FROM (")
.append(String.join(" UNION ALL ", branches))
.append(") metadata_objects ORDER BY CASE OBJECT_TYPE WHEN 'BASE TABLE' THEN 0 WHEN 'TABLE' THEN 0 WHEN 'VIEW' THEN 1 WHEN 'PROCEDURE' THEN 2 WHEN 'FUNCTION' THEN 3 ELSE 9 END, OBJECT_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (var stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ObjectInfo(rs.getString(1), normalizeTableType(rs.getString(2)), schema, null));
}
}
}
return constraints.withoutPaging().filterObjects(result);
});
}
@Override @Override
public ObjectSource getObjectSource(String schema, String name, String objectType) { public ObjectSource getObjectSource(String schema, String name, String objectType) {
return unchecked(() -> { return unchecked(() -> {
@@ -344,6 +438,58 @@ public final class SundbAgent extends BaseDatabaseAgent {
return "jdbc:sundb://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase(); return "jdbc:sundb://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase();
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static boolean includesSupportedObjects(MetadataListConstraints constraints) {
return constraints.includesTableLikeTypes()
|| constraints.objectTypeAllowed("PROCEDURE")
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendInformationSchemaTableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND TABLE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static void appendRoutineTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.objectTypeAllowed("PROCEDURE")) {
types.add("PROCEDURE");
}
if (constraints.objectTypeAllowed("FUNCTION")) {
types.add("FUNCTION");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND ROUTINE_TYPE IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static String normalizeTableType(String tableType) {
return "BASE TABLE".equals(tableType) ? "TABLE" : tableType;
}
private static Integer integerOrNull(ResultSet rs, String column) throws Exception { private static Integer integerOrNull(ResultSet rs, String column) throws Exception {
Object value = rs.getObject(column); Object value = rs.getObject(column);
return value instanceof Integer ? (Integer) value : null; return value instanceof Integer ? (Integer) value : null;
@@ -1,9 +1,11 @@
package com.dbx.agent.sundb; package com.dbx.agent.sundb;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.TestSupport; import com.dbx.agent.test.TestSupport;
import com.dbx.agent.test.JdbcAgentFake; import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.JdbcMetadataSqlFake; import com.dbx.agent.test.JdbcMetadataSqlFake;
import java.util.Collections; import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -20,4 +22,32 @@ class SundbAgentMetadataTest {
JdbcMetadataSqlFake.statements JdbcMetadataSqlFake.statements
); );
} }
@Test
void constrainedTableMetadataPushesFilterTypesAndPaging() {
SundbAgent agent = new SundbAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("app", new MetadataListConstraints("ord", 25, 50, List.of("TABLE", "VIEW")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM information_schema.TABLES"), sql);
Assertions.assertTrue(sql.contains("TABLE_TYPE IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("UPPER(TABLE_NAME) LIKE ? ESCAPE '\\\\'"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 25 OFFSET 50"), sql);
}
@Test
void constrainedObjectMetadataPushesRoutineTypesAndPaging() {
SundbAgent agent = new SundbAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listObjects("app", new MetadataListConstraints("sync", 10, null, List.of("PROCEDURE", "FUNCTION")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM information_schema.ROUTINES"), sql);
Assertions.assertTrue(sql.contains("ROUTINE_TYPE IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("ORDER BY CASE OBJECT_TYPE"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 10"), sql);
}
} }
@@ -8,6 +8,8 @@ import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo; import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MetadataSqlSupport;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo; import com.dbx.agent.TriggerInfo;
import java.sql.Connection; import java.sql.Connection;
@@ -87,6 +89,48 @@ public final class TrinoAgent extends AbstractJdbcAgent {
}); });
} }
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (isUnconstrained(normalized)) {
return listTables(schema);
}
if (!normalized.includesTableLikeTypes()) {
return List.of();
}
try {
return queryConstrainedTables(schema, normalized);
} catch (RuntimeException e) {
return normalized.filterTables(listTables(schema));
}
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = ?");
args.add(schema);
appendTrinoTableTypePredicate(sql, args, constraints);
MetadataSqlSupport.appendNameFilter(sql, args, "table_name", constraints);
sql.append(" ORDER BY table_name");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
try (java.sql.PreparedStatement stmt = requireConnected().prepareStatement(sql.toString())) {
MetadataSqlSupport.bind(stmt, args);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString("table_name"),
normalizeTableType(rs.getString("table_type")),
null
));
}
}
}
return constraints.withoutPaging().filterTables(result);
});
}
@Override @Override
public List<ColumnInfo> getColumns(String schema, String table) { public List<ColumnInfo> getColumns(String schema, String table) {
Connection conn = requireConnected(); Connection conn = requireConnected();
@@ -188,6 +232,32 @@ public final class TrinoAgent extends AbstractJdbcAgent {
return "jdbc:trino://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase(); return "jdbc:trino://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase();
} }
private static boolean isUnconstrained(MetadataListConstraints constraints) {
return !constraints.hasFilter() && !constraints.hasLimit() && !constraints.hasOffset() && !constraints.hasObjectTypes();
}
private static void appendTrinoTableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
}
if (constraints.tableTypeAllowed("MATERIALIZED_VIEW")) {
types.add("MATERIALIZED VIEW");
}
if (types.isEmpty()) {
sql.append(" AND 1 = 0");
return;
}
sql.append(" AND table_type IN (").append(MetadataSqlSupport.placeholders(types.size())).append(")");
args.addAll(types);
}
private static String quoteIdentifier(String identifier) { private static String quoteIdentifier(String identifier) {
return "\"" + identifier.replace("\"", "\"\"") + "\""; return "\"" + identifier.replace("\"", "\"\"") + "\"";
} }
@@ -255,6 +325,9 @@ public final class TrinoAgent extends AbstractJdbcAgent {
if ("BASE TABLE".equals(type)) { if ("BASE TABLE".equals(type)) {
return "TABLE"; return "TABLE";
} }
if ("MATERIALIZED VIEW".equals(type)) {
return "MATERIALIZED_VIEW";
}
return type; return type;
} }
@@ -1,7 +1,14 @@
package com.dbx.agent.trino; package com.dbx.agent.trino;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import com.dbx.agent.test.JdbcMetadataSqlFake;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
class TrinoAgentTest extends JdbcFakeExecutionBehaviorTest { class TrinoAgentTest extends JdbcFakeExecutionBehaviorTest {
@Override @Override
@@ -13,4 +20,18 @@ class TrinoAgentTest extends JdbcFakeExecutionBehaviorTest {
protected String resultSetSql() { protected String resultSetSql() {
return "CALL system.runtime.nodes()"; return "CALL system.runtime.nodes()";
} }
@Test
void constrainedTableMetadataPushesFilterTypesAndPaging() {
TrinoAgent agent = new TrinoAgent();
TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection());
agent.listTables("public", new MetadataListConstraints("ord", 25, 50, List.of("TABLE", "VIEW")));
String sql = JdbcMetadataSqlFake.statements.get(0);
Assertions.assertTrue(sql.contains("FROM information_schema.tables"), sql);
Assertions.assertTrue(sql.contains("table_type IN (?, ?)"), sql);
Assertions.assertTrue(sql.contains("UPPER(table_name) LIKE ? ESCAPE '\\\\'"), sql);
Assertions.assertTrue(sql.endsWith("LIMIT 25 OFFSET 50"), sql);
}
} }
+189 -26
View File
@@ -10,6 +10,7 @@ import (
"io" "io"
"os" "os"
"regexp" "regexp"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -196,6 +197,18 @@ type objectInfo struct {
Comment *string `json:"comment"` Comment *string `json:"comment"`
} }
type metadataListConstraints struct {
Filter string
Limit int
Offset int
ObjectTypes []string
}
type xuguMetadataListQuery struct {
SQL string
Args []any
}
type columnInfo struct { type columnInfo struct {
Name string `json:"name"` Name string `json:"name"`
DataType string `json:"data_type"` DataType string `json:"data_type"`
@@ -341,14 +354,14 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
return nil, false, err return nil, false, err
} }
schema := stringParam(params, "schema") schema := stringParam(params, "schema")
result, err := s.listTables(schema) result, err := s.listTables(schema, metadataListConstraintsFromParams(params))
return result, false, err return result, false, err
case "list_objects": case "list_objects":
if err := s.useDatabase(stringParam(params, "database")); err != nil { if err := s.useDatabase(stringParam(params, "database")); err != nil {
return nil, false, err return nil, false, err
} }
schema := stringParam(params, "schema") schema := stringParam(params, "schema")
result, err := s.listObjects(schema) result, err := s.listObjects(schema, metadataListConstraintsFromParams(params))
return result, false, err return result, false, err
case "list_data_types": case "list_data_types":
return xuguDataTypes, false, nil return xuguDataTypes, false, nil
@@ -793,22 +806,13 @@ func (s *server) normalizeSchema(schema string) (string, error) {
return strings.ToUpper(schema), nil return strings.ToUpper(schema), nil
} }
func (s *server) listTables(schema string) ([]tableInfo, error) { func (s *server) listTables(schema string, constraints metadataListConstraints) ([]tableInfo, error) {
schema, err := s.normalizeSchema(schema) schema, err := s.normalizeSchema(schema)
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := s.queryRows(` query := xuguListTablesQuery(schema, constraints)
SELECT t.TABLE_NAME, 'TABLE' AS TABLE_TYPE, t.COMMENTS rows, err := s.queryRows(query.SQL, query.Args)
FROM ALL_TABLES t
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)
UNION ALL
SELECT v.VIEW_NAME, 'VIEW' AS TABLE_TYPE, v.COMMENTS
FROM ALL_VIEWS v
JOIN ALL_SCHEMAS s ON s.DB_ID = v.DB_ID AND s.SCHEMA_ID = v.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)
ORDER BY 2, 1`, []any{schema, schema})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -824,22 +828,13 @@ ORDER BY 2, 1`, []any{schema, schema})
return emptyIfNil(result), rows.Err() return emptyIfNil(result), rows.Err()
} }
func (s *server) listObjects(schema string) ([]objectInfo, error) { func (s *server) listObjects(schema string, constraints metadataListConstraints) ([]objectInfo, error) {
schema, err := s.normalizeSchema(schema) schema, err := s.normalizeSchema(schema)
if err != nil { if err != nil {
return nil, err return nil, err
} }
rows, err := s.queryRows(` query := xuguListObjectsQuery(schema, constraints)
SELECT t.TABLE_NAME, 'TABLE' AS OBJECT_TYPE, t.COMMENTS rows, err := s.queryRows(query.SQL, query.Args)
FROM ALL_TABLES t
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)
UNION ALL
SELECT v.VIEW_NAME, 'VIEW' AS OBJECT_TYPE, v.COMMENTS
FROM ALL_VIEWS v
JOIN ALL_SCHEMAS s ON s.DB_ID = v.DB_ID AND s.SCHEMA_ID = v.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)
ORDER BY 2, 1`, []any{schema, schema})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -856,6 +851,159 @@ ORDER BY 2, 1`, []any{schema, schema})
return emptyIfNil(result), rows.Err() return emptyIfNil(result), rows.Err()
} }
func metadataListConstraintsFromParams(params map[string]json.RawMessage) metadataListConstraints {
objectTypes := stringSliceParam(params, "object_types")
if len(objectTypes) == 0 {
objectTypes = stringSliceParam(params, "objectTypes")
}
limit := intParam(params, "limit")
offset := intParam(params, "offset")
if limit < 0 {
limit = 0
}
if offset < 0 {
offset = 0
}
return metadataListConstraints{
Filter: stringParam(params, "filter"),
Limit: limit,
Offset: offset,
ObjectTypes: objectTypes,
}
}
func xuguListTablesQuery(schema string, constraints metadataListConstraints) xuguMetadataListQuery {
return xuguConstrainedMetadataListQuery(
`
SELECT t.TABLE_NAME, 'TABLE' AS TABLE_TYPE, t.COMMENTS
FROM ALL_TABLES t
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)
UNION ALL
SELECT v.VIEW_NAME, 'VIEW' AS TABLE_TYPE, v.COMMENTS
FROM ALL_VIEWS v
JOIN ALL_SCHEMAS s ON s.DB_ID = v.DB_ID AND s.SCHEMA_ID = v.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)`,
"TABLE_NAME, TABLE_TYPE, COMMENTS",
"TABLE_NAME",
"TABLE_TYPE",
[]any{schema, schema},
constraints,
)
}
func xuguListObjectsQuery(schema string, constraints metadataListConstraints) xuguMetadataListQuery {
return xuguConstrainedMetadataListQuery(
`
SELECT t.TABLE_NAME AS OBJECT_NAME, 'TABLE' AS OBJECT_TYPE, t.COMMENTS
FROM ALL_TABLES t
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)
UNION ALL
SELECT v.VIEW_NAME AS OBJECT_NAME, 'VIEW' AS OBJECT_TYPE, v.COMMENTS
FROM ALL_VIEWS v
JOIN ALL_SCHEMAS s ON s.DB_ID = v.DB_ID AND s.SCHEMA_ID = v.SCHEMA_ID
WHERE UPPER(s.SCHEMA_NAME) = UPPER(?)`,
"OBJECT_NAME, OBJECT_TYPE, COMMENTS",
"OBJECT_NAME",
"OBJECT_TYPE",
[]any{schema, schema},
constraints,
)
}
func xuguConstrainedMetadataListQuery(baseSQL, selectList, nameColumn, typeColumn string, baseArgs []any, constraints metadataListConstraints) xuguMetadataListQuery {
args := append([]any{}, baseArgs...)
where := make([]string, 0, 2)
if filter := strings.TrimSpace(constraints.Filter); filter != "" {
args = append(args, strings.ToUpper(xuguFuzzyLikePattern(filter)))
where = append(where, fmt.Sprintf("UPPER(%s) LIKE ? ESCAPE '\\'", nameColumn))
}
if len(constraints.ObjectTypes) > 0 {
objectTypes := normalizedXuguObjectTypes(constraints.ObjectTypes)
if len(objectTypes) == 0 {
where = append(where, "1 = 0")
} else {
placeholders := make([]string, 0, len(objectTypes))
for _, objectType := range objectTypes {
args = append(args, objectType)
placeholders = append(placeholders, "?")
}
where = append(where, fmt.Sprintf("%s IN (%s)", typeColumn, strings.Join(placeholders, ",")))
}
}
sqlText := fmt.Sprintf("SELECT %s\nFROM (\n%s\n)", selectList, baseSQL)
if len(where) > 0 {
sqlText += "\nWHERE " + strings.Join(where, " AND ")
}
sqlText += fmt.Sprintf("\nORDER BY %s, %s", typeColumn, nameColumn)
// Xugu documents ROWNUM as the safe pagination path when ORDER BY belongs
// to an inner query; LIMIT is not portable for this UNION metadata query.
if constraints.Limit > 0 {
args = append(args, constraints.Offset+constraints.Limit, constraints.Offset)
sqlText = fmt.Sprintf(
"SELECT %s\nFROM (\n SELECT DBX_Q.*, ROWNUM AS DBX_RN\n FROM (\n%s\n ) DBX_Q\n WHERE ROWNUM <= ?\n)\nWHERE DBX_RN > ?",
selectList,
sqlText,
)
} else if constraints.Offset > 0 {
args = append(args, constraints.Offset)
sqlText = fmt.Sprintf(
"SELECT %s\nFROM (\n SELECT DBX_Q.*, ROWNUM AS DBX_RN\n FROM (\n%s\n ) DBX_Q\n)\nWHERE DBX_RN > ?",
selectList,
sqlText,
)
}
return xuguMetadataListQuery{SQL: sqlText, Args: args}
}
func normalizedXuguObjectTypes(values []string) []string {
seen := map[string]bool{}
result := make([]string, 0, len(values))
for _, value := range values {
normalized := strings.ToUpper(strings.TrimSpace(value))
normalized = strings.ReplaceAll(normalized, "-", "_")
normalized = strings.ReplaceAll(normalized, " ", "_")
switch normalized {
case "TABLE", "BASE_TABLE":
normalized = "TABLE"
case "VIEW":
normalized = "VIEW"
default:
continue
}
if seen[normalized] {
continue
}
seen[normalized] = true
result = append(result, normalized)
}
sort.Strings(result)
return result
}
func xuguFuzzyLikePattern(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "%%"
}
var builder strings.Builder
builder.Grow(len(value)*2 + 2)
builder.WriteByte('%')
for _, ch := range value {
switch ch {
case '\\', '%', '_':
builder.WriteByte('\\')
}
builder.WriteRune(ch)
builder.WriteByte('%')
}
return builder.String()
}
func (s *server) getColumns(schema, table string) ([]columnInfo, error) { func (s *server) getColumns(schema, table string) ([]columnInfo, error) {
schema, err := s.normalizeSchema(schema) schema, err := s.normalizeSchema(schema)
if err != nil { if err != nil {
@@ -1825,6 +1973,21 @@ func intParam(params map[string]json.RawMessage, key string) int {
return value return value
} }
func stringSliceParam(params map[string]json.RawMessage, key string) []string {
if params == nil || len(params[key]) == 0 {
return nil
}
var values []string
if err := json.Unmarshal(params[key], &values); err == nil {
return values
}
var single string
if err := json.Unmarshal(params[key], &single); err == nil && strings.TrimSpace(single) != "" {
return []string{single}
}
return nil
}
func errorResponse(id json.RawMessage, err error) response { func errorResponse(id json.RawMessage, err error) response {
return response{JSONRPC: "2.0", ID: id, Error: &rpcError{Code: -1, Message: err.Error()}} return response{JSONRPC: "2.0", ID: id, Error: &rpcError{Code: -1, Message: err.Error()}}
} }
+67
View File
@@ -383,6 +383,73 @@ func TestXuguMetadataAccessErrorDetection(t *testing.T) {
} }
} }
func TestXuguListTablesQueryAppliesMetadataConstraints(t *testing.T) {
query := xuguListTablesQuery("APP", metadataListConstraints{
Filter: "ord_",
ObjectTypes: []string{"view", "table", "VIEW"},
Limit: 25,
Offset: 50,
})
for _, want := range []string{
"UPPER(TABLE_NAME) LIKE ? ESCAPE '\\'",
"TABLE_TYPE IN (?,?)",
"ORDER BY TABLE_TYPE, TABLE_NAME",
"ROWNUM <= ?",
"DBX_RN > ?",
} {
if !strings.Contains(query.SQL, want) {
t.Fatalf("expected SQL to contain %q:\n%s", want, query.SQL)
}
}
wantArgs := []any{"APP", "APP", `%O%R%D%\_%`, "TABLE", "VIEW", 75, 50}
assertArgs(t, query.Args, wantArgs)
}
func TestXuguListObjectsQueryRejectsUnsupportedObjectTypes(t *testing.T) {
query := xuguListObjectsQuery("APP", metadataListConstraints{
ObjectTypes: []string{"INDEX"},
Limit: 10,
})
if !strings.Contains(query.SQL, "1 = 0") {
t.Fatalf("unsupported object type should produce empty-result predicate:\n%s", query.SQL)
}
wantArgs := []any{"APP", "APP", 10, 0}
assertArgs(t, query.Args, wantArgs)
}
func TestMetadataListConstraintsFromParams(t *testing.T) {
params := map[string]json.RawMessage{
"filter": json.RawMessage(`"tab"`),
"limit": json.RawMessage(`30`),
"offset": json.RawMessage(`5`),
"object_types": json.RawMessage(`["TABLE","VIEW"]`),
}
constraints := metadataListConstraintsFromParams(params)
if constraints.Filter != "tab" || constraints.Limit != 30 || constraints.Offset != 5 {
t.Fatalf("unexpected constraints: %+v", constraints)
}
if len(constraints.ObjectTypes) != 2 || constraints.ObjectTypes[0] != "TABLE" || constraints.ObjectTypes[1] != "VIEW" {
t.Fatalf("unexpected object types: %+v", constraints.ObjectTypes)
}
}
func assertArgs(t *testing.T, got []any, want []any) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("args length = %d, want %d: got=%#v want=%#v", len(got), len(want), got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("arg %d = %#v, want %#v; args=%#v", i, got[i], want[i], got)
}
}
}
func TestParseForeignKeyColumns(t *testing.T) { func TestParseForeignKeyColumns(t *testing.T) {
local, ref := parseForeignKeyColumns(`("C1","C2")("ID1","ID2")`) local, ref := parseForeignKeyColumns(`("C1","C2")("ID1","ID2")`)
+23
View File
@@ -0,0 +1,23 @@
driver strategy scope reason
access intentional-fallback java-jdbc-metadata Uses Access JDBC metadata; no portable stable server-side paging API, common constraints filter locally.
bigquery native-pushdown java-sql Uses dataset INFORMATION_SCHEMA.TABLES with type, normal-name filter, stable order, and literal LIMIT/OFFSET.
cassandra intentional-fallback java-cql Uses Cassandra system_schema; portable LIKE/fuzzy paging is not safe across versions, common constraints filter locally.
dameng native-pushdown java-sql Uses ALL_OBJECTS with type, filter, stable order, and LIMIT/OFFSET with legacy fallback.
databend native-pushdown java-sql Uses system procedures for routine pushdown and shared table metadata constraints for table-like objects.
db2 native-pushdown java-sql Uses SYSCAT metadata with type, filter, stable order, and OFFSET/FETCH.
gbase8a native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
gbase8s native-pushdown java-sql Uses systables with type, filter, stable order, and SKIP/FIRST.
goldendb native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
h2 native-pushdown java-sql Uses INFORMATION_SCHEMA with type, filter, stable order, and LIMIT/OFFSET.
hive intentional-fallback java-sql Uses Hive metadata paths where stable server-side fuzzy paging is not portable, common constraints filter locally.
informix native-pushdown java-sql Uses systables/sysprocedures with type, filter, stable order, and SKIP/FIRST.
kingbase native-pushdown java-sql Uses sys_catalog/information_schema with type, filter, stable order, and LIMIT/OFFSET for both compatibility modes.
kylin intentional-fallback java-jdbc-metadata Uses JDBC metadata from Kylin driver; no portable server-side paging API, common constraints filter locally.
neo4j intentional-fallback java-graph-metadata Uses JDBC metadata with Cypher fallback; label discovery cannot guarantee relational-style filtered paging, common constraints filter locally.
oceanbase-oracle native-pushdown java-sql Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging.
oracle-go native-pushdown native-go Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging, with legacy profile guard.
snowflake native-pushdown java-sql Uses INFORMATION_SCHEMA with type, filter, stable order, and literal LIMIT/OFFSET.
sundb native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
tdengine intentional-fallback java-show-metadata Uses SHOW STABLES/TABLES; stable server-side fuzzy paging is not portable, common constraints filter locally.
trino native-pushdown java-sql Uses information_schema.tables with type, filter, stable order, and literal LIMIT/OFFSET.
xugu native-pushdown native-go Uses ALL_TABLES/ALL_VIEWS metadata with type, filter, stable order, and ROWNUM paging.
1 driver strategy scope reason
2 access intentional-fallback java-jdbc-metadata Uses Access JDBC metadata; no portable stable server-side paging API, common constraints filter locally.
3 bigquery native-pushdown java-sql Uses dataset INFORMATION_SCHEMA.TABLES with type, normal-name filter, stable order, and literal LIMIT/OFFSET.
4 cassandra intentional-fallback java-cql Uses Cassandra system_schema; portable LIKE/fuzzy paging is not safe across versions, common constraints filter locally.
5 dameng native-pushdown java-sql Uses ALL_OBJECTS with type, filter, stable order, and LIMIT/OFFSET with legacy fallback.
6 databend native-pushdown java-sql Uses system procedures for routine pushdown and shared table metadata constraints for table-like objects.
7 db2 native-pushdown java-sql Uses SYSCAT metadata with type, filter, stable order, and OFFSET/FETCH.
8 gbase8a native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
9 gbase8s native-pushdown java-sql Uses systables with type, filter, stable order, and SKIP/FIRST.
10 goldendb native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
11 h2 native-pushdown java-sql Uses INFORMATION_SCHEMA with type, filter, stable order, and LIMIT/OFFSET.
12 hive intentional-fallback java-sql Uses Hive metadata paths where stable server-side fuzzy paging is not portable, common constraints filter locally.
13 informix native-pushdown java-sql Uses systables/sysprocedures with type, filter, stable order, and SKIP/FIRST.
14 kingbase native-pushdown java-sql Uses sys_catalog/information_schema with type, filter, stable order, and LIMIT/OFFSET for both compatibility modes.
15 kylin intentional-fallback java-jdbc-metadata Uses JDBC metadata from Kylin driver; no portable server-side paging API, common constraints filter locally.
16 neo4j intentional-fallback java-graph-metadata Uses JDBC metadata with Cypher fallback; label discovery cannot guarantee relational-style filtered paging, common constraints filter locally.
17 oceanbase-oracle native-pushdown java-sql Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging.
18 oracle-go native-pushdown native-go Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging, with legacy profile guard.
19 snowflake native-pushdown java-sql Uses INFORMATION_SCHEMA with type, filter, stable order, and literal LIMIT/OFFSET.
20 sundb native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
21 tdengine intentional-fallback java-show-metadata Uses SHOW STABLES/TABLES; stable server-side fuzzy paging is not portable, common constraints filter locally.
22 trino native-pushdown java-sql Uses information_schema.tables with type, filter, stable order, and literal LIMIT/OFFSET.
23 xugu native-pushdown native-go Uses ALL_TABLES/ALL_VIEWS metadata with type, filter, stable order, and ROWNUM paging.
@@ -88,7 +88,7 @@ public final class JdbcMetadataSqlFake {
if ("getUpdateCount".equals(name)) { if ("getUpdateCount".equals(name)) {
return 0; return 0;
} }
if ("setString".equals(name)) { if ("setString".equals(name) || "setInt".equals(name) || "setObject".equals(name)) {
statements.add("param:" + args[0] + "=" + args[1]); statements.add("param:" + args[0] + "=" + args[1]);
return null; return null;
} }
@@ -480,6 +480,9 @@ async function startInsert() {
} }
} }
perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined }); perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined });
if (ok > 0) {
store.invalidateMetadataCache(cid, db, props.prefillSchema || undefined, r.tableName);
}
} }
executeResults.value = perTable; executeResults.value = perTable;
currentStep.value = "result"; currentStep.value = "result";
@@ -197,6 +197,7 @@ async function startImport() {
}, },
); );
toast(t("tableImport.success", { count: summary.rowsImported }), 2500); toast(t("tableImport.success", { count: summary.rowsImported }), 2500);
store.invalidateMetadataCache(props.prefillConnectionId, props.prefillDatabase || "", props.prefillSchema || undefined, props.prefillTable);
open.value = false; open.value = false;
} catch (e: any) { } catch (e: any) {
errorMessage.value = String(e?.message || e); errorMessage.value = String(e?.message || e);
@@ -77,8 +77,9 @@ import { buildTableDeleteTemplate, buildTableInsertTemplate, buildTableSelectTem
import { connectionFilePath, defaultSqliteBackupFileName, isMemorySqlitePath, sqliteBackupSourcePath } from "@/lib/connection/connectionFile"; import { connectionFilePath, defaultSqliteBackupFileName, isMemorySqlitePath, sqliteBackupSourcePath } from "@/lib/connection/connectionFile";
import { revealPathInFileManager } from "@/lib/backend/tauri"; import { revealPathInFileManager } from "@/lib/backend/tauri";
import { clearActiveTableReferencePayload, createTableReferencePayload, createTableReferenceDropEvent, setActiveTableReferencePayload, type QueryEditorTableReferencePayload } from "@/lib/editor/queryEditorTableDrop"; import { clearActiveTableReferencePayload, createTableReferencePayload, createTableReferenceDropEvent, setActiveTableReferencePayload, type QueryEditorTableReferencePayload } from "@/lib/editor/queryEditorTableDrop";
import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { getCachedTableMetadata, loadTableMetadata, TABLE_METADATA_CACHE_TTL_MS, tableMetadataToDataTabMeta } from "@/lib/metadata/tableMetadataCache";
import { supportsDatabaseCreation, supportsDatabaseSearch, supportsFieldLineage, supportsObjectBrowserTreeNode, supportsSchemaDiagram, supportsSqlFileExecution, supportsTableImport, supportsTableTruncate, supportsTableStructureEditing, usesTreeSchemaMode } from "@/lib/database/databaseCapabilities"; import { supportsDatabaseCreation, supportsDatabaseSearch, supportsFieldLineage, supportsObjectBrowserTreeNode, supportsSchemaDiagram, supportsSqlFileExecution, supportsTableImport, supportsTableTruncate, supportsTableStructureEditing, usesTreeSchemaMode } from "@/lib/database/databaseCapabilities";
import { copyNameForTreeNode, isDocumentBrowserTreeNode, objectSourceKindForTreeNode, shouldRunTreeNodeRowAction, treeNodeRowAction, treeNodeRowDoubleClickAction } from "@/lib/sidebar/treeNodeClick"; import { copyNameForTreeNode, isDocumentBrowserTreeNode, objectSourceKindForTreeNode, shouldRunTreeNodeRowAction, treeNodeRowAction, treeNodeRowDoubleClickAction } from "@/lib/sidebar/treeNodeClick";
import { isCopySidebarSelectionShortcut, isEditSidebarConnectionShortcut, isPasteSidebarSelectionShortcut } from "@/lib/editor/keyboardShortcuts"; import { isCopySidebarSelectionShortcut, isEditSidebarConnectionShortcut, isPasteSidebarSelectionShortcut } from "@/lib/editor/keyboardShortcuts";
@@ -223,7 +224,7 @@ const useWindowsSidebarCommentFont = isWindows();
type StructureCopyFormat = "tsv" | "markdown"; type StructureCopyFormat = "tsv" | "markdown";
type DuplicateStructureSource = TreeNode & { connectionId: string; database: string }; type DuplicateStructureSource = TreeNode & { connectionId: string; database: string };
const DATA_TAB_METADATA_TTL_MS = 30_000; const DATA_TAB_METADATA_TTL_MS = TABLE_METADATA_CACHE_TTL_MS;
const { getDatabaseOptions } = useDatabaseOptions(); const { getDatabaseOptions } = useDatabaseOptions();
const showVisibleDatabasesDialog = ref(false); const showVisibleDatabasesDialog = ref(false);
const showVisibleSchemasDialog = ref(false); const showVisibleSchemasDialog = ref(false);
@@ -1121,6 +1122,9 @@ async function openData() {
}); });
const tableSchema = connectionObjectTreeNodeSchema(config, node.database, node.schema); const tableSchema = connectionObjectTreeNodeSchema(config, node.database, node.schema);
const tableType = node.type === "view" ? "VIEW" : node.type === "materialized_view" ? "MATERIALIZED_VIEW" : (node.tableType ?? "TABLE"); const tableType = node.type === "view" ? "VIEW" : node.type === "materialized_view" ? "MATERIALIZED_VIEW" : (node.tableType ?? "TABLE");
const querySchema = config ? connectionObjectTreeQuerySchema(config, node.database, tableSchema) : (tableSchema ?? "");
const effectiveDbType = effectiveDatabaseTypeForConnection(config);
const metadataDatabaseType = effectiveDbType || config?.db_type || "";
const isSameDataTableTab = (tab: (typeof queryStore.tabs)[number]) => tab.mode === "data" && tab.connectionId === node.connectionId && tab.database === node.database && (tab.schema || "") === (tableSchema || "") && (tab.tableMeta?.tableName || tab.title) === node.label; const isSameDataTableTab = (tab: (typeof queryStore.tabs)[number]) => tab.mode === "data" && tab.connectionId === node.connectionId && tab.database === node.database && (tab.schema || "") === (tableSchema || "") && (tab.tableMeta?.tableName || tab.title) === node.label;
const existingSameTableTab = queryStore.tabs.find(isSameDataTableTab); const existingSameTableTab = queryStore.tabs.find(isSameDataTableTab);
const resetReusedDataTabState = (tab: (typeof queryStore.tabs)[number]) => { const resetReusedDataTabState = (tab: (typeof queryStore.tabs)[number]) => {
@@ -1186,7 +1190,21 @@ async function openData() {
} }
const existingTableMeta = tab?.tableMeta; const existingTableMeta = tab?.tableMeta;
const existingTableMetaAgeMs = tab?.tableMetaUpdatedAt ? Date.now() - tab.tableMetaUpdatedAt : Number.POSITIVE_INFINITY; const existingTableMetaAgeMs = tab?.tableMetaUpdatedAt ? Date.now() - tab.tableMetaUpdatedAt : Number.POSITIVE_INFINITY;
const cachedTableMeta = existingTableMeta?.tableName === node.label && existingTableMeta.schema === tableSchema && existingTableMeta.tableType === tableType && existingTableMeta.columns.length > 0 && existingTableMetaAgeMs < DATA_TAB_METADATA_TTL_MS ? existingTableMeta : undefined; const sharedCachedTableMeta = config
? getCachedTableMetadata({
connectionId: node.connectionId,
database: node.database,
schema: querySchema,
tableName: node.label,
tableType,
databaseType: metadataDatabaseType,
driverProfile: config.driver_profile || config.db_type,
})
: undefined;
const tabCachedTableMeta = existingTableMeta?.tableName === node.label && existingTableMeta.schema === tableSchema && existingTableMeta.tableType === tableType && existingTableMeta.columns.length > 0 && existingTableMetaAgeMs < DATA_TAB_METADATA_TTL_MS ? existingTableMeta : undefined;
const cachedTableMeta = sharedCachedTableMeta ? tableMetadataToDataTabMeta(sharedCachedTableMeta.metadata, tableSchema) : tabCachedTableMeta;
const cachedTableMetaAgeMs = sharedCachedTableMeta?.ageMs ?? existingTableMetaAgeMs;
const cachedTableMetaSource = sharedCachedTableMeta ? "shared" : tabCachedTableMeta ? "tab" : undefined;
queryStore.setTableMeta( queryStore.setTableMeta(
tabId, tabId,
cachedTableMeta ?? { cachedTableMeta ?? {
@@ -1218,8 +1236,6 @@ async function openData() {
logPhase("ensure-connected", { tabId }); logPhase("ensure-connected", { tabId });
if (!config) throw new Error("Connection config not found"); if (!config) throw new Error("Connection config not found");
const querySchema = connectionObjectTreeQuerySchema(config, node.database, tableSchema);
const effectiveDbType = effectiveDatabaseTypeForConnection(config);
const limit = tableOpenPageLimit(settingsStore.editorSettings.pageSize); const limit = tableOpenPageLimit(settingsStore.editorSettings.pageSize);
const refreshTableMetaInBackground = async () => { const refreshTableMetaInBackground = async () => {
const metadataStartedAt = performance.now(); const metadataStartedAt = performance.now();
@@ -1231,30 +1247,34 @@ async function openData() {
elapsed: elapsed(), elapsed: elapsed(),
}); });
try { try {
const nextColumns = await api.getColumns(node.connectionId, node.database, querySchema, node.label); const loadedMetadata = await loadTableMetadata({
const indexes = await api.listIndexes(node.connectionId, node.database, querySchema, node.label).catch(() => []); connectionId: node.connectionId,
database: node.database,
schema: querySchema,
tableName: node.label,
tableType,
databaseType: metadataDatabaseType,
driverProfile: config.driver_profile || config.db_type,
traceLogger: (event) => console.debug("[DBX][openData:metadata:trace]", { sourceTraceId: traceId, ...event }),
});
if (!isCurrentDataTab()) { if (!isCurrentDataTab()) {
console.info("[DBX][openData:metadata:stale]", { console.info("[DBX][openData:metadata:stale]", {
traceId, traceId,
tabId, tabId,
columnCount: nextColumns.length, columnCount: loadedMetadata.metadata.columns.length,
elapsed: elapsed(), elapsed: elapsed(),
}); });
return; return;
} }
const nextPrimaryKeys = editableRowIdentifierColumns(effectiveDbType, nextColumns, indexes, tableType); const nextTableMeta = tableMetadataToDataTabMeta(loadedMetadata.metadata, tableSchema);
queryStore.setTableMeta(tabId, { queryStore.setTableMeta(tabId, nextTableMeta);
schema: tableSchema,
tableName: node.label,
tableType,
columns: nextColumns,
primaryKeys: nextPrimaryKeys,
});
console.info("[DBX][openData:metadata:done]", { console.info("[DBX][openData:metadata:done]", {
traceId, traceId,
tabId, tabId,
columnCount: nextColumns.length, columnCount: nextTableMeta.columns.length,
primaryKeyCount: nextPrimaryKeys.length, primaryKeyCount: nextTableMeta.primaryKeys.length,
cacheStatus: loadedMetadata.cacheStatus,
ageMs: Math.round(loadedMetadata.ageMs),
elapsed: elapsed(), elapsed: elapsed(),
metadataMs: Math.round(performance.now() - metadataStartedAt), metadataMs: Math.round(performance.now() - metadataStartedAt),
}); });
@@ -1269,7 +1289,8 @@ async function openData() {
tabId, tabId,
columnCount: cachedTableMeta.columns.length, columnCount: cachedTableMeta.columns.length,
primaryKeyCount: cachedTableMeta.primaryKeys.length, primaryKeyCount: cachedTableMeta.primaryKeys.length,
ageMs: Math.round(existingTableMetaAgeMs), source: cachedTableMetaSource,
ageMs: Math.round(cachedTableMetaAgeMs),
elapsed: elapsed(), elapsed: elapsed(),
}); });
} else { } else {
+4 -1
View File
@@ -518,13 +518,16 @@ export async function getTableComment(_connectionId: string, _database: string,
throw new Error("Table comment lookup is not available in the web backend"); throw new Error("Table comment lookup is not available in the web backend");
} }
export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[]): Promise<ObjectInfo[]> { export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[], filter?: string, limit?: number, offset?: number): Promise<ObjectInfo[]> {
return get( return get(
`/api/schema/objects?${qs({ `/api/schema/objects?${qs({
connection_id: connectionId, connection_id: connectionId,
database, database,
schema, schema,
object_types: objectTypes?.join(","), object_types: objectTypes?.join(","),
filter,
limit,
offset,
})}`, })}`,
); );
} }
+2 -2
View File
@@ -610,8 +610,8 @@ export async function getTableComment(connectionId: string, database: string, sc
return invoke("get_table_comment", { connectionId, database, schema, table }); return invoke("get_table_comment", { connectionId, database, schema, table });
} }
export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[]): Promise<ObjectInfo[]> { export async function listObjects(connectionId: string, database: string, schema: string, objectTypes?: SidebarObjectKind[], filter?: string, limit?: number, offset?: number): Promise<ObjectInfo[]> {
return invoke("list_objects", { connectionId, database, schema, objectTypes }); return invoke("list_objects", { connectionId, database, schema, objectTypes, filter, limit, offset });
} }
export async function listObjectStatistics(connectionId: string, database: string, schema: string): Promise<ObjectStatistics[]> { export async function listObjectStatistics(connectionId: string, database: string, schema: string): Promise<ObjectStatistics[]> {
@@ -0,0 +1,145 @@
import { metadataScopeKey, type MetadataScopeInput } from "./metadataLoadScope";
export type MetadataLoadCacheStatus = "none" | "hit" | "miss" | "stale" | "refresh";
export interface MetadataLoadTraceDetails {
cacheStatus?: MetadataLoadCacheStatus;
resultCount?: number;
deduped?: boolean;
force?: boolean;
stale?: boolean;
superseded?: boolean;
error?: unknown;
}
export interface MetadataLoadTraceEvent extends MetadataLoadTraceDetails {
event: string;
traceId: string;
scopeKey: string;
elapsedMs: number;
}
export type MetadataLoadTraceLogger = (event: MetadataLoadTraceEvent) => void;
export interface MetadataLoadTrace {
traceId: string;
scopeKey: string;
startedAt: number;
elapsedMs: () => number;
event: (event: string, details?: MetadataLoadTraceDetails) => MetadataLoadTraceEvent;
}
export interface MetadataLoadCoordinatorEvent {
event: "start" | "dedupe" | "done" | "error";
key: string;
kind: string;
active: number;
force: boolean;
}
export type MetadataLoadCoordinatorLogger = (event: MetadataLoadCoordinatorEvent) => void;
export interface MetadataLoadCoordinatorOptions {
force?: boolean;
kind?: string;
}
interface InFlightLoad<T> {
kind: string;
promise: Promise<T>;
}
function defaultTraceId() {
return Math.random().toString(16).slice(2, 10);
}
function defaultNow() {
return typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
}
export function createMetadataLoadTrace(
scope: MetadataScopeInput | string,
options?: {
traceId?: string;
now?: () => number;
},
): MetadataLoadTrace {
const now = options?.now ?? defaultNow;
const startedAt = now();
const scopeKey = typeof scope === "string" ? scope : metadataScopeKey(scope);
const traceId = options?.traceId ?? defaultTraceId();
const elapsedMs = () => Math.max(0, Math.round(now() - startedAt));
return {
traceId,
scopeKey,
startedAt,
elapsedMs,
event: (event, details = {}) => ({
event,
traceId,
scopeKey,
elapsedMs: elapsedMs(),
...details,
}),
};
}
export function logMetadataLoadTrace(logger: MetadataLoadTraceLogger | undefined, trace: MetadataLoadTrace, event: string, details?: MetadataLoadTraceDetails) {
logger?.(trace.event(event, details));
}
export class MetadataLoadCoordinator {
private readonly inFlight = new Map<string, InFlightLoad<unknown>>();
constructor(private readonly logger?: MetadataLoadCoordinatorLogger) {}
run<T>(scope: MetadataScopeInput | string, load: () => Promise<T>, options?: MetadataLoadCoordinatorOptions): Promise<T> {
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
const kind = options?.kind ?? (typeof scope === "string" ? "metadata" : scope.kind);
const force = options?.force === true;
const existing = force ? undefined : this.inFlight.get(key);
if (existing) {
this.logger?.({ event: "dedupe", key, kind: existing.kind, active: this.inFlight.size, force: false });
return existing.promise as Promise<T>;
}
this.logger?.({ event: "start", key, kind, active: this.inFlight.size + 1, force });
const promise = Promise.resolve()
.then(load)
.then(
(value) => {
this.logger?.({ event: "done", key, kind, active: this.inFlight.size, force });
return value;
},
(error) => {
this.logger?.({ event: "error", key, kind, active: this.inFlight.size, force });
throw error;
},
)
.finally(() => {
const current = this.inFlight.get(key);
if (current?.promise === promise) {
this.inFlight.delete(key);
}
});
if (!force) {
this.inFlight.set(key, { kind, promise });
}
return promise;
}
has(scope: MetadataScopeInput | string): boolean {
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
return this.inFlight.has(key);
}
clear(scope?: MetadataScopeInput | string) {
if (scope === undefined) {
this.inFlight.clear();
return;
}
const key = typeof scope === "string" ? scope : metadataScopeKey(scope);
this.inFlight.delete(key);
}
}
@@ -0,0 +1,86 @@
export type MetadataScopeValue = string | number | boolean | null | undefined;
export interface MetadataScopeInput {
kind: string;
connectionId?: string | null;
database?: string | null;
schema?: string | null;
nodeKind?: string | null;
tableName?: string | null;
tableType?: string | null;
objectTypes?: readonly (string | null | undefined)[] | null;
searchFilter?: string | null;
limit?: number | null;
offset?: number | null;
sidebarDisplayMode?: string | null;
driverProfile?: string | null;
extra?: Record<string, MetadataScopeValue | readonly MetadataScopeValue[]> | null;
}
function normalizeString(value: string | null | undefined): string | undefined {
if (value == null) return undefined;
return value;
}
function normalizeSearchFilter(value: string | null | undefined): string | undefined {
const normalized = value?.trim();
return normalized ? normalized : undefined;
}
function normalizeNumber(value: number | null | undefined): number | undefined {
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
}
function normalizeObjectTypes(values: readonly (string | null | undefined)[] | null | undefined): string[] | undefined {
if (!values?.length) return undefined;
const normalized = values
.filter((value): value is string => typeof value === "string" && value.trim().length > 0)
.map((value) => value.trim().toUpperCase().replace(/\s+/g, "_"))
.sort();
return normalized.length > 0 ? [...new Set(normalized)] : undefined;
}
function stableEntries(record: Record<string, unknown>): [string, unknown][] {
return Object.entries(record)
.filter(([, value]) => value !== undefined)
.sort(([left], [right]) => left.localeCompare(right));
}
function normalizeExtra(extra: MetadataScopeInput["extra"]): Record<string, unknown> | undefined {
if (!extra) return undefined;
const entries = stableEntries(extra)
.map(([key, value]) => {
if (Array.isArray(value)) {
return [key, value.filter((item) => item !== undefined)] as const;
}
return [key, value] as const;
})
.filter(([, value]) => {
if (Array.isArray(value)) return value.length > 0;
return value !== undefined;
});
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
}
export function metadataScopeParts(input: MetadataScopeInput): Record<string, unknown> {
return {
kind: input.kind,
connectionId: normalizeString(input.connectionId),
database: normalizeString(input.database),
schema: normalizeString(input.schema),
nodeKind: normalizeString(input.nodeKind),
tableName: normalizeString(input.tableName),
tableType: normalizeString(input.tableType),
objectTypes: normalizeObjectTypes(input.objectTypes),
searchFilter: normalizeSearchFilter(input.searchFilter),
limit: normalizeNumber(input.limit),
offset: normalizeNumber(input.offset),
sidebarDisplayMode: normalizeString(input.sidebarDisplayMode),
driverProfile: normalizeString(input.driverProfile),
extra: normalizeExtra(input.extra),
};
}
export function metadataScopeKey(input: MetadataScopeInput): string {
return JSON.stringify(stableEntries(metadataScopeParts(input)));
}
@@ -0,0 +1,99 @@
import { metadataScopeKey, metadataScopeParts, type MetadataScopeInput } from "./metadataLoadScope";
export interface MetadataCacheHit<T> {
value: T;
cachedAt: number;
ageMs: number;
stale: boolean;
}
export type MetadataCacheInvalidation = Partial<Omit<MetadataScopeInput, "extra">>;
interface MetadataCacheEntry<T> {
scope: ReturnType<typeof metadataScopeParts>;
value: T;
cachedAt: number;
}
export interface MetadataResultCacheOptions {
ttlMs: number;
maxEntries: number;
now?: () => number;
}
function normalizeInvalidation(match: MetadataCacheInvalidation): Partial<ReturnType<typeof metadataScopeParts>> {
const normalized = metadataScopeParts({ ...match, kind: match.kind ?? "__metadata_cache_match__" });
const result: Partial<ReturnType<typeof metadataScopeParts>> = {};
for (const key of Object.keys(match) as (keyof MetadataCacheInvalidation)[]) {
const value = normalized[key as keyof typeof normalized];
if (value !== undefined) {
(result as Record<string, unknown>)[key] = value;
}
}
return result;
}
function valuesEqual(left: unknown, right: unknown): boolean {
if (Array.isArray(left) || Array.isArray(right) || (left && typeof left === "object") || (right && typeof right === "object")) {
return JSON.stringify(left) === JSON.stringify(right);
}
return left === right;
}
export class MetadataResultCache<T> {
private readonly entries = new Map<string, MetadataCacheEntry<T>>();
private readonly now: () => number;
constructor(private readonly options: MetadataResultCacheOptions) {
this.now = options.now ?? Date.now;
}
get(scope: MetadataScopeInput, options?: { allowStale?: boolean }): MetadataCacheHit<T> | undefined {
const key = metadataScopeKey(scope);
const entry = this.entries.get(key);
if (!entry) return undefined;
const ageMs = Math.max(0, this.now() - entry.cachedAt);
const stale = ageMs > this.options.ttlMs;
if (stale && !options?.allowStale) return undefined;
this.entries.delete(key);
this.entries.set(key, entry);
return { value: entry.value, cachedAt: entry.cachedAt, ageMs, stale };
}
set(scope: MetadataScopeInput, value: T): void {
const key = metadataScopeKey(scope);
this.entries.delete(key);
this.entries.set(key, { scope: metadataScopeParts(scope), value, cachedAt: this.now() });
this.evictOldest();
}
invalidate(match: MetadataCacheInvalidation): number {
const normalized = normalizeInvalidation(match);
let removed = 0;
for (const [key, entry] of this.entries) {
const matches = Object.entries(normalized).every(([field, value]) => valuesEqual(entry.scope[field as keyof typeof entry.scope], value));
if (matches) {
this.entries.delete(key);
removed++;
}
}
return removed;
}
clear(): void {
this.entries.clear();
}
get size(): number {
return this.entries.size;
}
private evictOldest(): void {
while (this.entries.size > this.options.maxEntries) {
const oldest = this.entries.keys().next().value;
if (oldest === undefined) return;
this.entries.delete(oldest);
}
}
}
@@ -0,0 +1,126 @@
import type { ColumnInfo, DatabaseType, IndexInfo, QueryTab } from "@/types/database";
import * as api from "@/lib/backend/api";
import { editableRowIdentifierColumns } from "@/lib/table/tableEditing";
import { createMetadataLoadTrace, logMetadataLoadTrace, MetadataLoadCoordinator, type MetadataLoadCacheStatus, type MetadataLoadTraceLogger } from "./metadataLoadCoordinator";
import type { MetadataScopeInput } from "./metadataLoadScope";
import { MetadataResultCache, type MetadataCacheInvalidation } from "./metadataResultCache";
export const TABLE_METADATA_CACHE_TTL_MS = 30_000;
const TABLE_METADATA_CACHE_MAX_ENTRIES = 120;
export interface TableMetadata {
schema?: string;
tableName: string;
tableType?: string;
columns: ColumnInfo[];
indexes: IndexInfo[];
primaryKeys: string[];
cachedAt: number;
}
export interface TableMetadataRequest {
connectionId: string;
database: string;
schema?: string;
tableName: string;
tableType?: string;
databaseType: DatabaseType | string;
driverProfile?: string;
force?: boolean;
traceLogger?: MetadataLoadTraceLogger;
}
export interface TableMetadataLoadResult {
metadata: TableMetadata;
cacheStatus: MetadataLoadCacheStatus;
ageMs: number;
}
const tableMetadataCache = new MetadataResultCache<TableMetadata>({
ttlMs: TABLE_METADATA_CACHE_TTL_MS,
maxEntries: TABLE_METADATA_CACHE_MAX_ENTRIES,
});
const tableMetadataCoordinator = new MetadataLoadCoordinator((event) => {
console.debug("[DBX][metadata-load:table-coordinator]", event);
});
export function tableMetadataScope(request: Pick<TableMetadataRequest, "connectionId" | "database" | "schema" | "tableName" | "tableType" | "driverProfile" | "databaseType">): MetadataScopeInput {
return {
kind: "table-metadata",
connectionId: request.connectionId,
database: request.database,
schema: request.schema ?? "",
tableName: request.tableName,
tableType: request.tableType,
driverProfile: request.driverProfile || request.databaseType,
};
}
export function getCachedTableMetadata(request: Pick<TableMetadataRequest, "connectionId" | "database" | "schema" | "tableName" | "tableType" | "driverProfile" | "databaseType">): TableMetadataLoadResult | undefined {
const hit = tableMetadataCache.get(tableMetadataScope(request));
if (!hit) return undefined;
return { metadata: hit.value, cacheStatus: hit.stale ? "stale" : "hit", ageMs: hit.ageMs };
}
export function tableMetadataToDataTabMeta(metadata: TableMetadata, schema = metadata.schema): NonNullable<QueryTab["tableMeta"]> {
return {
schema,
tableName: metadata.tableName,
tableType: metadata.tableType,
columns: metadata.columns,
primaryKeys: metadata.primaryKeys,
};
}
export async function loadTableMetadata(request: TableMetadataRequest): Promise<TableMetadataLoadResult> {
const scope = tableMetadataScope(request);
const trace = createMetadataLoadTrace(scope);
if (!request.force) {
const cached = tableMetadataCache.get(scope);
if (cached) {
logMetadataLoadTrace(request.traceLogger, trace, "cache-hit", {
cacheStatus: cached.stale ? "stale" : "hit",
resultCount: cached.value.columns.length,
stale: cached.stale,
});
return { metadata: cached.value, cacheStatus: cached.stale ? "stale" : "hit", ageMs: cached.ageMs };
}
}
logMetadataLoadTrace(request.traceLogger, trace, "cache-miss", { cacheStatus: request.force ? "refresh" : "miss", force: request.force === true });
const metadata = await tableMetadataCoordinator.run(
scope,
async () => {
const columns = await api.getColumns(request.connectionId, request.database, request.schema ?? "", request.tableName);
const indexes = await api.listIndexes(request.connectionId, request.database, request.schema ?? "", request.tableName).catch((): IndexInfo[] => []);
const primaryKeys = editableRowIdentifierColumns(request.databaseType as DatabaseType, columns, indexes, request.tableType);
return {
schema: request.schema || undefined,
tableName: request.tableName,
tableType: request.tableType,
columns,
indexes,
primaryKeys,
cachedAt: Date.now(),
};
},
{ force: request.force, kind: scope.kind },
);
tableMetadataCache.set(scope, metadata);
logMetadataLoadTrace(request.traceLogger, trace, "done", {
cacheStatus: request.force ? "refresh" : "miss",
resultCount: metadata.columns.length,
force: request.force === true,
});
return { metadata, cacheStatus: request.force ? "refresh" : "miss", ageMs: 0 };
}
export function invalidateTableMetadataCache(match: MetadataCacheInvalidation): number {
return tableMetadataCache.invalidate(match);
}
export function clearTableMetadataCache(): void {
tableMetadataCache.clear();
}
@@ -2,7 +2,7 @@ import type { FlatTreeNode } from "@/composables/useFlatTree";
import type { TreeNode, TreeNodeType } from "@/types/database"; import type { TreeNode, TreeNodeType } from "@/types/database";
const simpleObjectParentTypes = new Set<TreeNodeType>(["database", "schema", "linked-server-schema"]); const simpleObjectParentTypes = new Set<TreeNodeType>(["database", "schema", "linked-server-schema"]);
const simpleObjectChildTypes = new Set<TreeNodeType>(["table", "view", "materialized_view", "procedure", "function", "sequence", "package", "package-body", "load-more"]); const tableSearchableChildTypes = new Set<TreeNodeType>(["table", "view", "materialized_view", "load-more"]);
export function isSidebarTableSearchControlNode(node: TreeNode): boolean { export function isSidebarTableSearchControlNode(node: TreeNode): boolean {
return node.type === "table-search-control"; return node.type === "table-search-control";
@@ -12,18 +12,19 @@ function tableSearchControlId(parentId: string): string {
return `${parentId}:__table_search`; return `${parentId}:__table_search`;
} }
function parentHasDirectTableList(node: TreeNode): boolean { function parentHasSearchableTableList(node: TreeNode): boolean {
return !!node.children?.some((child) => simpleObjectChildTypes.has(child.type)); // Routine-only schemas should not show a table-specific search box.
return !!node.children?.some((child) => tableSearchableChildTypes.has(child.type));
} }
function shouldInsertTableSearchControl(item: FlatTreeNode, sidebarObjectDisplay: "simple" | "grouped", activeQueries: Readonly<Record<string, string | undefined>>): boolean { function shouldInsertTableSearchControl(item: FlatTreeNode, sidebarObjectDisplay: "simple" | "grouped", activeQueries: Readonly<Record<string, string | undefined>>): boolean {
const node = item.node; const node = item.node;
if (!node.isExpanded) return false; if (!node.isExpanded) return false;
if (sidebarObjectDisplay === "grouped") { if (sidebarObjectDisplay === "grouped") {
return node.type === "group-tables"; return node.type === "group-tables" && (parentHasSearchableTableList(node) || !!activeQueries[node.id]?.trim());
} }
if (!simpleObjectParentTypes.has(node.type)) return false; if (!simpleObjectParentTypes.has(node.type)) return false;
return parentHasDirectTableList(node) || !!activeQueries[node.id]?.trim(); return parentHasSearchableTableList(node) || !!activeQueries[node.id]?.trim();
} }
function buildTableSearchControlNode(parent: TreeNode): TreeNode { function buildTableSearchControlNode(parent: TreeNode): TreeNode {
@@ -0,0 +1,99 @@
import { createPinia, setActivePinia } from "pinia";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { ConnectionConfig, TableInfo, TreeNode } from "@/types/database";
function installLocalStorage() {
const data = new Map<string, string>();
vi.stubGlobal("localStorage", {
getItem: vi.fn((key: string) => data.get(key) ?? null),
setItem: vi.fn((key: string, value: string) => data.set(key, value)),
removeItem: vi.fn((key: string) => data.delete(key)),
});
}
function postgresConnection(): ConnectionConfig {
return {
id: "pg-1",
name: "Postgres",
db_type: "postgres",
host: "127.0.0.1",
port: 5432,
username: "postgres",
password: "",
database: "app",
} as ConnectionConfig;
}
describe("connectionStore metadata loading", () => {
beforeEach(() => {
vi.resetModules();
vi.unstubAllGlobals();
installLocalStorage();
setActivePinia(createPinia());
});
it("renders simple-mode table children without waiting for supplemental objects", async () => {
const tables: TableInfo[] = [{ name: "users", table_type: "TABLE", comment: null }];
const listTables = vi.fn().mockResolvedValue(tables);
const listObjects = vi.fn(() => new Promise(() => undefined));
vi.doMock("@/lib/backend/tauriRuntime", () => ({ isTauriRuntime: () => false }));
vi.doMock("@/lib/backend/api", () => ({
checkConnectionHealth: vi.fn().mockResolvedValue(undefined),
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
listObjects,
listTables,
loadSchemaCache: vi.fn().mockResolvedValue(null),
saveSchemaCache: vi.fn().mockResolvedValue(undefined),
saveConnections: vi.fn().mockResolvedValue(undefined),
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
}));
const { useConnectionStore } = await import("@/stores/connectionStore");
const { useSettingsStore } = await import("@/stores/settingsStore");
const store = useConnectionStore();
const settingsStore = useSettingsStore();
settingsStore.editorSettings.sidebarObjectDisplay = "simple";
const connection = postgresConnection();
const schemaNode: TreeNode = {
id: "pg-1:app:public",
label: "public",
type: "schema",
connectionId: connection.id,
database: "app",
schema: "public",
isExpanded: false,
children: [],
};
store.connections = [connection];
store.connectedIds.add(connection.id);
store.treeNodes = [
{
id: connection.id,
label: connection.name,
type: "connection",
connectionId: connection.id,
isExpanded: true,
children: [
{
id: "pg-1:app",
label: "app",
type: "database",
connectionId: connection.id,
database: "app",
isExpanded: true,
children: [schemaNode],
},
],
},
];
const result = await Promise.race([store.loadTables(connection.id, "app", "public").then(() => "done"), new Promise((resolve) => setTimeout(() => resolve("timeout"), 50))]);
expect(result).toBe("done");
expect(listTables).toHaveBeenCalledWith(connection.id, "app", "public", undefined, 1001, 0);
expect(listObjects).toHaveBeenCalled();
expect(schemaNode.children?.map((node) => node.label)).toEqual(["users"]);
});
});
File diff suppressed because it is too large. Load diff
+20 -7
View File
@@ -27,10 +27,11 @@ import { redisCommandResultToQueryResult } from "@/lib/redis/redisQueryResult";
import { nextRedisCommandDb } from "@/lib/redis/redisCommandSession"; import { nextRedisCommandDb } from "@/lib/redis/redisCommandSession";
import { isRedisMutatingCommand } from "@/lib/redis/redisCommandTable"; import { isRedisMutatingCommand } from "@/lib/redis/redisCommandTable";
import { usesAgentCursorForQuery } from "@/lib/database/databaseDriverManifest"; import { usesAgentCursorForQuery } from "@/lib/database/databaseDriverManifest";
import { canUseKeylessRowPredicate, editableRowIdentifierColumns } from "@/lib/table/tableEditing"; import { canUseKeylessRowPredicate } from "@/lib/table/tableEditing";
import { TABLE_DATA_EXPORT_PAGE_SIZE } from "@/lib/table/tableDataExport"; import { TABLE_DATA_EXPORT_PAGE_SIZE } from "@/lib/table/tableDataExport";
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta"; import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { loadTableMetadata } from "@/lib/metadata/tableMetadataCache";
import { quoteTableIdentifier } from "@/lib/table/tableSelectSql"; import { quoteTableIdentifier } from "@/lib/table/tableSelectSql";
import { connectionUsesDatabaseObjectTreeMode, connectionUsesSchemaExecutionContext, effectiveDatabaseTypeForConnection, metadataSchemaForConnection } from "@/lib/database/jdbcDialect"; import { connectionUsesDatabaseObjectTreeMode, connectionUsesSchemaExecutionContext, effectiveDatabaseTypeForConnection, metadataSchemaForConnection } from "@/lib/database/jdbcDialect";
import { frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout"; import { frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
@@ -1771,21 +1772,33 @@ export const useQueryStore = defineStore("query", () => {
const metadataAnalysis = normalizeOracleLikeQueryAnalysis(dbType, analysis, metadataSchema || undefined, metadataTableName); const metadataAnalysis = normalizeOracleLikeQueryAnalysis(dbType, analysis, metadataSchema || undefined, metadataTableName);
try { try {
console.info("[DBX][executeTabSql:metadata:get-columns:start]", { console.info("[DBX][executeTabSql:metadata:table:start]", {
traceId, traceId,
schema: metadataSchema, schema: metadataSchema,
table: metadataTableName, table: metadataTableName,
elapsed: elapsed?.(), elapsed: elapsed?.(),
}); });
const columns = await api.getColumns(tab.connectionId, tab.database, metadataSchema, metadataTableName); const loadedMetadata = await loadTableMetadata({
console.info("[DBX][executeTabSql:metadata:get-columns:done]", { connectionId: tab.connectionId,
database: tab.database,
schema: metadataSchema,
tableName: metadataTableName,
tableType: tab.tableMeta?.tableType,
databaseType: dbType,
driverProfile: conn?.driver_profile || conn?.db_type,
traceLogger: (event) => console.debug("[DBX][executeTabSql:metadata:table-trace]", { sourceTraceId: traceId, ...event }),
});
const columns = loadedMetadata.metadata.columns;
const primaryKeys = loadedMetadata.metadata.primaryKeys;
console.info("[DBX][executeTabSql:metadata:table:done]", {
traceId, traceId,
columnCount: columns.length, columnCount: columns.length,
primaryKeyCount: primaryKeys.length,
cacheStatus: loadedMetadata.cacheStatus,
ageMs: Math.round(loadedMetadata.ageMs),
elapsed: elapsed?.(), elapsed: elapsed?.(),
}); });
const indexes = await api.listIndexes(tab.connectionId, tab.database, metadataSchema, metadataTableName).catch(() => []); const tableType = loadedMetadata.metadata.tableType;
const tableType = tab.tableMeta?.tableType;
const primaryKeys = editableRowIdentifierColumns(dbType as DatabaseType, columns, indexes, tableType);
const tableMeta = { const tableMeta = {
schema: metadataSchema || undefined, schema: metadataSchema || undefined,
tableName: metadataTableName, tableName: metadataTableName,
+11 -2
View File
@@ -782,8 +782,17 @@ pub async fn export_database_sql_core(
let mut functions: Vec<String> = Vec::new(); let mut functions: Vec<String> = Vec::new();
if request.include_objects && request.selected_tables.is_empty() { if request.include_objects && request.selected_tables.is_empty() {
if let Ok(objects) = if let Ok(objects) = crate::schema::list_objects_core(
crate::schema::list_objects_core(state, &request.connection_id, &request.database, &request.schema).await state,
&request.connection_id,
&request.database,
&request.schema,
None,
None,
None,
None,
)
.await
{ {
for obj in &objects { for obj in &objects {
let ot = obj.object_type.to_uppercase(); let ot = obj.object_type.to_uppercase();
+49 -2
View File
@@ -628,8 +628,30 @@ impl AgentDriverClient {
schema: &str, schema: &str,
object_types: Option<&[String]>, object_types: Option<&[String]>,
timeout_duration: Option<Duration>, timeout_duration: Option<Duration>,
) -> Result<T, String> {
self.list_tables_constrained(database, schema, None, None, None, object_types, timeout_duration).await
}
pub async fn list_tables_constrained<T: DeserializeOwned + Send + 'static>(
&mut self,
database: &str,
schema: &str,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
timeout_duration: Option<Duration>,
) -> Result<T, String> { ) -> Result<T, String> {
let mut params = agent_schema_params(database, schema); let mut params = agent_schema_params(database, schema);
if let Some(filter) = filter.map(str::trim).filter(|filter| !filter.is_empty()) {
params["filter"] = serde_json::json!(filter);
}
if let Some(limit) = limit {
params["limit"] = serde_json::json!(limit);
}
if let Some(offset) = offset {
params["offset"] = serde_json::json!(offset);
}
if let Some(object_types) = object_types { if let Some(object_types) = object_types {
params["object_types"] = serde_json::json!(object_types); params["object_types"] = serde_json::json!(object_types);
} }
@@ -642,8 +664,33 @@ impl AgentDriverClient {
schema: &str, schema: &str,
timeout_duration: Option<Duration>, timeout_duration: Option<Duration>,
) -> Result<T, String> { ) -> Result<T, String> {
self.call_method_with_timeout(AgentMethod::ListObjects, agent_schema_params(database, schema), timeout_duration) self.list_objects_constrained(database, schema, None, None, None, None, timeout_duration).await
.await }
pub async fn list_objects_constrained<T: DeserializeOwned + Send + 'static>(
&mut self,
database: &str,
schema: &str,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
timeout_duration: Option<Duration>,
) -> Result<T, String> {
let mut params = agent_schema_params(database, schema);
if let Some(filter) = filter.map(str::trim).filter(|filter| !filter.is_empty()) {
params["filter"] = serde_json::json!(filter);
}
if let Some(limit) = limit {
params["limit"] = serde_json::json!(limit);
}
if let Some(offset) = offset {
params["offset"] = serde_json::json!(offset);
}
if let Some(object_types) = object_types {
params["object_types"] = serde_json::json!(object_types);
}
self.call_method_with_timeout(AgentMethod::ListObjects, params, timeout_duration).await
} }
pub async fn list_data_types<T: DeserializeOwned + Send + 'static>( pub async fn list_data_types<T: DeserializeOwned + Send + 'static>(
+215 -18
View File
@@ -1453,10 +1453,18 @@ async fn list_tables_once(
.await .await
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types)); .map(|tables| filter_table_infos(tables, filter, limit, offset, object_types));
} }
let mut params =
serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema });
if let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) {
params["filter"] = serde_json::json!(filter);
}
if let Some(object_types) = object_types {
params["object_types"] = serde_json::json!(object_types);
}
return session return session
.invoke_with_timeout::<Vec<db::TableInfo>>( .invoke_with_timeout::<Vec<db::TableInfo>>(
"listTables", "listTables",
serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema }), params,
agent_metadata_timeout(Some(config.as_ref())), agent_metadata_timeout(Some(config.as_ref())),
) )
.await .await
@@ -1520,16 +1528,33 @@ async fn list_tables_once(
try_sqlserver!(connections, &pool_key, list_tables, schema, filter, limit, offset); try_sqlserver!(connections, &pool_key, list_tables, schema, filter, limit, offset);
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) { if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
let is_oracle = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle); let is_oracle = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle);
let use_oracle_agent_paging = db_config.as_ref().is_some_and(is_default_oracle_agent_config);
let timeout_duration = agent_metadata_timeout(db_config.as_ref()); let timeout_duration = agent_metadata_timeout(db_config.as_ref());
let fallback_config = db_config.clone(); let fallback_config = db_config.clone();
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = client.lock().await;
let agent_limit = if use_oracle_agent_paging { limit } else { None };
let agent_offset = if use_oracle_agent_paging { offset } else { None };
match client match client
.list_tables_filtered::<Vec<db::TableInfo>>(database, schema, object_types, timeout_duration) .list_tables_constrained::<Vec<db::TableInfo>>(
database,
schema,
filter,
agent_limit,
agent_offset,
object_types,
timeout_duration,
)
.await .await
{ {
Ok(tables) if !tables.is_empty() => { Ok(tables) if !tables.is_empty() => {
let mut tables = filter_table_infos(tables, filter, limit, offset, object_types); let final_offset =
if oracle_agent_paging_likely_applied(use_oracle_agent_paging, limit, tables.len()) {
Some(0)
} else {
offset
};
let mut tables = filter_table_infos(tables, filter, limit, final_offset, object_types);
if is_oracle { if is_oracle {
load_oracle_table_comments_for_tables( load_oracle_table_comments_for_tables(
&mut client, &mut client,
@@ -1685,6 +1710,50 @@ fn filter_table_infos(
.collect() .collect()
} }
fn filter_object_infos(
objects: Vec<db::ObjectInfo>,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
) -> Vec<db::ObjectInfo> {
let filter = filter.unwrap_or("");
let limit = limit.unwrap_or(usize::MAX);
let offset = offset.unwrap_or(0);
objects
.into_iter()
.filter(|object| crate::sql::contains_or_fuzzy_match(&object.name, filter))
.filter(|object| object_info_matches_object_types(object, object_types))
.skip(offset)
.take(limit)
.collect()
}
fn object_info_matches_object_types(object: &db::ObjectInfo, object_types: Option<&[String]>) -> bool {
let Some(object_types) = object_types else {
return true;
};
if object_types.is_empty() {
return true;
}
let object_type = normalize_object_info_object_type(&object.object_type);
object_types.iter().any(|expected| normalize_object_info_object_type(expected) == object_type)
}
fn normalize_object_info_object_type(value: &str) -> String {
let upper = value.to_ascii_uppercase().replace(' ', "_");
if upper.contains("MATERIALIZED") && upper.contains("VIEW") {
return "MATERIALIZED_VIEW".to_string();
}
if upper == "BASE_TABLE" || upper.contains("TABLE") {
return "TABLE".to_string();
}
if upper.contains("VIEW") {
return "VIEW".to_string();
}
upper
}
fn table_info_matches_object_types(table: &db::TableInfo, object_types: Option<&[String]>) -> bool { fn table_info_matches_object_types(table: &db::TableInfo, object_types: Option<&[String]>) -> bool {
let Some(object_types) = object_types else { let Some(object_types) = object_types else {
return true; return true;
@@ -1750,8 +1819,12 @@ async fn external_driver_presto_like_objects(
config: &ConnectionConfig, config: &ConnectionConfig,
database: &str, database: &str,
schema: &str, schema: &str,
filter: Option<&str>,
object_types: Option<&[String]>,
) -> Result<Vec<db::ObjectInfo>, String> { ) -> Result<Vec<db::ObjectInfo>, String> {
let tables = external_driver_presto_like_tables(session, config, database, schema, None, None, None).await?; let tables = external_driver_presto_like_tables(session, config, database, schema, filter, None, None)
.await
.map(|tables| filter_table_infos(tables, filter, None, None, object_types))?;
Ok(tables Ok(tables
.into_iter() .into_iter()
.map(|table| db::ObjectInfo { .map(|table| db::ObjectInfo {
@@ -1953,10 +2026,10 @@ mod tests {
use super::db; use super::db;
use super::{ use super::{
clickhouse_metadata_database, deduplicate_column_infos, filter_mysql_system_databases_for_config, clickhouse_metadata_database, deduplicate_column_infos, filter_mysql_system_databases_for_config,
filter_table_infos, filter_visible_schema_names, is_agent_postgres_metadata_fallback_config, filter_object_infos, filter_table_infos, filter_visible_schema_names,
is_retryable_metadata_error, mysql_object_source_sql, mysql_table_metadata_catalog, is_agent_postgres_metadata_fallback_config, is_retryable_metadata_error, mysql_object_source_sql,
normalize_information_schema_table_type, oracle_columns_from_query_result, oracle_columns_sql, mysql_table_metadata_catalog, normalize_information_schema_table_type, oracle_columns_from_query_result,
oracle_object_statistics_dba_segments_sql, oracle_object_statistics_from_query_result, oracle_columns_sql, oracle_object_statistics_dba_segments_sql, oracle_object_statistics_from_query_result,
oracle_object_statistics_rows_only_sql, oracle_object_statistics_sql, oracle_object_statistics_rows_only_sql, oracle_object_statistics_sql,
oracle_object_statistics_user_segments_sql, oracle_table_comment_from_query_result, oracle_table_comment_sql, oracle_object_statistics_user_segments_sql, oracle_table_comment_from_query_result, oracle_table_comment_sql,
oracle_table_comments_from_query_result, oracle_table_comments_sql, presto_like_columns_from_query_result, oracle_table_comments_from_query_result, oracle_table_comments_sql, presto_like_columns_from_query_result,
@@ -2083,6 +2156,30 @@ mod tests {
assert_eq!(visible_schema_filter(Some(&config), "OTHER", true), None); assert_eq!(visible_schema_filter(Some(&config), "OTHER", true), None);
} }
#[test]
fn default_oracle_agent_config_excludes_legacy_profiles() {
let mut config = test_connection_config(DatabaseType::Oracle);
assert!(super::is_default_oracle_agent_config(&config));
config.driver_profile = Some("oracle".to_string());
assert!(super::is_default_oracle_agent_config(&config));
config.driver_profile = Some("oracle-legacy".to_string());
assert!(!super::is_default_oracle_agent_config(&config));
config.driver_profile = Some("oracle-10g".to_string());
assert!(!super::is_default_oracle_agent_config(&config));
}
#[test]
fn oracle_agent_paging_detection_avoids_double_offset_only_when_page_sized() {
assert!(super::oracle_agent_paging_likely_applied(true, Some(500), 500));
assert!(super::oracle_agent_paging_likely_applied(true, Some(500), 120));
assert!(!super::oracle_agent_paging_likely_applied(true, Some(500), 501));
assert!(!super::oracle_agent_paging_likely_applied(false, Some(500), 120));
assert!(!super::oracle_agent_paging_likely_applied(true, None, 120));
}
#[test] #[test]
fn filter_visible_schema_names_preserves_database_order() { fn filter_visible_schema_names_preserves_database_order() {
let schemas = vec!["APP".to_string(), "SYS".to_string(), "REPORTING".to_string()]; let schemas = vec!["APP".to_string(), "SYS".to_string(), "REPORTING".to_string()];
@@ -2101,6 +2198,20 @@ mod tests {
} }
} }
fn test_object_info(name: &str, object_type: &str) -> super::db::ObjectInfo {
super::db::ObjectInfo {
name: name.to_string(),
object_type: object_type.to_string(),
schema: Some("app".to_string()),
signature: None,
comment: None,
created_at: None,
updated_at: None,
parent_schema: None,
parent_name: None,
}
}
fn test_database_info(name: &str) -> super::db::DatabaseInfo { fn test_database_info(name: &str) -> super::db::DatabaseInfo {
super::db::DatabaseInfo { name: name.to_string() } super::db::DatabaseInfo { name: name.to_string() }
} }
@@ -2220,6 +2331,22 @@ mod tests {
assert_eq!(filtered[0].name, "active_users"); assert_eq!(filtered[0].name, "active_users");
} }
#[test]
fn filter_object_infos_filters_object_type_before_offset_and_limit() {
let objects = vec![
test_object_info("sync_user", "PROCEDURE"),
test_object_info("find_user", "FUNCTION"),
test_object_info("fetch_name", "FUNCTION"),
test_object_info("orders", "TABLE"),
];
let object_types = vec!["FUNCTION".to_string()];
let filtered = filter_object_infos(objects, Some("fn"), Some(1), Some(1), Some(&object_types));
assert_eq!(filtered.len(), 1);
assert_eq!(filtered[0].name, "fetch_name");
}
#[test] #[test]
fn presto_like_information_schema_sql_uses_catalog_and_schema_without_system_jdbc() { fn presto_like_information_schema_sql_uses_catalog_and_schema_without_system_jdbc() {
let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics", None, None); let sql = presto_like_information_schema_tables_sql("hive", "sales_analytics", None, None);
@@ -2793,9 +2920,26 @@ pub async fn list_objects_core(
connection_id: &str, connection_id: &str,
database: &str, database: &str,
schema: &str, schema: &str,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
) -> Result<Vec<db::ObjectInfo>, String> { ) -> Result<Vec<db::ObjectInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || { let use_oracle_agent_paging =
list_objects_once(state, connection_id, database, schema) connection_config(state, connection_id).await.as_ref().is_some_and(is_default_oracle_agent_config);
retry_metadata_connection(state, connection_id, Some(database), || async {
let objects = list_objects_once(state, connection_id, database, schema, filter, limit, offset, object_types)
.await
.map(|objects| {
let final_offset = if oracle_agent_paging_likely_applied(use_oracle_agent_paging, limit, objects.len())
{
Some(0)
} else {
offset
};
filter_object_infos(objects, filter, limit, final_offset, object_types)
})?;
Ok(objects)
}) })
.await .await
} }
@@ -3109,6 +3253,10 @@ async fn list_objects_once(
connection_id: &str, connection_id: &str,
database: &str, database: &str,
schema: &str, schema: &str,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
) -> Result<Vec<db::ObjectInfo>, String> { ) -> Result<Vec<db::ObjectInfo>, String> {
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
@@ -3144,12 +3292,28 @@ async fn list_objects_once(
let session = session.clone(); let session = session.clone();
drop(connections); drop(connections);
if uses_presto_like_information_schema_tables(&config.db_type) { if uses_presto_like_information_schema_tables(&config.db_type) {
return external_driver_presto_like_objects(session, config.as_ref(), database, schema).await; return external_driver_presto_like_objects(
session,
config.as_ref(),
database,
schema,
filter,
object_types,
)
.await;
}
let mut params =
serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema });
if let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) {
params["filter"] = serde_json::json!(filter);
}
if let Some(object_types) = object_types {
params["object_types"] = serde_json::json!(object_types);
} }
return session return session
.invoke_with_timeout::<Vec<db::ObjectInfo>>( .invoke_with_timeout::<Vec<db::ObjectInfo>>(
"listObjects", "listObjects",
serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema }), params,
agent_metadata_timeout(Some(config.as_ref())), agent_metadata_timeout(Some(config.as_ref())),
) )
.await; .await;
@@ -3157,18 +3321,41 @@ async fn list_objects_once(
try_sqlserver!(connections, &pool_key, list_objects, schema); try_sqlserver!(connections, &pool_key, list_objects, schema);
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) { if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
let is_oracle = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle); let is_oracle = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle);
let use_oracle_agent_paging = db_config.as_ref().is_some_and(is_default_oracle_agent_config);
let timeout_duration = agent_metadata_timeout(db_config.as_ref());
let fallback_config = db_config.clone(); let fallback_config = db_config.clone();
drop(connections); drop(connections);
if is_oracle { if is_oracle && !use_oracle_agent_paging {
return oracle_agent_list_objects(client, database, schema, agent_metadata_timeout(db_config.as_ref())) return oracle_agent_list_objects(client, database, schema, timeout_duration).await;
.await;
} }
let mut client = client.lock().await; let mut client = client.lock().await;
let agent_limit = if use_oracle_agent_paging { limit } else { None };
let agent_offset = if use_oracle_agent_paging { offset } else { None };
match client match client
.list_objects::<Vec<db::ObjectInfo>>(database, schema, agent_metadata_timeout(db_config.as_ref())) .list_objects_constrained::<Vec<db::ObjectInfo>>(
database,
schema,
filter,
agent_limit,
agent_offset,
object_types,
timeout_duration,
)
.await .await
{ {
Ok(objects) if !objects.is_empty() => return Ok(objects), Ok(mut objects) if !objects.is_empty() => {
if is_oracle {
load_oracle_table_comments_for_objects(
&mut client,
database,
schema,
&mut objects,
timeout_duration,
)
.await?;
}
return Ok(objects);
}
Ok(objects) => { Ok(objects) => {
if let Some(config) = fallback_config.as_ref() { if let Some(config) = fallback_config.as_ref() {
match native_postgres_metadata_pool(state, connection_id, database, config).await { match native_postgres_metadata_pool(state, connection_id, database, config).await {
@@ -3338,7 +3525,7 @@ async fn list_completion_objects_once(
PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(filter_completion_objects), PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(filter_completion_objects),
PoolKind::SqlServer(_) => { PoolKind::SqlServer(_) => {
drop(connections); drop(connections);
let objects = list_objects_once(state, connection_id, database, schema).await?; let objects = list_objects_once(state, connection_id, database, schema, None, None, None, None).await?;
Ok(filter_completion_objects(objects)) Ok(filter_completion_objects(objects))
} }
_ => Ok(Vec::new()), _ => Ok(Vec::new()),
@@ -4070,6 +4257,16 @@ fn is_opengauss_family_config(config: &ConnectionConfig) -> bool {
|| matches!(config.driver_profile.as_deref(), Some("opengauss" | "gaussdb")) || matches!(config.driver_profile.as_deref(), Some("opengauss" | "gaussdb"))
} }
fn is_default_oracle_agent_config(config: &ConnectionConfig) -> bool {
// Only the default go-oracle agent handles filtered/paged metadata; legacy profiles keep Rust fallback paging.
matches!(config.db_type, DatabaseType::Oracle)
&& !matches!(config.driver_profile.as_deref(), Some("oracle-legacy" | "oracle-10g"))
}
fn oracle_agent_paging_likely_applied(enabled: bool, limit: Option<usize>, returned_len: usize) -> bool {
enabled && limit.is_some_and(|limit| returned_len <= limit)
}
fn is_doris_family_config(config: &ConnectionConfig) -> bool { fn is_doris_family_config(config: &ConnectionConfig) -> bool {
matches!(config.db_type, DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::ManticoreSearch) matches!(config.db_type, DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::ManticoreSearch)
|| matches!(config.driver_profile.as_deref(), Some("doris" | "selectdb" | "starrocks" | "manticoresearch")) || matches!(config.driver_profile.as_deref(), Some("doris" | "selectdb" | "starrocks" | "manticoresearch"))
@@ -0,0 +1,169 @@
import assert from "node:assert/strict";
import { test } from "vitest";
import { createMetadataLoadTrace, MetadataLoadCoordinator } from "../../apps/desktop/src/lib/metadata/metadataLoadCoordinator.ts";
import { metadataScopeKey } from "../../apps/desktop/src/lib/metadata/metadataLoadScope.ts";
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
test("metadata scope key treats equivalent object type sets as equal", () => {
const left = metadataScopeKey({
kind: "object-group",
connectionId: "conn",
database: "app",
schema: "public",
nodeKind: "group-tables",
objectTypes: ["view", "TABLE", "TABLE"],
searchFilter: " user ",
limit: 501,
offset: 0,
sidebarDisplayMode: "grouped",
driverProfile: "postgres",
});
const right = metadataScopeKey({
kind: "object-group",
connectionId: "conn",
database: "app",
schema: "public",
nodeKind: "group-tables",
objectTypes: ["TABLE", "VIEW"],
searchFilter: "user",
limit: 501,
offset: 0,
sidebarDisplayMode: "grouped",
driverProfile: "postgres",
});
assert.equal(left, right);
});
test("metadata scope key separates result-affecting fields", () => {
const base = {
kind: "sidebar-table-search",
connectionId: "conn",
database: "app",
schema: "public",
objectTypes: ["TABLE"],
limit: 500,
offset: 0,
sidebarDisplayMode: "simple",
driverProfile: "postgres",
};
assert.notEqual(metadataScopeKey({ ...base, searchFilter: "user" }), metadataScopeKey({ ...base, searchFilter: "order" }));
assert.notEqual(metadataScopeKey({ ...base, offset: 0 }), metadataScopeKey({ ...base, offset: 500 }));
assert.notEqual(metadataScopeKey({ ...base, objectTypes: ["TABLE"] }), metadataScopeKey({ ...base, objectTypes: ["VIEW"] }));
assert.notEqual(metadataScopeKey({ ...base, sidebarDisplayMode: "simple" }), metadataScopeKey({ ...base, sidebarDisplayMode: "grouped" }));
});
test("metadata load coordinator deduplicates equivalent in-flight loads", async () => {
const events: string[] = [];
const coordinator = new MetadataLoadCoordinator((event) => events.push(event.event));
const gate = deferred<string>();
let calls = 0;
const scope = { kind: "database-schemas", connectionId: "conn", database: "app", driverProfile: "postgres" };
const first = coordinator.run(scope, async () => {
calls++;
return gate.promise;
});
const second = coordinator.run(scope, async () => {
calls++;
return "unexpected";
});
assert.equal(calls, 0);
await Promise.resolve();
assert.equal(calls, 1);
assert.equal(first, second);
assert.deepEqual(events.slice(0, 2), ["start", "dedupe"]);
gate.resolve("schemas");
assert.equal(await first, "schemas");
assert.equal(await second, "schemas");
assert.equal(coordinator.has(scope), false);
});
test("metadata load coordinator keeps different filters independent", async () => {
const coordinator = new MetadataLoadCoordinator();
const first = deferred<string>();
const second = deferred<string>();
let calls = 0;
const user = coordinator.run({ kind: "sidebar-table-search", connectionId: "conn", database: "app", schema: "public", searchFilter: "user" }, async () => {
calls++;
return first.promise;
});
const order = coordinator.run({ kind: "sidebar-table-search", connectionId: "conn", database: "app", schema: "public", searchFilter: "order" }, async () => {
calls++;
return second.promise;
});
await Promise.resolve();
assert.equal(calls, 2);
first.resolve("users");
second.resolve("orders");
assert.deepEqual(await Promise.all([user, order]), ["users", "orders"]);
});
test("metadata load coordinator force bypasses in-flight reuse", async () => {
const coordinator = new MetadataLoadCoordinator();
const slow = deferred<string>();
let calls = 0;
const scope = { kind: "connection-databases", connectionId: "conn" };
const normal = coordinator.run(scope, async () => {
calls++;
return slow.promise;
});
const forced = coordinator.run(
scope,
async () => {
calls++;
return "fresh";
},
{ force: true },
);
assert.equal(await forced, "fresh");
assert.equal(calls, 2);
slow.resolve("cached");
assert.equal(await normal, "cached");
});
test("metadata load trace reports scope, elapsed time, cache and request state", () => {
let now = 100;
const trace = createMetadataLoadTrace(
{
kind: "table-metadata",
connectionId: "conn",
database: "app",
schema: "public",
tableName: "users",
},
{ traceId: "trace-1", now: () => now },
);
now = 137;
assert.deepEqual(trace.event("done", { cacheStatus: "stale", resultCount: 12, deduped: true, superseded: false }), {
event: "done",
traceId: "trace-1",
scopeKey: metadataScopeKey({
kind: "table-metadata",
connectionId: "conn",
database: "app",
schema: "public",
tableName: "users",
}),
elapsedMs: 37,
cacheStatus: "stale",
resultCount: 12,
deduped: true,
superseded: false,
});
});
@@ -0,0 +1,54 @@
import assert from "node:assert/strict";
import { test } from "vitest";
import { MetadataResultCache } from "../../apps/desktop/src/lib/metadata/metadataResultCache.ts";
test("metadata result cache returns fresh hits and exposes stale hits when requested", () => {
let now = 1_000;
const cache = new MetadataResultCache<string>({ ttlMs: 50, maxEntries: 10, now: () => now });
const scope = { kind: "table-list-page", connectionId: "conn", database: "app", schema: "public", limit: 501, offset: 0 };
cache.set(scope, "tables");
assert.equal(cache.get(scope)?.value, "tables");
now = 1_100;
assert.equal(cache.get(scope), undefined);
const stale = cache.get(scope, { allowStale: true });
assert.equal(stale?.value, "tables");
assert.equal(stale?.stale, true);
assert.equal(stale?.ageMs, 100);
});
test("metadata result cache invalidates affected scopes without clearing unrelated databases", () => {
const cache = new MetadataResultCache<string>({ ttlMs: 30_000, maxEntries: 10, now: () => 1 });
const publicTables = { kind: "table-list-page", connectionId: "conn", database: "app", schema: "public", objectTypes: ["VIEW", "TABLE"], limit: 501, offset: 0 };
const auditTables = { kind: "table-list-page", connectionId: "conn", database: "app", schema: "audit", objectTypes: ["TABLE"], limit: 501, offset: 0 };
const otherDatabase = { kind: "table-list-page", connectionId: "conn", database: "other", schema: "public", objectTypes: ["TABLE"], limit: 501, offset: 0 };
const otherConnection = { kind: "table-list-page", connectionId: "other", database: "app", schema: "public", objectTypes: ["TABLE"], limit: 501, offset: 0 };
cache.set(publicTables, "public");
cache.set(auditTables, "audit");
cache.set(otherDatabase, "other-db");
cache.set(otherConnection, "other-conn");
assert.equal(cache.invalidate({ connectionId: "conn", database: "app", schema: "public" }), 1);
assert.equal(cache.get(publicTables), undefined);
assert.equal(cache.get(auditTables)?.value, "audit");
assert.equal(cache.get(otherDatabase)?.value, "other-db");
assert.equal(cache.get(otherConnection)?.value, "other-conn");
});
test("metadata result cache evicts least recently used entries", () => {
const cache = new MetadataResultCache<string>({ ttlMs: 30_000, maxEntries: 2, now: () => 1 });
const first = { kind: "object-list-page", connectionId: "conn", database: "app", schema: "public", nodeKind: "group-functions" };
const second = { kind: "object-list-page", connectionId: "conn", database: "app", schema: "audit", nodeKind: "group-functions" };
const third = { kind: "object-list-page", connectionId: "conn", database: "other", schema: "public", nodeKind: "group-functions" };
cache.set(first, "first");
cache.set(second, "second");
assert.equal(cache.get(first)?.value, "first");
cache.set(third, "third");
assert.equal(cache.get(first)?.value, "first");
assert.equal(cache.get(second), undefined);
assert.equal(cache.get(third)?.value, "third");
});
@@ -71,7 +71,7 @@ test("inserts a grouped local table search control only for expanded table group
connectionId: "conn", connectionId: "conn",
database: "app", database: "app",
isExpanded: true, isExpanded: true,
children: [], children: [{ id: "conn:app:orders", label: "orders", type: "table", connectionId: "conn", database: "app" }],
}; };
const viewGroup: TreeNode = { const viewGroup: TreeNode = {
id: "conn:app:__views", id: "conn:app:__views",
@@ -96,6 +96,65 @@ test("inserts a grouped local table search control only for expanded table group
assert.equal(nodes[1].node.tableSearchParentId, "conn:app:__tables"); assert.equal(nodes[1].node.tableSearchParentId, "conn:app:__tables");
}); });
test("does not insert a local table search control for empty table scopes", () => {
const tableGroup: TreeNode = {
id: "conn:app:__tables",
label: "tree.tables",
type: "group-tables",
connectionId: "conn",
database: "app",
isExpanded: true,
children: [],
};
const routineOnlySchema: TreeNode = {
id: "conn:app:public",
label: "public",
type: "schema",
connectionId: "conn",
database: "app",
schema: "public",
isExpanded: true,
children: [{ id: "conn:app:public:sync_orders", label: "sync_orders", type: "procedure", connectionId: "conn", database: "app", schema: "public" }],
};
const groupedNodes = insertSidebarTableSearchControls([flat(tableGroup, 1)], {
enabled: true,
sidebarObjectDisplay: "grouped",
activeQueries: {},
});
const simpleNodes = insertSidebarTableSearchControls([flat(routineOnlySchema, 1)], {
enabled: true,
sidebarObjectDisplay: "simple",
activeQueries: {},
});
assert.deepEqual(groupedNodes.map((item) => item.node.type), ["group-tables"]);
assert.deepEqual(simpleNodes.map((item) => item.node.type), ["schema"]);
});
test("keeps a grouped local table search control visible when the current search has no results", () => {
const tableGroup: TreeNode = {
id: "conn:app:__tables",
label: "tree.tables",
type: "group-tables",
connectionId: "conn",
database: "app",
isExpanded: true,
children: [],
};
const nodes = insertSidebarTableSearchControls([flat(tableGroup, 1)], {
enabled: true,
sidebarObjectDisplay: "grouped",
activeQueries: { "conn:app:__tables": "invoice" },
});
assert.deepEqual(
nodes.map((item) => item.node.type),
["group-tables", "table-search-control"],
);
});
test("uses isolated virtual scroller pools for each local table search input", () => { test("uses isolated virtual scroller pools for each local table search input", () => {
const first: TreeNode = { const first: TreeNode = {
id: "conn:first:__tables", id: "conn:first:__tables",
@@ -104,7 +163,7 @@ test("uses isolated virtual scroller pools for each local table search input", (
connectionId: "conn", connectionId: "conn",
database: "first", database: "first",
isExpanded: true, isExpanded: true,
children: [], children: [{ id: "conn:first:orders", label: "orders", type: "table", connectionId: "conn", database: "first" }],
}; };
const second: TreeNode = { const second: TreeNode = {
id: "conn:second:__tables", id: "conn:second:__tables",
@@ -113,7 +172,7 @@ test("uses isolated virtual scroller pools for each local table search input", (
connectionId: "conn", connectionId: "conn",
database: "second", database: "second",
isExpanded: true, isExpanded: true,
children: [], children: [{ id: "conn:second:orders", label: "orders", type: "table", connectionId: "conn", database: "second" }],
}; };
const nodes = insertSidebarTableSearchControls([flat(first, 1), flat(second, 1)], { const nodes = insertSidebarTableSearchControls([flat(first, 1), flat(second, 1)], {
@@ -0,0 +1,101 @@
import assert from "node:assert/strict";
import { beforeEach, test, vi } from "vitest";
import type { ColumnInfo, IndexInfo } from "../../apps/desktop/src/types/database.ts";
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
beforeEach(() => {
vi.resetModules();
});
test("table metadata loader deduplicates equivalent in-flight requests and caches the result", async () => {
const columnsGate = deferred<ColumnInfo[]>();
const indexes: IndexInfo[] = [{ name: "users_pkey", columns: ["id"], is_unique: true, is_primary: true }];
const getColumns = vi.fn(() => columnsGate.promise);
const listIndexes = vi.fn().mockResolvedValue(indexes);
vi.doMock("@/lib/backend/api", () => ({ getColumns, listIndexes }));
const { clearTableMetadataCache, loadTableMetadata } = await import("../../apps/desktop/src/lib/metadata/tableMetadataCache.ts");
clearTableMetadataCache();
const request = {
connectionId: "conn",
database: "app",
schema: "public",
tableName: "users",
tableType: "TABLE",
databaseType: "postgres",
driverProfile: "postgres",
};
const first = loadTableMetadata(request);
const second = loadTableMetadata(request);
await Promise.resolve();
assert.equal(getColumns.mock.calls.length, 1);
columnsGate.resolve([
{
name: "id",
data_type: "integer",
is_nullable: false,
column_default: null,
is_primary_key: true,
extra: null,
},
]);
const [firstResult, secondResult] = await Promise.all([first, second]);
assert.deepEqual(firstResult.metadata.primaryKeys, ["id"]);
assert.equal(secondResult.metadata.columns.length, 1);
assert.equal(listIndexes.mock.calls.length, 1);
const cached = await loadTableMetadata(request);
assert.equal(cached.cacheStatus, "hit");
assert.equal(getColumns.mock.calls.length, 1);
});
test("table metadata invalidation keeps unrelated schemas cached", async () => {
const getColumns = vi.fn(
async (_connectionId: string, _database: string, _schema: string, table: string): Promise<ColumnInfo[]> => [
{
name: `${table}_id`,
data_type: "integer",
is_nullable: false,
column_default: null,
is_primary_key: true,
extra: null,
},
],
);
const listIndexes = vi.fn(async (): Promise<IndexInfo[]> => []);
vi.doMock("@/lib/backend/api", () => ({ getColumns, listIndexes }));
const { clearTableMetadataCache, getCachedTableMetadata, invalidateTableMetadataCache, loadTableMetadata } = await import("../../apps/desktop/src/lib/metadata/tableMetadataCache.ts");
clearTableMetadataCache();
const publicRequest = {
connectionId: "conn",
database: "app",
schema: "public",
tableName: "users",
tableType: "TABLE",
databaseType: "postgres",
driverProfile: "postgres",
};
const auditRequest = { ...publicRequest, schema: "audit", tableName: "audit_log" };
await loadTableMetadata(publicRequest);
await loadTableMetadata(auditRequest);
assert.equal(getColumns.mock.calls.length, 2);
assert.equal(invalidateTableMetadataCache({ connectionId: "conn", database: "app", schema: "public" }), 1);
assert.equal(getCachedTableMetadata(publicRequest), undefined);
assert.equal(getCachedTableMetadata(auditRequest)?.metadata.tableName, "audit_log");
});
@@ -263,12 +263,18 @@ public final class DbxJdbcPlugin {
optionalText(params, "database"), optionalText(params, "database"),
optionalText(params, "schema"), optionalText(params, "schema"),
optionalText(params, "filter"), optionalText(params, "filter"),
nonNegativeInt(params, "limit", 0) nonNegativeInt(params, "limit", 0),
nonNegativeInt(params, "offset", 0),
optionalStringList(params, "object_types")
); );
case "listObjects", "list_objects" -> listObjects( case "listObjects", "list_objects" -> listObjects(
connection, connection,
optionalText(params, "database"), optionalText(params, "database"),
optionalText(params, "schema") optionalText(params, "schema"),
optionalText(params, "filter"),
nonNegativeInt(params, "limit", 0),
nonNegativeInt(params, "offset", 0),
optionalStringList(params, "object_types")
); );
case "listDataTypes", "list_data_types" -> listDataTypes(connection, optionalText(params, "database")); case "listDataTypes", "list_data_types" -> listDataTypes(connection, optionalText(params, "database"));
case "getObjectSource", "get_object_source" -> getObjectSource( case "getObjectSource", "get_object_source" -> getObjectSource(
@@ -1121,45 +1127,82 @@ public final class DbxJdbcPlugin {
return result; return result;
} }
private static JsonNode listTables(JsonNode connection, String database, String schema, String filter, int limit) throws SQLException { private static JsonNode listTables(
JsonNode connection,
String database,
String schema,
String filter,
int limit,
int offset,
List<String> objectTypes
) throws SQLException {
ArrayNode result = MAPPER.createArrayNode(); ArrayNode result = MAPPER.createArrayNode();
Connection conn = openConnection(connection); Connection conn = openConnection(connection);
JdbcDriverQuirks quirks = driverQuirks(connection); JdbcDriverQuirks quirks = driverQuirks(connection);
if (quirks.useOracleMetadata()) { if (quirks.useOracleMetadata()) {
return oracleListTables(conn, oracleEffectiveSchema(conn, schema)); return filterMetadataNodes(
(ArrayNode) oracleListTables(conn, oracleEffectiveSchema(conn, schema)),
filter,
limit,
offset,
objectTypes,
"table_type",
true
);
} }
if (usePrestoInformationSchemaTables(connection)) { if (usePrestoInformationSchemaTables(connection)) {
return prestoListTables(conn, database, schema, filter, limit); return prestoListTables(conn, database, schema, filter, limit, offset, objectTypes);
} }
DatabaseMetaData meta = conn.getMetaData(); DatabaseMetaData meta = conn.getMetaData();
String[] types = jdbcTableTypes(meta); String[] types = constrainedJdbcTableTypes(jdbcTableTypes(meta), objectTypes);
if (types.length == 0) {
return result;
}
String catalog = metadataCatalog(database, quirks); String catalog = metadataCatalog(database, quirks);
String schemaPattern = resolveSchemaPattern(meta, database, schema, quirks); String schemaPattern = resolveSchemaPattern(meta, database, schema, quirks);
appendTables(result, meta, catalog, schemaPattern, types); appendTables(result, meta, catalog, schemaPattern, types);
if (result.isEmpty() && catalog != null) { if (result.isEmpty() && catalog != null) {
appendTables(result, meta, null, schemaPattern, types); appendTables(result, meta, null, schemaPattern, types);
} }
return result; return filterMetadataNodes(result, filter, limit, offset, objectTypes, "table_type", true);
} }
private static JsonNode listObjects(JsonNode connection, String database, String schema) throws SQLException { private static JsonNode listObjects(
JsonNode connection,
String database,
String schema,
String filter,
int limit,
int offset,
List<String> objectTypes
) throws SQLException {
ArrayNode result = MAPPER.createArrayNode(); ArrayNode result = MAPPER.createArrayNode();
Connection conn = openConnection(connection); Connection conn = openConnection(connection);
if (driverQuirks(connection).useOracleMetadata()) { if (driverQuirks(connection).useOracleMetadata()) {
return oracleListObjects(conn, oracleEffectiveSchema(conn, schema), schema); return filterMetadataNodes(
(ArrayNode) oracleListObjects(conn, oracleEffectiveSchema(conn, schema), schema),
filter,
limit,
offset,
objectTypes,
"object_type",
false
);
} }
if (usePrestoInformationSchemaTables(connection)) { if (usePrestoInformationSchemaTables(connection)) {
return prestoListObjects(conn, database, schema); return prestoListObjects(conn, database, schema, filter, limit, offset, objectTypes);
} }
DatabaseMetaData meta = conn.getMetaData(); DatabaseMetaData meta = conn.getMetaData();
JdbcDriverQuirks quirks = driverQuirks(connection); JdbcDriverQuirks quirks = driverQuirks(connection);
String catalog = metadataCatalog(database, quirks); String catalog = metadataCatalog(database, quirks);
String schemaPattern = resolveSchemaPattern(meta, database, schema, quirks); String schemaPattern = resolveSchemaPattern(meta, database, schema, quirks);
String[] tableTypes = jdbcTableTypes(meta); String[] tableTypes = constrainedJdbcTableTypes(jdbcTableTypes(meta), objectTypes);
appendTableObjects(result, meta, catalog, schemaPattern, schema, tableTypes); if (tableTypes.length > 0) {
if (result.isEmpty() && catalog != null) { appendTableObjects(result, meta, catalog, schemaPattern, schema, tableTypes);
appendTableObjects(result, meta, null, schemaPattern, schema, tableTypes); if (result.isEmpty() && catalog != null) {
appendTableObjects(result, meta, null, schemaPattern, schema, tableTypes);
}
} }
try (ResultSet rs = meta.getProcedures(catalog, schemaPattern, "%")) { try (ResultSet rs = meta.getProcedures(catalog, schemaPattern, "%")) {
@@ -1195,7 +1238,7 @@ public final class DbxJdbcPlugin {
} catch (SQLException ignored) { } catch (SQLException ignored) {
} }
return result; return filterMetadataNodes(result, filter, limit, offset, objectTypes, "object_type", false);
} }
private static JsonNode listDataTypes(JsonNode connection, String database) throws SQLException { private static JsonNode listDataTypes(JsonNode connection, String database) throws SQLException {
@@ -1378,6 +1421,111 @@ public final class DbxJdbcPlugin {
return DEFAULT_TABLE_TYPES; return DEFAULT_TABLE_TYPES;
} }
private static String[] constrainedJdbcTableTypes(String[] tableTypes, List<String> objectTypes) {
Set<String> allowed = normalizedObjectTypes(objectTypes);
if (allowed.isEmpty()) {
return tableTypes;
}
List<String> result = new ArrayList<>();
for (String tableType : tableTypes) {
if (allowed.contains(normalizeTableObjectType(tableType))) {
result.add(tableType);
}
}
return result.toArray(new String[0]);
}
private static ArrayNode filterMetadataNodes(
ArrayNode source,
String filter,
int limit,
int offset,
List<String> objectTypes,
String typeField,
boolean defaultBlankTypeToTable
) {
ArrayNode result = MAPPER.createArrayNode();
Set<String> allowedTypes = normalizedObjectTypes(objectTypes);
String normalizedFilter = filter == null ? "" : filter.trim().toLowerCase(Locale.ROOT);
int start = Math.max(0, offset);
int max = limit <= 0 ? Integer.MAX_VALUE : limit;
int skipped = 0;
for (JsonNode item : source) {
if (!metadataNameMatches(item.path("name").asText(""), normalizedFilter)) {
continue;
}
String type = item.path(typeField).asText("");
String normalizedType = defaultBlankTypeToTable ? normalizeTableObjectType(type) : normalizeObjectType(type);
if (!allowedTypes.isEmpty() && (normalizedType.isEmpty() || !allowedTypes.contains(normalizedType))) {
continue;
}
if (skipped++ < start) {
continue;
}
if (result.size() >= max) {
break;
}
result.add(item);
}
return result;
}
private static boolean metadataNameMatches(String name, String filter) {
if (filter == null || filter.isEmpty()) {
return true;
}
String candidate = name == null ? "" : name.toLowerCase(Locale.ROOT);
return candidate.contains(filter) || (filter.length() >= 2 && fuzzySubsequenceMatches(candidate, filter));
}
private static boolean fuzzySubsequenceMatches(String candidate, String expected) {
int cursor = 0;
for (int i = 0; i < expected.length(); i++) {
cursor = candidate.indexOf(expected.charAt(i), cursor);
if (cursor < 0) {
return false;
}
cursor++;
}
return true;
}
private static Set<String> normalizedObjectTypes(List<String> objectTypes) {
Set<String> result = new HashSet<>();
if (objectTypes == null) {
return result;
}
for (String objectType : objectTypes) {
String normalized = normalizeObjectType(objectType);
if (!normalized.isEmpty()) {
result.add(normalized);
}
}
return result;
}
private static String normalizeTableObjectType(String value) {
String normalized = normalizeObjectType(value);
return normalized.isEmpty() ? "TABLE" : normalized;
}
private static String normalizeObjectType(String value) {
if (value == null || value.isBlank()) {
return "";
}
String upper = value.trim().toUpperCase(Locale.ROOT).replace(' ', '_');
if (upper.contains("MATERIALIZED") && upper.contains("VIEW")) {
return "MATERIALIZED_VIEW";
}
if ("BASE_TABLE".equals(upper) || upper.contains("TABLE")) {
return "TABLE";
}
if (upper.contains("VIEW")) {
return "VIEW";
}
return upper;
}
private static void appendTableObjects( private static void appendTableObjects(
ArrayNode result, ArrayNode result,
DatabaseMetaData meta, DatabaseMetaData meta,
@@ -1403,9 +1551,18 @@ public final class DbxJdbcPlugin {
return urlMatchesPrefix(url, "jdbc:presto:") || urlMatchesPrefix(url, "jdbc:trino:"); return urlMatchesPrefix(url, "jdbc:presto:") || urlMatchesPrefix(url, "jdbc:trino:");
} }
private static JsonNode prestoListTables(Connection conn, String database, String schema, String filter, int limit) throws SQLException { private static JsonNode prestoListTables(
Connection conn,
String database,
String schema,
String filter,
int limit,
int offset,
List<String> objectTypes
) throws SQLException {
ArrayNode result = MAPPER.createArrayNode(); ArrayNode result = MAPPER.createArrayNode();
try (PreparedStatement ps = conn.prepareStatement(prestoInformationSchemaTablesSql(database, filter, limit))) { int queryLimit = limit > 0 ? Math.max(1, limit + Math.max(0, offset)) : 0;
try (PreparedStatement ps = conn.prepareStatement(prestoInformationSchemaTablesSql(database, filter, queryLimit))) {
ps.setString(1, schema); ps.setString(1, schema);
if (emptyToNull(filter) != null) { if (emptyToNull(filter) != null) {
ps.setString(2, escapeLikePattern(filter.toLowerCase(Locale.ROOT)) + "%"); ps.setString(2, escapeLikePattern(filter.toLowerCase(Locale.ROOT)) + "%");
@@ -1420,13 +1577,25 @@ public final class DbxJdbcPlugin {
} }
} }
} }
return result; return filterMetadataNodes(result, filter, limit, offset, objectTypes, "table_type", true);
} }
private static JsonNode prestoListObjects(Connection conn, String database, String schema) throws SQLException { private static JsonNode prestoListObjects(
Connection conn,
String database,
String schema,
String filter,
int limit,
int offset,
List<String> objectTypes
) throws SQLException {
ArrayNode result = MAPPER.createArrayNode(); ArrayNode result = MAPPER.createArrayNode();
try (PreparedStatement ps = conn.prepareStatement(prestoInformationSchemaTablesSql(database, null, 0))) { int queryLimit = limit > 0 ? Math.max(1, limit + Math.max(0, offset)) : 0;
try (PreparedStatement ps = conn.prepareStatement(prestoInformationSchemaTablesSql(database, filter, queryLimit))) {
ps.setString(1, schema); ps.setString(1, schema);
if (emptyToNull(filter) != null) {
ps.setString(2, escapeLikePattern(filter.toLowerCase(Locale.ROOT)) + "%");
}
try (ResultSet rs = ps.executeQuery()) { try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) { while (rs.next()) {
ObjectNode item = MAPPER.createObjectNode(); ObjectNode item = MAPPER.createObjectNode();
@@ -1438,7 +1607,7 @@ public final class DbxJdbcPlugin {
} }
} }
} }
return result; return filterMetadataNodes(result, filter, limit, offset, objectTypes, "object_type", false);
} }
private static JsonNode prestoGetColumns(Connection conn, String database, String schema, String table) throws SQLException { private static JsonNode prestoGetColumns(Connection conn, String database, String schema, String table) throws SQLException {
@@ -2310,6 +2479,34 @@ public final class DbxJdbcPlugin {
return text.isEmpty() ? null : text; return text.isEmpty() ? null : text;
} }
private static List<String> optionalStringList(JsonNode node, String field) {
JsonNode value = node.path(field);
if (value.isMissingNode() || value.isNull()) {
return null;
}
List<String> result = new ArrayList<>();
if (value.isArray()) {
for (JsonNode item : value) {
String text = item.asText("").trim();
if (!text.isEmpty()) {
result.add(text);
}
}
return result;
}
String text = value.asText("").trim();
if (text.isEmpty()) {
return null;
}
for (String part : text.split(",")) {
String item = part.trim();
if (!item.isEmpty()) {
result.add(item);
}
}
return result;
}
private static int positiveInt(JsonNode node, String field, int defaultValue) { private static int positiveInt(JsonNode node, String field, int defaultValue) {
return Math.max(1, nonNegativeInt(node, field, defaultValue)); return Math.max(1, nonNegativeInt(node, field, defaultValue));
} }
@@ -731,6 +731,43 @@ final class DbxJdbcPluginTest {
assertEquals("PEOPLE", response.path("result").path(0).path("name").asText()); assertEquals("PEOPLE", response.path("result").path(0).path("name").asText());
} }
@Test
void listTablesAppliesMetadataConstraints() throws Exception {
request("executeQuery", """
{
"connection": %s,
"sql": "CREATE SCHEMA IF NOT EXISTS app"
}
""".formatted(CONNECTION));
request("executeQuery", """
{
"connection": %s,
"sql": "CREATE TABLE IF NOT EXISTS app.people (id INT PRIMARY KEY)"
}
""".formatted(CONNECTION));
request("executeQuery", """
{
"connection": %s,
"sql": "CREATE TABLE IF NOT EXISTS app.people_archive (id INT PRIMARY KEY)"
}
""".formatted(CONNECTION));
JsonNode response = request("listTables", """
{
"connection": %s,
"schema": "APP",
"filter": "people",
"limit": 1,
"offset": 1,
"object_types": ["TABLE"]
}
""".formatted(CONNECTION));
assertFalse(response.has("error"), response.toString());
assertEquals(1, response.path("result").size());
assertEquals("PEOPLE_ARCHIVE", response.path("result").path(0).path("name").asText());
}
@Test @Test
void listDatabasesIncludesConfiguredDatabaseWhenDriverDoesNotReturnIt() throws Exception { void listDatabasesIncludesConfiguredDatabaseWhenDriverDoesNotReturnIt() throws Exception {
String connection = """ String connection = """
@@ -794,6 +831,43 @@ final class DbxJdbcPluginTest {
assertEquals("PEOPLE", response.path("result").path(0).path("name").asText()); assertEquals("PEOPLE", response.path("result").path(0).path("name").asText());
} }
@Test
void listObjectsAppliesMetadataConstraints() throws Exception {
request("executeQuery", """
{
"connection": %s,
"sql": "CREATE SCHEMA IF NOT EXISTS app"
}
""".formatted(CONNECTION));
request("executeQuery", """
{
"connection": %s,
"sql": "CREATE TABLE IF NOT EXISTS app.people (id INT PRIMARY KEY)"
}
""".formatted(CONNECTION));
request("executeQuery", """
{
"connection": %s,
"sql": "CREATE TABLE IF NOT EXISTS app.people_archive (id INT PRIMARY KEY)"
}
""".formatted(CONNECTION));
JsonNode response = request("listObjects", """
{
"connection": %s,
"schema": "APP",
"filter": "people",
"limit": 1,
"offset": 1,
"object_types": ["TABLE"]
}
""".formatted(CONNECTION));
assertFalse(response.has("error"), response.toString());
assertEquals(1, response.path("result").size());
assertEquals("PEOPLE_ARCHIVE", response.path("result").path(0).path("name").asText());
}
@Test @Test
void getColumnsFallsBackWhenCatalogFiltersEverything() throws Exception { void getColumnsFallsBackWhenCatalogFiltersEverything() throws Exception {
createPeopleTable(); createPeopleTable();
+15 -1
View File
@@ -138,8 +138,22 @@ pub async fn list_objects(
connection_id: String, connection_id: String,
database: String, database: String,
schema: String, schema: String,
filter: Option<String>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<Vec<String>>,
) -> Result<Vec<db::ObjectInfo>, String> { ) -> Result<Vec<db::ObjectInfo>, String> {
dbx_core::schema::list_objects_core(&state, &connection_id, &database, &schema).await dbx_core::schema::list_objects_core(
&state,
&connection_id,
&database,
&schema,
filter.as_deref(),
limit,
offset,
object_types.as_deref(),
)
.await
} }
#[tauri::command] #[tauri::command]