From a7a0b696054c02757d780bbb840917767a6ba738 Mon Sep 17 00:00:00 2001 From: skyler <1156263951@qq.com> Date: Thu, 30 Jul 2026 21:10:01 +0800 Subject: [PATCH] feat(agent): add shared JDBC pooling and metadata sessions --- agents/README.md | 20 + agents/README.zh-CN.md | 20 + agents/build.gradle | 14 + agents/common/build.gradle | 4 + .../java/com/dbx/agent/AbstractJdbcAgent.java | 260 ++++++- .../com/dbx/agent/JdbcConnectionAffinity.java | 177 +++++ .../dbx/agent/JdbcConnectionPoolRegistry.java | 516 ++++++++++++++ .../main/java/com/dbx/agent/JdbcExecutor.java | 29 +- .../com/dbx/agent/JdbcSchemaSwitcher.java | 147 +++- .../java/com/dbx/agent/JsonRpcServer.java | 98 ++- .../dbx/agent/MultiSessionJsonRpcServer.java | 151 +++- .../com/dbx/agent/AbstractJdbcAgentTest.java | 25 + .../agent/CommonJavaCompatibilityTest.java | 33 + .../dbx/agent/JdbcConnectionPoolingTest.java | 647 ++++++++++++++++++ agents/docs/agent-protocol-v2.md | 8 +- .../examples/jdbc-agent-template/build.gradle | 2 + .../com/dbx/agent/template/TemplateAgent.java | 4 +- .../com/dbx/agent/access/AccessAgent.java | 51 +- .../com/dbx/agent/dameng/DamengAgent.java | 69 +- .../main/java/com/dbx/agent/db2/Db2Agent.java | 42 +- .../com/dbx/agent/gbase8s/Gbase8sAgent.java | 3 +- .../com/dbx/agent/goldendb/GoldendbAgent.java | 40 +- .../java/com/dbx/agent/hive/HiveAgent.java | 4 +- .../com/dbx/agent/informix/InformixAgent.java | 64 +- .../dbx/agent/informix/InformixAgentTest.java | 10 +- .../java/com/dbx/agent/neo4j/Neo4jAgent.java | 43 +- .../com/dbx/agent/neo4j/Neo4jAgentTest.java | 24 +- .../oceanbaseoracle/OceanBaseOracleAgent.java | 13 + .../java/com/dbx/agent/spark/SparkAgent.java | 9 +- .../java/com/dbx/agent/sundb/SundbAgent.java | 46 +- .../com/dbx/agent/tdengine/TDengineAgent.java | 109 ++- .../dbx/agent/tdengine/TDengineAgentTest.java | 68 +- agents/scripts/validate_agents.py | 33 +- agents/scripts/validate_agents_test.py | 26 + crates/dbx-core/src/connection.rs | 119 +++- crates/dbx-core/src/db/agent_driver.rs | 51 +- crates/dbx-core/src/schema.rs | 286 +++++++- 37 files changed, 2867 insertions(+), 398 deletions(-) create mode 100644 agents/common/src/main/java/com/dbx/agent/JdbcConnectionAffinity.java create mode 100644 agents/common/src/main/java/com/dbx/agent/JdbcConnectionPoolRegistry.java create mode 100644 agents/common/src/test/java/com/dbx/agent/JdbcConnectionPoolingTest.java diff --git a/agents/README.md b/agents/README.md index f3c79c088d..3ccf969f17 100644 --- a/agents/README.md +++ b/agents/README.md @@ -51,6 +51,26 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo Most Java agents target JRE 21. Native agents, such as `duckdb`, `oracle`, `kingbase`, and `xugu`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents. +## JDBC Connection Pooling + +All multi-session Java JDBC agents share HikariCP pools inside one Agent runtime through `AbstractJdbcAgent`. Ordinary metadata and short query requests borrow and return a connection, while paged cursors and explicit session-state SQL keep their connection pinned until the cursor or logical session closes. Stateful connections are evicted instead of being reused by another session. Agent-specific URL, transport fallback, encrypted-file, and native-driver behavior is preserved through shared lifecycle hooks. + +The default maximum is 8 physical connections per immutable connection identity, with 0 minimum idle connections. This keeps short-query connection pressure bounded while allowing up to 8 concurrently pinned paged cursors or stateful sessions. The defaults can be overridden with JVM system properties or environment variables: + +| System property | Environment variable | Default | +|---|---|---:| +| `dbx.agent.jdbc.pool.enabled` | `DBX_AGENT_JDBC_POOL_ENABLED` | `true` | +| `dbx.agent.jdbc.pool.maximumPoolSize` | `DBX_AGENT_JDBC_POOL_MAXIMUM_POOL_SIZE` | `8` | +| `dbx.agent.jdbc.pool.minimumIdle` | `DBX_AGENT_JDBC_POOL_MINIMUM_IDLE` | `0` | +| `dbx.agent.jdbc.pool.connectionTimeoutMillis` | `DBX_AGENT_JDBC_POOL_CONNECTION_TIMEOUT_MILLIS` | `30000` | +| `dbx.agent.jdbc.pool.validationTimeoutMillis` | `DBX_AGENT_JDBC_POOL_VALIDATION_TIMEOUT_MILLIS` | `5000` | +| `dbx.agent.jdbc.pool.idleTimeoutMillis` | `DBX_AGENT_JDBC_POOL_IDLE_TIMEOUT_MILLIS` | `120000` | +| `dbx.agent.jdbc.pool.maxLifetimeMillis` | `DBX_AGENT_JDBC_POOL_MAX_LIFETIME_MILLIS` | `1800000` | +| `dbx.agent.jdbc.pool.retireMillis` | `DBX_AGENT_JDBC_POOL_RETIRE_MILLIS` | `300000` | + +HikariCP is shaded into each pooled Agent JAR. Existing installations already using the managed JRE 21 do not need to reinstall or replace the JRE. +Set `DBX_AGENT_JDBC_POOL_ENABLED=false` for a runtime-level compatibility fallback to the previous one-connection-per-logical-session behavior. + ## Choosing a Driver Language For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely. diff --git a/agents/README.zh-CN.md b/agents/README.zh-CN.md index 2309dd3fce..5c47c95a5f 100644 --- a/agents/README.zh-CN.md +++ b/agents/README.zh-CN.md @@ -51,6 +51,26 @@ DBX 的 Agent 驱动 —— 通过 JDBC 和原生数据库驱动支持各种数 多数 Java agent 以 JRE 21 为目标。原生 agent(如 `oracle`、`kingbase` 和 `xugu`)不需要 JRE。对 Java agent,DBX 会自动下载并管理 JRE 21 安装。 +## JDBC 连接池 + +所有多会话 Java JDBC agent 都通过 `AbstractJdbcAgent` 在同一个 Agent 运行时内共享 HikariCP 连接池。普通元数据请求和短查询按请求借还连接;分页游标和显式会话态 SQL 会固定连接,直到游标或逻辑会话关闭。带有会话状态的连接会直接淘汰,不会复用于其他会话。各 Agent 特有的 URL、传输协议兜底、加密文件和原生驱动行为通过共享生命周期钩子保留。 + +默认每个不可变连接身份最多建立 8 个物理连接,最小空闲连接数为 0。该默认值在限制短查询连接压力的同时,允许最多 8 个分页游标或会话态逻辑会话并发固定连接。可通过 JVM system property 或环境变量覆盖: + +| System property | 环境变量 | 默认值 | +|---|---|---:| +| `dbx.agent.jdbc.pool.enabled` | `DBX_AGENT_JDBC_POOL_ENABLED` | `true` | +| `dbx.agent.jdbc.pool.maximumPoolSize` | `DBX_AGENT_JDBC_POOL_MAXIMUM_POOL_SIZE` | `8` | +| `dbx.agent.jdbc.pool.minimumIdle` | `DBX_AGENT_JDBC_POOL_MINIMUM_IDLE` | `0` | +| `dbx.agent.jdbc.pool.connectionTimeoutMillis` | `DBX_AGENT_JDBC_POOL_CONNECTION_TIMEOUT_MILLIS` | `30000` | +| `dbx.agent.jdbc.pool.validationTimeoutMillis` | `DBX_AGENT_JDBC_POOL_VALIDATION_TIMEOUT_MILLIS` | `5000` | +| `dbx.agent.jdbc.pool.idleTimeoutMillis` | `DBX_AGENT_JDBC_POOL_IDLE_TIMEOUT_MILLIS` | `120000` | +| `dbx.agent.jdbc.pool.maxLifetimeMillis` | `DBX_AGENT_JDBC_POOL_MAX_LIFETIME_MILLIS` | `1800000` | +| `dbx.agent.jdbc.pool.retireMillis` | `DBX_AGENT_JDBC_POOL_RETIRE_MILLIS` | `300000` | + +HikariCP 会直接打进启用连接池的 Agent JAR。已经使用 DBX 托管 JRE 21 的安装无需重新安装或替换 JRE。 +如遇特定老驱动兼容问题,可设置 `DBX_AGENT_JDBC_POOL_ENABLED=false`,让该运行时回退到原来的“每个逻辑会话一个连接”行为。 + ## 选择驱动实现语言 对于新 agent,只要存在成熟、许可证兼容的原生驱动,优先选择**原生(Go 或 Rust)驱动**而非 Java/JDBC agent。原生 agent 以单一自包含可执行文件发布,无需 JRE,可显著降低内存占用和启动时间 —— 完全避开 Java agent 即便空闲也要付出的 JVM 基线开销。 diff --git a/agents/build.gradle b/agents/build.gradle index 3699aff094..8621eb5599 100644 --- a/agents/build.gradle +++ b/agents/build.gradle @@ -4,6 +4,13 @@ plugins { def infrastructureProjects = ['common', 'test-support'] as Set def legacyStandaloneProjects = ['mongodb', 'kafka', 'rocketmq', 'rabbitmq'] as Set +def pooledJdbcProjects = [ + 'access', 'bigquery', 'cassandra', 'dameng', 'databend', 'databricks', 'db2', 'exasol', + 'firebird', 'gbase8a', 'gbase8s', 'goldendb', 'h2', 'h2-legacy', 'highgo', 'hive', + 'informix', 'iotdb', 'iris', 'kylin', 'neo4j', 'oceanbase-oracle', 'oscar', 'saphana', + 'snowflake', 'spark', 'sqlserver-legacy', 'sundb', 'tdengine', 'teradata', 'trino', 'uxdb', + 'vastbase', 'vertica', 'yashandb' +] as Set def agentProjects = subprojects.findAll { !infrastructureProjects.contains(it.name) } def jdbcAgentProjects = agentProjects.findAll { !legacyStandaloneProjects.contains(it.name) } @@ -51,3 +58,10 @@ configure(jdbcAgentProjects) { testImplementation project(':test-support') } } + +configure(agentProjects.findAll { pooledJdbcProjects.contains(it.name) }) { + dependencies { + implementation 'com.zaxxer:HikariCP:7.0.2' + runtimeOnly 'org.slf4j:slf4j-nop:2.0.17' + } +} diff --git a/agents/common/build.gradle b/agents/common/build.gradle index ee0d25b0ff..9249c20f5c 100644 --- a/agents/common/build.gradle +++ b/agents/common/build.gradle @@ -1,3 +1,7 @@ dependencies { implementation 'com.google.code.gson:gson:2.12.1' + compileOnly 'com.zaxxer:HikariCP:7.0.2' + testImplementation 'com.zaxxer:HikariCP:7.0.2' + testImplementation 'com.h2database:h2:2.3.232' + testRuntimeOnly 'org.slf4j:slf4j-nop:2.0.17' } diff --git a/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java b/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java index d61cd4628b..03acc1b941 100644 --- a/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java @@ -19,9 +19,16 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { private static final String GAUSSDB_COMPATIBILITY_SQL = "SELECT datcompatibility FROM pg_catalog.pg_database WHERE datname = current_database()"; - private Connection connection; + private volatile Connection connection; private String configuredDatabase = ""; private String identifierQuote = ""; + private JdbcConnectionPoolRegistry poolRegistry; + private JdbcConnectionPoolRegistry.Lease pooledLease; + private ConnectParams connectParams; + private String poolIdentity; + private boolean requestActive; + private boolean leasePinnedAtRequestStart; + private boolean sessionAffinity; @Override public final Connection getConnection() { @@ -36,11 +43,37 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { @Override public final void connect(ConnectParams params) { uncheckedVoid(() -> { + closeCurrentConnection(); + sessionAffinity = false; + requestActive = false; + leasePinnedAtRequestStart = false; loadDriver(params); - connection = openConnection(params); configuredDatabase = params.getDatabase(); - afterConnect(params, connection); - identifierQuote = resolveIdentifierQuote(params, connection); + connectParams = params; + poolIdentity = buildPoolIdentity(params); + if (poolRegistry == null) { + connection = openInitializedConnection(params); + afterConnect(params, connection); + identifierQuote = resolveIdentifierQuote(params, connection); + return; + } + + JdbcConnectionPoolRegistry.Lease lease = borrowPooledConnection(); + connection = lease.connection(); + try { + afterConnect(params, connection); + identifierQuote = resolveIdentifierQuote(params, connection); + } catch (Exception error) { + lease.evict(); + throw error; + } finally { + if (!preparePooledConnectionForReturn()) { + lease.evict(); + } else { + lease.close(); + } + connection = null; + } }); } @@ -53,8 +86,8 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { public final Map testConnectionWithInfo(ConnectParams params) { return unchecked(() -> { loadDriver(params); - try (Connection conn = openConnection(params)) { - boolean valid = conn.isValid(5); + try (Connection conn = openTestConnection(params)) { + boolean valid = conn != null && conn.isValid(5); Map result = new LinkedHashMap<>(); result.put("ok", valid); if (valid) { @@ -73,7 +106,7 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { return JdbcDatabaseInfo.from(getConnection()); } - private void loadDriver(ConnectParams params) throws Exception { + protected void loadDriver(ConnectParams params) throws Exception { List driverPaths = params.getJdbc_driver_paths(); String driverClass = params.getJdbc_driver_class(); if (driverClass == null || driverClass.isEmpty()) { @@ -214,16 +247,90 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { } @Override - public void disconnect() { + public synchronized void disconnect() { uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; + closeCurrentConnection(); + afterDisconnect(); + connectParams = null; + poolIdentity = null; + sessionAffinity = false; + requestActive = false; + leasePinnedAtRequestStart = false; identifierQuote = ""; }); } + final synchronized void attachConnectionPoolRegistry(JdbcConnectionPoolRegistry registry) { + if (connectParams != null || connection != null || pooledLease != null) { + throw new IllegalStateException("JDBC pool registry must be attached before connecting"); + } + poolRegistry = registry; + } + + final synchronized boolean usesConnectionPool() { + return poolRegistry != null; + } + + final synchronized void beginPooledRequest() throws Exception { + if (poolRegistry == null) { + return; + } + if (connectParams == null) { + throw new IllegalStateException("Not connected"); + } + requestActive = true; + leasePinnedAtRequestStart = pooledLease != null; + if (pooledLease == null) { + try { + pooledLease = borrowPooledConnection(); + } catch (Exception error) { + requestActive = false; + throw error; + } + } + connection = pooledLease.connection(); + } + + final synchronized void finishPooledRequest( + JdbcExecutor executor, + boolean succeeded, + boolean requiresSessionAffinity, + boolean evictAfterRequest + ) { + if (poolRegistry == null) { + return; + } + requestActive = false; + if (succeeded && requiresSessionAffinity) { + sessionAffinity = true; + JdbcSchemaSwitcher.forget(connection); + } + if (pooledLease == null) { + connection = null; + return; + } + if (evictAfterRequest && leasePinnedAtRequestStart) { + sessionAffinity = true; + } else if (evictAfterRequest || (!succeeded && !leasePinnedAtRequestStart)) { + releasePooledConnection(true); + return; + } + if (sessionAffinity || executor.hasOpenSessions() || executor.hasActiveStatements()) { + return; + } + releasePooledConnection(!preparePooledConnectionForReturn()); + } + + final synchronized void releaseIdlePooledConnection(JdbcExecutor executor) { + if (poolRegistry == null || requestActive || sessionAffinity || pooledLease == null) { + return; + } + if (executor.hasOpenSessions() || executor.hasActiveStatements()) { + return; + } + releasePooledConnection(!preparePooledConnectionForReturn()); + } + private static String resolveIdentifierQuote(ConnectParams params, Connection connection) { if (isPostgresCompatibleJdbc(params)) { try (Statement statement = connection.createStatement()) { @@ -301,16 +408,46 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { return DriverManager.getConnection(buildJdbcUrl(params), params.getUsername(), params.getPassword()); } + protected Connection openTestConnection(ConnectParams params) throws Exception { + return openConnection(params); + } + + protected void afterPhysicalConnect(ConnectParams params, Connection connection) throws Exception { + } + protected void afterConnect(ConnectParams params, Connection connection) throws Exception { } + protected void afterDisconnect() throws Exception { + } + protected void beforeQueryExecution(Connection connection, int timeoutSecs) throws Exception { } + protected void beforePooledConnectionReturn(Connection connection) throws Exception { + } + + protected final void applySchemaContext(Connection connection, String schema) throws Exception { + JdbcSchemaSwitcher.apply(connection, schema, this::setSchemaSQL, this::resetSchemaSQL); + } + protected String getConfiguredDatabase() { return configuredDatabase; } + protected static T unwrapConnection(Connection connection, Class connectionType) { + if (connectionType.isInstance(connection)) { + return connectionType.cast(connection); + } + try { + if (connection.isWrapperFor(connectionType)) { + return connection.unwrap(connectionType); + } + } catch (Exception ignored) { + } + return null; + } + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value; @@ -349,4 +486,103 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { protected JdbcExecutor.ResultValueReader resultValueReader() { return this::resultValue; } + + private Connection openInitializedConnection(ConnectParams params) throws Exception { + Connection opened = openConnection(params); + boolean initialized = false; + try { + afterPhysicalConnect(params, opened); + initialized = true; + return opened; + } finally { + if (!initialized) { + try { + opened.close(); + } catch (Exception ignored) { + } + } + } + } + + private JdbcConnectionPoolRegistry.Lease borrowPooledConnection() throws Exception { + ConnectParams params = connectParams; + if (params == null || poolIdentity == null || poolRegistry == null) { + throw new IllegalStateException("Not connected"); + } + return poolRegistry.borrow(poolIdentity, () -> openInitializedConnection(params)); + } + + private void closeCurrentConnection() throws Exception { + if (pooledLease != null) { + boolean evict = sessionAffinity || !preparePooledConnectionForReturn(); + releasePooledConnection(evict); + } else if (connection != null) { + connection.close(); + connection = null; + } + } + + private boolean preparePooledConnectionForReturn() { + try { + beforePooledConnectionReturn(connection); + } catch (Exception ignored) { + return false; + } + return JdbcSchemaSwitcher.resetBeforeReturn(connection); + } + + private void releasePooledConnection(boolean evict) { + JdbcConnectionPoolRegistry.Lease lease = pooledLease; + pooledLease = null; + connection = null; + leasePinnedAtRequestStart = false; + if (lease == null) { + return; + } + if (evict) { + lease.evict(); + } else { + lease.close(); + } + } + + private String buildPoolIdentity(ConnectParams params) { + StringBuilder identity = new StringBuilder(); + appendPoolIdentity(identity, "agentClass", getClass().getName()); + appendPoolIdentity(identity, "driverClass", effectiveDriverClass(params)); + appendPoolIdentity(identity, "jdbcUrl", buildJdbcUrl(params)); + appendPoolIdentity(identity, "host", params.getHost()); + appendPoolIdentity(identity, "port", Integer.toString(params.getPort())); + appendPoolIdentity(identity, "database", params.getDatabase()); + appendPoolIdentity(identity, "username", params.getUsername()); + appendPoolIdentity(identity, "password", params.getPassword()); + appendPoolIdentity(identity, "urlParams", params.getUrl_params()); + appendPoolIdentity(identity, "connectionString", params.getConnection_string()); + appendPoolIdentity(identity, "portExplicit", Boolean.toString(params.isPort_explicit())); + appendPoolIdentity(identity, "mysqlCompatMode", Boolean.toString(params.isMysql_compat_mode())); + appendPoolIdentity(identity, "ssl", Boolean.toString(params.isSsl())); + appendPoolIdentity(identity, "caCertPath", params.getCa_cert_path()); + appendPoolIdentity(identity, "clientCertPath", params.getClient_cert_path()); + appendPoolIdentity(identity, "clientKeyPath", params.getClient_key_path()); + appendPoolIdentity(identity, "gbaseServer", params.getGbase_server()); + appendPoolIdentity(identity, "informixServer", params.getInformix_server()); + List driverPaths = params.getJdbc_driver_paths(); + if (driverPaths != null) { + for (int index = 0; index < driverPaths.size(); index++) { + appendPoolIdentity(identity, "driverPath" + index, driverPaths.get(index)); + } + } + return identity.toString(); + } + + private String effectiveDriverClass(ConnectParams params) { + String configured = params.getJdbc_driver_class(); + return configured == null || configured.isEmpty() ? driverClass() : configured; + } + + private static void appendPoolIdentity(StringBuilder identity, String key, String value) { + String normalized = value == null ? "" : value; + identity.append(key.length()).append(':').append(key) + .append('=').append(normalized.length()).append(':').append(normalized).append(';'); + } } diff --git a/agents/common/src/main/java/com/dbx/agent/JdbcConnectionAffinity.java b/agents/common/src/main/java/com/dbx/agent/JdbcConnectionAffinity.java new file mode 100644 index 0000000000..3b0a34cdc5 --- /dev/null +++ b/agents/common/src/main/java/com/dbx/agent/JdbcConnectionAffinity.java @@ -0,0 +1,177 @@ +package com.dbx.agent; + +import java.util.Locale; +import java.util.regex.Pattern; + +final class JdbcConnectionAffinity { + private static final Pattern STATEFUL_STATEMENT = Pattern.compile( + "(?is)(?:^|;)\\s*(?:BEGIN(?:\\s+(?:WORK|TRANSACTION))?\\b|START\\s+TRANSACTION\\b|SET\\b|RESET\\b|UNSET\\b|" + + "USE\\b|DATABASE\\b|ALTER\\s+SESSION\\b|CREATE\\s+(?:(?:GLOBAL|LOCAL)\\s+)?TEMP(?:ORARY)?\\b|" + + "CREATE\\s+(?:(?:SET|MULTISET)\\s+)?VOLATILE\\b|CREATE\\s+TABLE\\s+(?:#{1,2}|\\[#{1,2})|" + + "SELECT\\s+.+?\\s+INTO\\s+(?:(?:(?:GLOBAL|LOCAL)\\s+)?TEMP(?:ORARY)?\\b|#{1,2}|\\[#{1,2})|" + + "DECLARE\\b|PREPARE\\b|DEALLOCATE\\b|ATTACH\\b|DETACH\\b|PRAGMA\\b|CALL\\b|EXEC(?:UTE)?\\b|DO\\b|" + + "LISTEN\\b|UNLISTEN\\b|LOCK\\s+TABLES?\\b|UNLOCK\\s+TABLES?\\b|LOAD\\b|INSTALL\\b|" + + "ADD\\s+(?:JAR|FILE|ARCHIVE)\\b|DELETE\\s+(?:JAR|FILE|ARCHIVE)\\b|CACHE\\s+TABLE\\b|UNCACHE\\s+TABLE\\b)" + ); + private static final Pattern STATEFUL_FUNCTION = Pattern.compile( + "(?is)\\b(?:SET_CONFIG|PG_ADVISORY_(?:XACT_)?LOCK|GET_LOCK|RELEASE_LOCK|SP_GETAPPLOCK|DBMS_LOCK)\\s*\\(" + ); + private static final Pattern USER_VARIABLE_ASSIGNMENT = Pattern.compile("(?is)(?:SET\\s+)?@[A-Z0-9_$]+\\s*(?::=|=)"); + private static final Pattern TEMPORARY_TABLE_REFERENCE = Pattern.compile( + "(?is)(?:^|[^A-Z0-9_$])(?:#{1,2}|\\[#{1,2})[A-Z0-9_$]+" + ); + + private JdbcConnectionAffinity() { + } + + static boolean requiresSessionAffinity(String sql) { + if (sql == null) { + return false; + } + String normalized = sanitizeSql(sql).trim().toUpperCase(Locale.ROOT); + if (normalized.isEmpty()) { + return false; + } + return STATEFUL_STATEMENT.matcher(normalized).find() + || STATEFUL_FUNCTION.matcher(normalized).find() + || USER_VARIABLE_ASSIGNMENT.matcher(normalized).find() + || TEMPORARY_TABLE_REFERENCE.matcher(normalized).find(); + } + + private static String sanitizeSql(String sql) { + StringBuilder sanitized = new StringBuilder(sql.length()); + int index = 0; + while (index < sql.length()) { + char current = sql.charAt(index); + if (current == '-' && index + 1 < sql.length() && sql.charAt(index + 1) == '-') { + index = sanitizeLineComment(sql, sanitized, index); + } else if (current == '#' && isLineCommentStart(sql, index)) { + index = sanitizeLineComment(sql, sanitized, index); + } else if (current == '/' && index + 1 < sql.length() && sql.charAt(index + 1) == '*') { + index = sanitizeBlockComment(sql, sanitized, index); + } else if (current == '\'' || current == '"' || current == '`') { + index = sanitizeQuoted(sql, sanitized, index, current); + } else if (current == '[') { + index = sanitizeQuoted(sql, sanitized, index, ']'); + } else if (current == '$') { + String delimiter = dollarQuoteDelimiter(sql, index); + if (delimiter == null) { + sanitized.append(current); + index += 1; + } else { + int closing = sql.indexOf(delimiter, index + delimiter.length()); + if (closing < 0) { + sanitized.append(current); + index += 1; + } else { + int end = closing + delimiter.length(); + appendSanitized(sanitized, sql, index, end); + index = end; + } + } + } else { + sanitized.append(current); + index += 1; + } + } + return sanitized.toString(); + } + + private static int sanitizeLineComment(String sql, StringBuilder sanitized, int start) { + int end = start + (sql.charAt(start) == '#' ? 1 : 2); + while (end < sql.length() && sql.charAt(end) != '\n' && sql.charAt(end) != '\r') { + end += 1; + } + appendSanitized(sanitized, sql, start, end); + return end; + } + + private static int sanitizeBlockComment(String sql, StringBuilder sanitized, int start) { + int closing = sql.indexOf("*/", start + 2); + int end = closing < 0 ? sql.length() : closing + 2; + int executableStart = executableCommentContentStart(sql, start, closing); + if (executableStart >= 0) { + appendSanitized(sanitized, sql, start, executableStart); + sanitized.append(sanitizeSql(sql.substring(executableStart, closing))); + appendSanitized(sanitized, sql, closing, end); + return end; + } + appendSanitized(sanitized, sql, start, end); + return end; + } + + private static boolean isLineCommentStart(String sql, int index) { + for (int previous = index - 1; previous >= 0; previous--) { + char current = sql.charAt(previous); + if (current == '\n' || current == '\r') { + return true; + } + if (!Character.isWhitespace(current)) { + return current == ';'; + } + } + return true; + } + + private static int executableCommentContentStart(String sql, int start, int closing) { + if (closing < 0) { + return -1; + } + int contentStart; + if (sql.startsWith("/*!", start)) { + contentStart = start + 3; + } else if (sql.startsWith("/*M!", start) || sql.startsWith("/*m!", start)) { + contentStart = start + 4; + } else { + return -1; + } + while (contentStart < closing && Character.isDigit(sql.charAt(contentStart))) { + contentStart += 1; + } + return contentStart; + } + + private static int sanitizeQuoted(String sql, StringBuilder sanitized, int start, char closing) { + int index = start + 1; + while (index < sql.length()) { + char current = sql.charAt(index); + if (current == closing) { + if (index + 1 < sql.length() && sql.charAt(index + 1) == closing) { + index += 2; + continue; + } + index += 1; + break; + } + index += 1; + } + if (closing == ']' && start + 1 < sql.length() && sql.charAt(start + 1) == '#') { + sanitized.append(sql, start, index); + } else { + appendSanitized(sanitized, sql, start, index); + } + return index; + } + + private static String dollarQuoteDelimiter(String sql, int start) { + int end = start + 1; + while (end < sql.length()) { + char current = sql.charAt(end); + if (current == '$') { + return sql.substring(start, end + 1); + } + if (!(Character.isLetterOrDigit(current) || current == '_')) { + return null; + } + end += 1; + } + return null; + } + + private static void appendSanitized(StringBuilder sanitized, String sql, int start, int end) { + for (int index = start; index < end; index++) { + char current = sql.charAt(index); + sanitized.append(current == '\n' || current == '\r' ? current : ' '); + } + } +} diff --git a/agents/common/src/main/java/com/dbx/agent/JdbcConnectionPoolRegistry.java b/agents/common/src/main/java/com/dbx/agent/JdbcConnectionPoolRegistry.java new file mode 100644 index 0000000000..885e34abec --- /dev/null +++ b/agents/common/src/main/java/com/dbx/agent/JdbcConnectionPoolRegistry.java @@ -0,0 +1,516 @@ +package com.dbx.agent; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +final class JdbcConnectionPoolRegistry implements AutoCloseable { + private static final int DEFAULT_MAXIMUM_POOL_SIZE = 8; + private static final int DEFAULT_MINIMUM_IDLE = 0; + private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = 30_000L; + private static final long DEFAULT_VALIDATION_TIMEOUT_MILLIS = 5_000L; + private static final long DEFAULT_IDLE_TIMEOUT_MILLIS = 120_000L; + private static final long DEFAULT_MAX_LIFETIME_MILLIS = 1_800_000L; + private static final long DEFAULT_POOL_RETIRE_MILLIS = 300_000L; + private final Map pools = new ConcurrentHashMap<>(); + private final PoolSettings settings; + private final AtomicBoolean closed = new AtomicBoolean(); + + JdbcConnectionPoolRegistry() { + this(PoolSettings.fromEnvironment()); + } + + JdbcConnectionPoolRegistry(PoolSettings settings) { + this.settings = Objects.requireNonNull(settings, "settings"); + } + + Lease borrow(String identity, ConnectionFactory connectionFactory) throws Exception { + String key = digest(identity); + while (true) { + if (closed.get()) { + throw new IllegalStateException("JDBC connection pool registry is closed"); + } + PoolEntry entry; + try { + entry = pools.computeIfAbsent(key, ignored -> createPoolEntry(key, connectionFactory)); + } catch (PoolCreationException error) { + throw error.unwrap(); + } + try { + return entry.borrow(); + } catch (PoolRetiredException ignored) { + pools.remove(key, entry); + } + } + } + + int poolCount() { + return pools.size(); + } + + boolean isEnabled() { + return settings.enabled; + } + + private PoolEntry createPoolEntry(String key, ConnectionFactory connectionFactory) { + ConnectionFactoryDataSource factoryDataSource = null; + try { + Connection initialConnection = Objects.requireNonNull( + connectionFactory.open(), + "JDBC connection factory returned null" + ); + factoryDataSource = new ConnectionFactoryDataSource(connectionFactory, initialConnection); + HikariConfig config = new HikariConfig(); + config.setPoolName("dbx-jdbc-" + key.substring(0, 12)); + config.setDataSource(factoryDataSource); + config.setMaximumPoolSize(settings.maximumPoolSize); + config.setMinimumIdle(settings.minimumIdle); + config.setConnectionTimeout(settings.connectionTimeoutMillis); + config.setValidationTimeout(settings.validationTimeoutMillis); + config.setIdleTimeout(settings.idleTimeoutMillis); + config.setMaxLifetime(settings.maxLifetimeMillis); + config.setInitializationFailTimeout(-1L); + config.setIsolateInternalQueries(true); + config.setThreadFactory(daemonThreadFactory("dbx-jdbc-pool-" + key.substring(0, 8))); + return new PoolEntry(new HikariDataSource(config), factoryDataSource, settings.poolRetireMillis); + } catch (Exception error) { + if (factoryDataSource != null) { + factoryDataSource.closeUnusedInitialConnection(); + } + throw new PoolCreationException(error); + } + } + + void retireUnusedPools() { + if (closed.get()) { + return; + } + long now = System.currentTimeMillis(); + List retired = new ArrayList<>(); + for (String key : pools.keySet()) { + pools.computeIfPresent(key, (ignored, entry) -> { + if (entry.tryRetire(now)) { + retired.add(entry); + return null; + } + return entry; + }); + } + for (PoolEntry entry : retired) { + entry.closeRetired(); + } + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + for (PoolEntry entry : pools.values()) { + entry.close(); + } + pools.clear(); + } + + private static String digest(String identity) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(identity.getBytes(StandardCharsets.UTF_8)); + StringBuilder result = new StringBuilder(hash.length * 2); + for (byte value : hash) { + result.append(Character.forDigit((value >>> 4) & 0x0f, 16)); + result.append(Character.forDigit(value & 0x0f, 16)); + } + return result.toString(); + } catch (Exception error) { + throw new IllegalStateException("SHA-256 is unavailable", error); + } + } + + private static ThreadFactory daemonThreadFactory(String name) { + return runnable -> { + Thread thread = new Thread(runnable, name); + thread.setDaemon(true); + return thread; + }; + } + + @FunctionalInterface + interface ConnectionFactory { + Connection open() throws Exception; + } + + static final class Lease implements AutoCloseable { + private final PoolEntry entry; + private final Connection connection; + private final AtomicBoolean closed = new AtomicBoolean(); + + private Lease(PoolEntry entry, Connection connection) { + this.entry = entry; + this.connection = connection; + } + + Connection connection() { + return connection; + } + + void evict() { + if (closed.compareAndSet(false, true)) { + entry.release(connection, true); + } + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + entry.release(connection, false); + } + } + } + + static final class PoolSettings { + private final boolean enabled; + private final int maximumPoolSize; + private final int minimumIdle; + private final long connectionTimeoutMillis; + private final long validationTimeoutMillis; + private final long idleTimeoutMillis; + private final long maxLifetimeMillis; + private final long poolRetireMillis; + + PoolSettings( + int maximumPoolSize, + int minimumIdle, + long connectionTimeoutMillis, + long validationTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + long poolRetireMillis + ) { + this( + true, + maximumPoolSize, + minimumIdle, + connectionTimeoutMillis, + validationTimeoutMillis, + idleTimeoutMillis, + maxLifetimeMillis, + poolRetireMillis + ); + } + + PoolSettings( + boolean enabled, + int maximumPoolSize, + int minimumIdle, + long connectionTimeoutMillis, + long validationTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + long poolRetireMillis + ) { + this.enabled = enabled; + this.maximumPoolSize = maximumPoolSize; + this.minimumIdle = minimumIdle; + this.connectionTimeoutMillis = connectionTimeoutMillis; + this.validationTimeoutMillis = validationTimeoutMillis; + this.idleTimeoutMillis = idleTimeoutMillis; + this.maxLifetimeMillis = maxLifetimeMillis; + this.poolRetireMillis = poolRetireMillis; + } + + static PoolSettings fromEnvironment() { + int maximumPoolSize = intSetting( + "dbx.agent.jdbc.pool.maximumPoolSize", + "DBX_AGENT_JDBC_POOL_MAXIMUM_POOL_SIZE", + DEFAULT_MAXIMUM_POOL_SIZE, + 1, + 32 + ); + int minimumIdle = intSetting( + "dbx.agent.jdbc.pool.minimumIdle", + "DBX_AGENT_JDBC_POOL_MINIMUM_IDLE", + DEFAULT_MINIMUM_IDLE, + 0, + maximumPoolSize + ); + long connectionTimeoutMillis = longSetting( + "dbx.agent.jdbc.pool.connectionTimeoutMillis", + "DBX_AGENT_JDBC_POOL_CONNECTION_TIMEOUT_MILLIS", + DEFAULT_CONNECTION_TIMEOUT_MILLIS, + 250L + ); + long validationTimeoutMillis = Math.min( + connectionTimeoutMillis, + longSetting( + "dbx.agent.jdbc.pool.validationTimeoutMillis", + "DBX_AGENT_JDBC_POOL_VALIDATION_TIMEOUT_MILLIS", + DEFAULT_VALIDATION_TIMEOUT_MILLIS, + 250L + ) + ); + return new PoolSettings( + booleanSetting( + "dbx.agent.jdbc.pool.enabled", + "DBX_AGENT_JDBC_POOL_ENABLED", + true + ), + maximumPoolSize, + minimumIdle, + connectionTimeoutMillis, + validationTimeoutMillis, + longSetting( + "dbx.agent.jdbc.pool.idleTimeoutMillis", + "DBX_AGENT_JDBC_POOL_IDLE_TIMEOUT_MILLIS", + DEFAULT_IDLE_TIMEOUT_MILLIS, + 10_000L + ), + longSetting( + "dbx.agent.jdbc.pool.maxLifetimeMillis", + "DBX_AGENT_JDBC_POOL_MAX_LIFETIME_MILLIS", + DEFAULT_MAX_LIFETIME_MILLIS, + 30_000L + ), + longSetting( + "dbx.agent.jdbc.pool.retireMillis", + "DBX_AGENT_JDBC_POOL_RETIRE_MILLIS", + DEFAULT_POOL_RETIRE_MILLIS, + 60_000L + ) + ); + } + + private static int intSetting(String property, String environment, int defaultValue, int minimum, int maximum) { + String value = setting(property, environment); + if (value == null) { + return defaultValue; + } + try { + return Math.max(minimum, Math.min(maximum, Integer.parseInt(value.trim()))); + } catch (NumberFormatException ignored) { + return defaultValue; + } + } + + private static long longSetting(String property, String environment, long defaultValue, long minimum) { + String value = setting(property, environment); + if (value == null) { + return defaultValue; + } + try { + return Math.max(minimum, Long.parseLong(value.trim())); + } catch (NumberFormatException ignored) { + return defaultValue; + } + } + + private static boolean booleanSetting(String property, String environment, boolean defaultValue) { + String value = setting(property, environment); + if (value == null) { + return defaultValue; + } + String normalized = value.trim(); + if ("true".equalsIgnoreCase(normalized) || "1".equals(normalized) || "yes".equalsIgnoreCase(normalized)) { + return true; + } + if ("false".equalsIgnoreCase(normalized) || "0".equals(normalized) || "no".equalsIgnoreCase(normalized)) { + return false; + } + return defaultValue; + } + + private static String setting(String property, String environment) { + String value = System.getProperty(property); + return value == null || value.trim().isEmpty() ? System.getenv(environment) : value; + } + } + + private static final class PoolEntry implements AutoCloseable { + private final HikariDataSource dataSource; + private final ConnectionFactoryDataSource factoryDataSource; + private final long retireMillis; + private int activeLeases; + private boolean retired; + private boolean dataSourceClosed; + private volatile long lastReleasedAtMillis = System.currentTimeMillis(); + + private PoolEntry( + HikariDataSource dataSource, + ConnectionFactoryDataSource factoryDataSource, + long retireMillis + ) { + this.dataSource = dataSource; + this.factoryDataSource = factoryDataSource; + this.retireMillis = retireMillis; + } + + private Lease borrow() throws SQLException { + synchronized (this) { + if (retired) { + throw new PoolRetiredException(); + } + activeLeases += 1; + } + try { + return new Lease(this, dataSource.getConnection()); + } catch (SQLException | RuntimeException error) { + synchronized (this) { + activeLeases -= 1; + lastReleasedAtMillis = System.currentTimeMillis(); + } + throw error; + } + } + + private void release(Connection connection, boolean evict) { + try { + if (evict) { + dataSource.evictConnection(connection); + } + connection.close(); + } catch (Exception ignored) { + } finally { + synchronized (this) { + lastReleasedAtMillis = System.currentTimeMillis(); + activeLeases -= 1; + } + } + } + + private synchronized boolean tryRetire(long nowMillis) { + if (retired || activeLeases != 0 || nowMillis - lastReleasedAtMillis < retireMillis) { + return false; + } + retired = true; + return true; + } + + private void closeRetired() { + synchronized (this) { + if (dataSourceClosed) { + return; + } + dataSourceClosed = true; + } + factoryDataSource.closeUnusedInitialConnection(); + dataSource.close(); + } + + @Override + public void close() { + synchronized (this) { + retired = true; + } + closeRetired(); + } + } + + private static final class PoolRetiredException extends SQLException { + private PoolRetiredException() { + super("JDBC connection pool was retired"); + } + } + + private static final class ConnectionFactoryDataSource implements DataSource { + private final ConnectionFactory connectionFactory; + private final AtomicReference initialConnection; + + private ConnectionFactoryDataSource(ConnectionFactory connectionFactory, Connection initialConnection) { + this.connectionFactory = connectionFactory; + this.initialConnection = new AtomicReference<>(initialConnection); + } + + @Override + public Connection getConnection() throws SQLException { + Connection opened = initialConnection.getAndSet(null); + if (opened != null) { + return opened; + } + try { + return connectionFactory.open(); + } catch (SQLException error) { + throw error; + } catch (Exception error) { + throw new SQLException("Failed to open JDBC connection", error); + } + } + + private void closeUnusedInitialConnection() { + Connection opened = initialConnection.getAndSet(null); + if (opened == null) { + return; + } + try { + opened.close(); + } catch (Exception ignored) { + } + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + return getConnection(); + } + + @Override + public PrintWriter getLogWriter() { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) { + } + + @Override + public void setLoginTimeout(int seconds) { + } + + @Override + public int getLoginTimeout() { + return 0; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + return Logger.getLogger("com.dbx.agent.jdbc.pool"); + } + + @Override + public T unwrap(Class iface) throws SQLException { + if (iface.isInstance(this)) { + return iface.cast(this); + } + throw new SQLException("Not a wrapper for " + iface.getName()); + } + + @Override + public boolean isWrapperFor(Class iface) { + return iface.isInstance(this); + } + } + + private static final class PoolCreationException extends RuntimeException { + private PoolCreationException(Exception cause) { + super(cause); + } + + private Exception unwrap() { + Throwable cause = getCause(); + return cause instanceof Exception ? (Exception) cause : this; + } + } +} diff --git a/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java b/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java index 3f937ff391..a7e2898d83 100644 --- a/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java +++ b/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java @@ -254,6 +254,7 @@ public final class JdbcExecutor { applySchema(conn, schema, setSchemaSql, resetSchemaSql); Statement stmt = conn.createStatement(); + QuerySession createdSession = null; activeStatements.add(stmt); try { applyQueryTimeout(stmt, options.getTimeoutSecs()); @@ -303,9 +304,13 @@ public final class JdbcExecutor { Math.max(options.getMaxRows(), 1), valueReader ); + createdSession = session; targetSessions.put(sessionId, session); return readSessionPage(targetSessions, session, options.getPageSize(), elapsed); } catch (Exception e) { + if (createdSession != null) { + closeSession(targetSessions, createdSession.id); + } activeStatements.remove(stmt); try { stmt.close(); @@ -356,6 +361,23 @@ public final class JdbcExecutor { closeAllTableReadSessions(); } + public boolean hasOpenSessions() { + return !sessions.isEmpty() || !tableReadSessions.isEmpty(); + } + + public boolean hasActiveStatements() { + return !activeStatements.isEmpty(); + } + + public int expireIdleResources() { + return expireIdleQuerySessions() + expireIdleTableReadSessions(); + } + + int expireIdleResources(long nowMillis, long idleTimeoutMillis) { + return expireIdleQuerySessions(nowMillis, idleTimeoutMillis) + + expireIdleTableReadSessions(nowMillis, idleTimeoutMillis); + } + public int expireIdleQuerySessions() { return expireIdleQuerySessions(System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS); } @@ -516,7 +538,12 @@ public final class JdbcExecutor { throw new IllegalArgumentException(missingMessage); } synchronized (session) { - return readSessionPage(targetSessions, session, pageSize, 0L); + try { + return readSessionPage(targetSessions, session, pageSize, 0L); + } catch (RuntimeException | Error error) { + closeSession(targetSessions, sessionId); + throw error; + } } } diff --git a/agents/common/src/main/java/com/dbx/agent/JdbcSchemaSwitcher.java b/agents/common/src/main/java/com/dbx/agent/JdbcSchemaSwitcher.java index 32d4282594..9afc686fe2 100644 --- a/agents/common/src/main/java/com/dbx/agent/JdbcSchemaSwitcher.java +++ b/agents/common/src/main/java/com/dbx/agent/JdbcSchemaSwitcher.java @@ -10,7 +10,7 @@ import java.util.function.Function; import java.util.function.Supplier; final class JdbcSchemaSwitcher { - private static final Map RESET_SQL_BY_CONNECTION = + private static final Map RESET_STATE_BY_CONNECTION = Collections.synchronizedMap(new WeakHashMap<>()); private JdbcSchemaSwitcher() { @@ -27,37 +27,33 @@ final class JdbcSchemaSwitcher { Supplier resetSchemaSql ) throws Exception { if (schema == null || schema.trim().isEmpty()) { - String resetSql = RESET_SQL_BY_CONNECTION.remove(conn); - if (resetSql == null) { - return; - } - SchemaSqlResult resetResult = applySchemaSql(conn, () -> resetSql); - if (resetResult.attempted && resetResult.error != null) { - RESET_SQL_BY_CONNECTION.put(conn, resetSql); - throw resetResult.error; + if (!restore(conn, true)) { + throw new SQLException("Failed to restore the original JDBC schema context"); } return; } + ResetState resetState = resetState(conn); SchemaSqlResult schemaSqlResult = applySchemaSql(conn, () -> setSchemaSql.apply(schema)); Exception schemaSqlError = schemaSqlResult.error; if (schemaSqlError == null) { - rememberResetSql(conn, resetSchemaSql); + resetState.mode = ResetMode.SQL; + rememberResetSql(resetState, resetSchemaSql); return; } try { conn.setSchema(schema); - rememberResetSql(conn, resetSchemaSql); + resetState.mode = ResetMode.SCHEMA; return; - } catch (SQLException | AbstractMethodError ignored) { + } catch (SQLException | UnsupportedOperationException | AbstractMethodError ignored) { // Some JDBC drivers only expose schema switching through SQL. } try { conn.setCatalog(schema); - rememberResetSql(conn, resetSchemaSql); + resetState.mode = ResetMode.CATALOG; return; - } catch (SQLException | AbstractMethodError ignored) { + } catch (SQLException | UnsupportedOperationException | AbstractMethodError ignored) { // Last fallback failed as well; surface the SQL-switch error below. } @@ -66,11 +62,58 @@ final class JdbcSchemaSwitcher { } } - private static void rememberResetSql(Connection conn, Supplier resetSchemaSql) { + static boolean resetBeforeReturn(Connection conn) { + return restore(conn, false); + } + + static void forget(Connection conn) { + RESET_STATE_BY_CONNECTION.remove(conn); + } + + private static boolean restore(Connection conn, boolean preserveOnFailure) { + ResetState resetState = RESET_STATE_BY_CONNECTION.remove(conn); + if (resetState == null) { + return true; + } + try { + boolean restored = resetState.restore(conn); + if (!restored && preserveOnFailure) { + RESET_STATE_BY_CONNECTION.put(conn, resetState); + } + return restored; + } catch (Exception ignored) { + if (preserveOnFailure) { + RESET_STATE_BY_CONNECTION.put(conn, resetState); + } + return false; + } + } + + private static ResetState resetState(Connection conn) { + synchronized (RESET_STATE_BY_CONNECTION) { + return RESET_STATE_BY_CONNECTION.computeIfAbsent(conn, JdbcSchemaSwitcher::captureResetState); + } + } + + private static ResetState captureResetState(Connection conn) { + JdbcValue schema = readJdbcValue(conn, true); + JdbcValue catalog = readJdbcValue(conn, false); + return new ResetState(schema, catalog); + } + + private static JdbcValue readJdbcValue(Connection conn, boolean schema) { + try { + return new JdbcValue<>(true, schema ? conn.getSchema() : conn.getCatalog()); + } catch (SQLException | UnsupportedOperationException | AbstractMethodError ignored) { + return new JdbcValue<>(false, null); + } + } + + private static void rememberResetSql(ResetState resetState, Supplier resetSchemaSql) { try { String sql = resetSchemaSql.get(); if (sql != null && !sql.trim().isEmpty()) { - RESET_SQL_BY_CONNECTION.put(conn, sql); + resetState.resetSql = sql; } } catch (RuntimeException ignored) { // Drivers without a reset command keep the legacy no-op behavior. @@ -95,6 +138,78 @@ final class JdbcSchemaSwitcher { } } + private enum ResetMode { + SQL, + SCHEMA, + CATALOG + } + + private static final class JdbcValue { + private final boolean supported; + private final T value; + + private JdbcValue(boolean supported, T value) { + this.supported = supported; + this.value = value; + } + } + + private static final class ResetState { + private final JdbcValue schema; + private final JdbcValue catalog; + private ResetMode mode; + private String resetSql; + + private ResetState(JdbcValue schema, JdbcValue catalog) { + this.schema = schema; + this.catalog = catalog; + } + + private boolean restore(Connection conn) throws Exception { + if (mode == ResetMode.SQL && resetSql != null) { + SchemaSqlResult result = applySchemaSql(conn, () -> resetSql); + return result.attempted && result.error == null; + } + if (mode == ResetMode.SCHEMA) { + return restoreSchema(conn); + } + if (mode == ResetMode.CATALOG) { + return restoreCatalog(conn); + } + if (mode == ResetMode.SQL) { + if (schema.supported && schema.value != null && restoreSchema(conn)) { + return true; + } + return catalog.supported && catalog.value != null && restoreCatalog(conn); + } + return true; + } + + private boolean restoreSchema(Connection conn) { + if (!schema.supported) { + return false; + } + try { + conn.setSchema(schema.value); + return true; + } catch (SQLException | UnsupportedOperationException | AbstractMethodError ignored) { + return false; + } + } + + private boolean restoreCatalog(Connection conn) { + if (!catalog.supported) { + return false; + } + try { + conn.setCatalog(catalog.value); + return true; + } catch (SQLException | UnsupportedOperationException | AbstractMethodError ignored) { + return false; + } + } + } + private static final class SchemaSqlResult { private final boolean attempted; private final Exception error; diff --git a/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java b/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java index c7e2e1c963..4b2aa0d91a 100644 --- a/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java +++ b/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java @@ -89,11 +89,53 @@ public final class JsonRpcServer { } Object dispatchForRuntime(String method, JsonObject params) throws Exception { - return AgentExecutionContext.withJdbcExecutor(jdbcExecutor, () -> dispatch(method, params)); + return AgentExecutionContext.withJdbcExecutor(jdbcExecutor, () -> { + AbstractJdbcAgent jdbcAgent = pooledJdbcAgent(); + boolean manageConnection = jdbcAgent != null && requiresConnectedConnection(method); + if (manageConnection) { + jdbcAgent.beginPooledRequest(); + } + boolean succeeded = false; + try { + Object result = dispatch(method, params); + succeeded = true; + return result; + } finally { + if (manageConnection) { + jdbcAgent.finishPooledRequest( + jdbcExecutor, + succeeded, + requiresSessionAffinity(method, params), + evictAfterRequest(method) + ); + } + } + }); } void cancelActiveStatements() { jdbcExecutor.cancelActiveStatements(); + AbstractJdbcAgent jdbcAgent = pooledJdbcAgent(); + if (jdbcAgent != null) { + jdbcAgent.releaseIdlePooledConnection(jdbcExecutor); + } + } + + void expireIdleResources() { + jdbcExecutor.expireIdleResources(); + releaseIdlePooledConnection(); + } + + void expireIdleResources(long nowMillis, long idleTimeoutMillis) { + jdbcExecutor.expireIdleResources(nowMillis, idleTimeoutMillis); + releaseIdlePooledConnection(); + } + + private void releaseIdlePooledConnection() { + AbstractJdbcAgent jdbcAgent = pooledJdbcAgent(); + if (jdbcAgent != null) { + jdbcAgent.releaseIdlePooledConnection(jdbcExecutor); + } } private Object dispatch(String method, JsonObject params) throws Exception { @@ -101,8 +143,9 @@ public final class JsonRpcServer { return AgentProtocol.handshakeResult(); } if (AgentProtocol.METHOD_CONNECT.equals(method)) { - lastConnectParams = gson.fromJson(params, ConnectParams.class); - agent.connect(lastConnectParams); + ConnectParams connectParams = gson.fromJson(params, ConnectParams.class); + agent.connect(connectParams); + lastConnectParams = connectParams; lastConnectionValidationTimeMillis = 0L; return Collections.singletonMap("ok", true); } @@ -302,6 +345,10 @@ public final class JsonRpcServer { if (lastConnectParams == null || !shouldValidateConnection(method)) { return; } + AbstractJdbcAgent jdbcAgent = pooledJdbcAgent(); + if (jdbcAgent != null) { + return; + } Connection conn = agent.getConnection(); if (conn == null) { return; @@ -344,6 +391,51 @@ public final class JsonRpcServer { && !AgentProtocol.METHOD_SHUTDOWN.equals(method); } + private AbstractJdbcAgent pooledJdbcAgent() { + if (agent instanceof AbstractJdbcAgent jdbcAgent && jdbcAgent.usesConnectionPool()) { + return jdbcAgent; + } + return null; + } + + private static boolean requiresConnectedConnection(String method) { + return !AgentProtocol.METHOD_HANDSHAKE.equals(method) + && !AgentProtocol.METHOD_CONNECT.equals(method) + && !AgentProtocol.METHOD_TEST_CONNECTION.equals(method) + && !AgentProtocol.METHOD_DISCONNECT.equals(method) + && !AgentProtocol.METHOD_SHUTDOWN.equals(method); + } + + private boolean requiresSessionAffinity(String method, JsonObject params) { + try { + if (AgentProtocol.METHOD_EXECUTE_QUERY.equals(method) + || AgentProtocol.METHOD_EXECUTE_QUERY_PAGE.equals(method) + || AgentProtocol.METHOD_START_TABLE_READ.equals(method)) { + return JdbcConnectionAffinity.requiresSessionAffinity(stringOrNull(params, "sql")); + } + if (AgentProtocol.METHOD_EXECUTE_TRANSACTION.equals(method) + || AgentProtocol.METHOD_EXECUTE_BATCH.equals(method)) { + JsonElement statements = params.get("statements"); + if (statements == null || !statements.isJsonArray()) { + return false; + } + for (JsonElement statement : statements.getAsJsonArray()) { + if (!statement.isJsonNull() + && JdbcConnectionAffinity.requiresSessionAffinity(statement.getAsString())) { + return true; + } + } + } + } catch (RuntimeException ignored) { + return false; + } + return false; + } + + private static boolean evictAfterRequest(String method) { + return AgentProtocol.METHOD_GET_EXPLAIN_INFO.equals(method); + } + private static JsonObject paramsObject(JsonObject req) { JsonElement params = req.get("params"); if (params == null || params instanceof JsonNull || !params.isJsonObject()) { diff --git a/agents/common/src/main/java/com/dbx/agent/MultiSessionJsonRpcServer.java b/agents/common/src/main/java/com/dbx/agent/MultiSessionJsonRpcServer.java index f3544776a0..f1155ed8b8 100644 --- a/agents/common/src/main/java/com/dbx/agent/MultiSessionJsonRpcServer.java +++ b/agents/common/src/main/java/com/dbx/agent/MultiSessionJsonRpcServer.java @@ -7,31 +7,58 @@ import com.google.gson.JsonParser; import java.io.BufferedReader; import java.io.InputStreamReader; +import java.io.PrintStream; import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; -public final class MultiSessionJsonRpcServer { +public final class MultiSessionJsonRpcServer implements AutoCloseable { private static final String LEGACY_SESSION_ID = "__legacy__"; private static final int MAX_SESSIONS = 256; + private static final long MAINTENANCE_INTERVAL_MILLIS = 60_000L; private final Supplier agentFactory; private final Map sessions = new ConcurrentHashMap<>(); private final ExecutorService requests = Executors.newCachedThreadPool(); + private final JdbcConnectionPoolRegistry poolRegistry; private final Gson gson = new Gson(); + private final PrintStream protocolOutput = System.out; private final Object outputLock = new Object(); + private final Object maintenanceLock = new Object(); + private final AtomicBoolean closed = new AtomicBoolean(); + private ScheduledExecutorService maintenance; public MultiSessionJsonRpcServer(Supplier agentFactory) { + this(agentFactory, new JdbcConnectionPoolRegistry()); + } + + MultiSessionJsonRpcServer( + Supplier agentFactory, + JdbcConnectionPoolRegistry.PoolSettings poolSettings + ) { + this(agentFactory, new JdbcConnectionPoolRegistry(poolSettings)); + } + + private MultiSessionJsonRpcServer( + Supplier agentFactory, + JdbcConnectionPoolRegistry poolRegistry + ) { this.agentFactory = agentFactory; + this.poolRegistry = poolRegistry; } public void run() { synchronized (outputLock) { - System.out.println("{\"ready\":true}"); - System.out.flush(); + protocolOutput.println("{\"ready\":true}"); + protocolOutput.flush(); } try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) { String line; @@ -40,15 +67,14 @@ public final class MultiSessionJsonRpcServer { String method = request.get("method").getAsString(); if (AgentProtocol.METHOD_SHUTDOWN.equals(method)) { writeResponse(handleRequest(request)); - closeAllSessions(); - requests.shutdown(); return; } requests.submit(() -> writeResponse(handleRequest(request))); } } catch (Exception e) { - closeAllSessions(); throw new RuntimeException(e); + } finally { + close(); } } @@ -105,7 +131,12 @@ public final class MultiSessionJsonRpcServer { if (sessions.size() >= MAX_SESSIONS && !sessions.containsKey(sessionId)) { throw new IllegalStateException("Agent session limit reached: " + MAX_SESSIONS); } - Session session = new Session(new JsonRpcServer(agentFactory.get())); + DatabaseAgent agent = agentFactory.get(); + if (poolRegistry.isEnabled() && agent instanceof AbstractJdbcAgent jdbcAgent) { + jdbcAgent.attachConnectionPoolRegistry(poolRegistry); + ensureMaintenanceStarted(); + } + Session session = new Session(new JsonRpcServer(agent)); Session existing = sessions.putIfAbsent(sessionId, session); if (existing != null) { throw new IllegalStateException("Agent session already exists: " + sessionId); @@ -141,6 +172,71 @@ public final class MultiSessionJsonRpcServer { } } + void runMaintenance() { + for (Session session : sessions.values()) { + try { + session.expireIdleResources(); + } catch (Exception ignored) { + } + } + poolRegistry.retireUnusedPools(); + } + + void runMaintenance(long nowMillis, long idleTimeoutMillis) { + for (Session session : sessions.values()) { + try { + session.expireIdleResources(nowMillis, idleTimeoutMillis); + } catch (Exception ignored) { + } + } + poolRegistry.retireUnusedPools(); + } + + private void ensureMaintenanceStarted() { + synchronized (maintenanceLock) { + if (maintenance != null || closed.get()) { + return; + } + maintenance = Executors.newSingleThreadScheduledExecutor(daemonThreadFactory("dbx-jdbc-maintenance")); + maintenance.scheduleWithFixedDelay( + this::runMaintenanceSafely, + MAINTENANCE_INTERVAL_MILLIS, + MAINTENANCE_INTERVAL_MILLIS, + TimeUnit.MILLISECONDS + ); + } + } + + private void runMaintenanceSafely() { + try { + runMaintenance(); + } catch (Throwable ignored) { + } + } + + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + synchronized (maintenanceLock) { + if (maintenance != null) { + maintenance.shutdownNow(); + } + } + closeAllSessions(); + requests.shutdown(); + poolRegistry.close(); + } + + private static ThreadFactory daemonThreadFactory(String name) { + return runnable -> { + Thread thread = new Thread(runnable, name); + thread.setDaemon(true); + return thread; + }; + } + private static String requiredSessionId(JsonObject params) { if (!params.has("agentSessionId") || params.get("agentSessionId").getAsString().trim().isEmpty()) { throw new IllegalArgumentException("agentSessionId is required"); @@ -150,31 +246,62 @@ public final class MultiSessionJsonRpcServer { private void writeResponse(JsonObject response) { synchronized (outputLock) { - System.out.println(gson.toJson(response)); - System.out.flush(); + protocolOutput.println(gson.toJson(response)); + protocolOutput.flush(); } } private static final class Session { private final JsonRpcServer server; + private final ReentrantLock lock = new ReentrantLock(); private Session(JsonRpcServer server) { this.server = server; } - private synchronized Object handle(String method, JsonObject params) throws Exception { - return server.dispatchForRuntime(method, params); + private Object handle(String method, JsonObject params) throws Exception { + lock.lock(); + try { + return server.dispatchForRuntime(method, params); + } finally { + lock.unlock(); + } } - private synchronized void close() { + private void close() { + lock.lock(); try { server.dispatchForRuntime(AgentProtocol.METHOD_DISCONNECT, new JsonObject()); } catch (Exception ignored) { + } finally { + lock.unlock(); } } private void cancel() { server.cancelActiveStatements(); } + + private void expireIdleResources() { + if (!lock.tryLock()) { + return; + } + try { + server.expireIdleResources(); + } finally { + lock.unlock(); + } + } + + private void expireIdleResources(long nowMillis, long idleTimeoutMillis) { + if (!lock.tryLock()) { + return; + } + try { + server.expireIdleResources(nowMillis, idleTimeoutMillis); + } finally { + lock.unlock(); + } + } } } diff --git a/agents/common/src/test/java/com/dbx/agent/AbstractJdbcAgentTest.java b/agents/common/src/test/java/com/dbx/agent/AbstractJdbcAgentTest.java index 7126552e89..7fba6d5ce1 100644 --- a/agents/common/src/test/java/com/dbx/agent/AbstractJdbcAgentTest.java +++ b/agents/common/src/test/java/com/dbx/agent/AbstractJdbcAgentTest.java @@ -43,6 +43,7 @@ class AbstractJdbcAgentTest { assertNull(agent.getConnection()); assertEquals(1, tracking.closeCount); + assertEquals(1, agent.afterDisconnectCount); } @Test @@ -112,6 +113,18 @@ class AbstractJdbcAgentTest { assertEquals(1, tracking.closeCount); } + @Test + void testConnectionCanSkipOpeningAPhysicalConnection() { + TrackingConnection tracking = new TrackingConnection(); + TestAgent agent = new TestAgent(tracking); + agent.skipTestConnectionOpen = true; + + assertFalse(agent.testConnection(new ConnectParams())); + + assertEquals(0, tracking.openCount); + assertEquals(0, tracking.closeCount); + } + @Test void databaseInfoKeepsSupportedFieldsWhenOneMetadataGetterFails() { DatabaseMetaData metadata = proxy(DatabaseMetaData.class, (method, args) -> { @@ -310,6 +323,8 @@ class AbstractJdbcAgentTest { private static class TestAgent extends AbstractJdbcAgent { private final TrackingConnection tracking; private int afterConnectCount; + private int afterDisconnectCount; + private boolean skipTestConnectionOpen; private TestAgent(TrackingConnection tracking) { this.tracking = tracking; @@ -331,11 +346,21 @@ class AbstractJdbcAgentTest { return tracking.connection(); } + @Override + protected Connection openTestConnection(ConnectParams params) { + return skipTestConnectionOpen ? null : openConnection(params); + } + @Override protected void afterConnect(ConnectParams params, Connection connection) { afterConnectCount += 1; } + @Override + protected void afterDisconnect() { + afterDisconnectCount += 1; + } + @Override public List listDatabases() { return Collections.emptyList(); diff --git a/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java b/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java index 19c0a71688..54d2c74069 100644 --- a/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java +++ b/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java @@ -6,8 +6,11 @@ import com.google.gson.JsonParser; import org.junit.jupiter.api.Test; import java.sql.Connection; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.PrintStream; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; @@ -154,6 +157,36 @@ class CommonJavaCompatibilityTest { assertEquals(0, created.get(1).disconnectCount); } + @Test + void multiSessionServerKeepsProtocolOutputWhenGlobalStdoutChanges() { + synchronized (System.class) { + InputStream originalInput = System.in; + PrintStream originalOutput = System.out; + ByteArrayOutputStream protocolBytes = new ByteArrayOutputStream(); + ByteArrayOutputStream redirectedBytes = new ByteArrayOutputStream(); + try (PrintStream protocolOutput = new PrintStream(protocolBytes, true, StandardCharsets.UTF_8); + PrintStream redirectedOutput = new PrintStream(redirectedBytes, true, StandardCharsets.UTF_8)) { + System.setOut(protocolOutput); + MultiSessionJsonRpcServer server = new MultiSessionJsonRpcServer(MinimalAgent::new); + System.setIn(new ByteArrayInputStream( + "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"shutdown\",\"params\":{}}\n" + .getBytes(StandardCharsets.UTF_8) + )); + System.setOut(redirectedOutput); + + server.run(); + + String protocol = protocolBytes.toString(StandardCharsets.UTF_8); + assertTrue(protocol.contains("{\"ready\":true}"), protocol); + assertTrue(protocol.contains("\"id\":1"), protocol); + assertEquals("", redirectedBytes.toString(StandardCharsets.UTF_8)); + } finally { + System.setIn(originalInput); + System.setOut(originalOutput); + } + } + } + @Test void multiSessionCancellationOnlyCancelsTargetSession() throws Exception { List created = Collections.synchronizedList(new ArrayList<>()); diff --git a/agents/common/src/test/java/com/dbx/agent/JdbcConnectionPoolingTest.java b/agents/common/src/test/java/com/dbx/agent/JdbcConnectionPoolingTest.java new file mode 100644 index 0000000000..77d9b4f1ce --- /dev/null +++ b/agents/common/src/test/java/com/dbx/agent/JdbcConnectionPoolingTest.java @@ -0,0 +1,647 @@ +package com.dbx.agent; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class JdbcConnectionPoolingTest { + private static final Gson GSON = new Gson(); + + @Test + void registryReusesOnePhysicalConnectionAcrossSequentialLeases() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + String url = h2Url("registry_reuse"); + try (JdbcConnectionPoolRegistry registry = new JdbcConnectionPoolRegistry(poolSettings(4))) { + for (int index = 0; index < 100; index++) { + try (JdbcConnectionPoolRegistry.Lease lease = registry.borrow( + "same-identity", + () -> openH2(url, physicalOpens) + )) { + try (Statement statement = lease.connection().createStatement()) { + assertTrue(statement.execute("SELECT 1")); + } + } + } + assertEquals(1, physicalOpens.get()); + assertEquals(1, registry.poolCount()); + } + } + + @Test + void registryCapsConcurrentPhysicalConnections() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger activeLeases = new AtomicInteger(); + AtomicInteger maximumActiveLeases = new AtomicInteger(); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch firstTwoBorrowed = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(8); + String url = h2Url("registry_cap"); + + try (JdbcConnectionPoolRegistry registry = new JdbcConnectionPoolRegistry(poolSettings(2))) { + List> futures = new java.util.ArrayList<>(); + for (int index = 0; index < 8; index++) { + futures.add(workers.submit(() -> { + start.await(); + try (JdbcConnectionPoolRegistry.Lease ignored = registry.borrow( + "same-identity", + () -> openH2(url, physicalOpens) + )) { + int active = activeLeases.incrementAndGet(); + maximumActiveLeases.accumulateAndGet(active, Math::max); + firstTwoBorrowed.countDown(); + release.await(); + activeLeases.decrementAndGet(); + } + return null; + })); + } + + start.countDown(); + assertTrue(firstTwoBorrowed.await(2, TimeUnit.SECONDS)); + Thread.sleep(150L); + assertEquals(2, physicalOpens.get()); + assertEquals(2, maximumActiveLeases.get()); + release.countDown(); + for (Future future : futures) { + future.get(3, TimeUnit.SECONDS); + } + assertEquals(2, physicalOpens.get()); + } finally { + release.countDown(); + workers.shutdownNow(); + } + } + + @Test + void registrySeparatesDifferentConnectionIdentities() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + String url = h2Url("registry_identity"); + try (JdbcConnectionPoolRegistry registry = new JdbcConnectionPoolRegistry(poolSettings(2))) { + try (JdbcConnectionPoolRegistry.Lease ignored = registry.borrow( + "identity-a", + () -> openH2(url, physicalOpens) + )) { + } + try (JdbcConnectionPoolRegistry.Lease ignored = registry.borrow( + "identity-b", + () -> openH2(url, physicalOpens) + )) { + } + + assertEquals(2, physicalOpens.get()); + assertEquals(2, registry.poolCount()); + } + } + + @Test + void registrySurfacesInitialPhysicalConnectionFailureWithoutWaitingForBorrowTimeout() { + JdbcConnectionPoolRegistry.PoolSettings settings = new JdbcConnectionPoolRegistry.PoolSettings( + 1, + 0, + 5_000L, + 1_000L, + 10_000L, + 30_000L, + 60_000L + ); + long startedAtMillis = System.currentTimeMillis(); + try (JdbcConnectionPoolRegistry registry = new JdbcConnectionPoolRegistry(settings)) { + SQLException error = assertThrows( + SQLException.class, + () -> registry.borrow("failing", () -> { + throw new SQLException("auth failed"); + }) + ); + long elapsedMillis = System.currentTimeMillis() - startedAtMillis; + assertTrue(elapsedMillis < 1_000L, () -> "initial failure took " + elapsedMillis + "ms"); + assertTrue(error.getMessage().contains("auth failed"), error::toString); + } + } + + @Test + void registryRetiresOnlyUnusedPools() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + String url = h2Url("registry_retire"); + JdbcConnectionPoolRegistry.PoolSettings settings = new JdbcConnectionPoolRegistry.PoolSettings( + 1, + 0, + 2_000L, + 1_000L, + 10_000L, + 30_000L, + 0L + ); + try (JdbcConnectionPoolRegistry registry = new JdbcConnectionPoolRegistry(settings)) { + JdbcConnectionPoolRegistry.Lease lease = registry.borrow( + "retired-identity", + () -> openH2(url, physicalOpens) + ); + registry.retireUnusedPools(); + assertEquals(1, registry.poolCount()); + + lease.close(); + registry.retireUnusedPools(); + assertEquals(0, registry.poolCount()); + + try (JdbcConnectionPoolRegistry.Lease ignored = registry.borrow( + "retired-identity", + () -> openH2(url, physicalOpens) + )) { + } + assertEquals(2, physicalOpens.get()); + } + } + + @Test + void ordinaryMultiSessionRequestsReuseOnePhysicalConnection() { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("multi_reuse"); + try (MultiSessionJsonRpcServer server = server(url, physicalOpens, 4)) { + for (int index = 0; index < 25; index++) { + String sessionId = "session-" + index; + openSession(server, requestIds, sessionId); + JsonObject result = query(server, requestIds, sessionId, "SELECT 1", null); + assertEquals(1, result.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsInt()); + closeSession(server, requestIds, sessionId); + } + assertEquals(1, physicalOpens.get()); + } + } + + @Test + void poolingCanBeDisabledForDriverCompatibility() { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("pool_disabled"); + JdbcConnectionPoolRegistry.PoolSettings disabled = new JdbcConnectionPoolRegistry.PoolSettings( + false, + 4, + 0, + 2_000L, + 1_000L, + 10_000L, + 30_000L, + 60_000L + ); + try (MultiSessionJsonRpcServer server = new MultiSessionJsonRpcServer( + () -> new H2TestAgent(url, physicalOpens), + disabled + )) { + openSession(server, requestIds, "legacy-a"); + openSession(server, requestIds, "legacy-b"); + assertEquals(2, physicalOpens.get()); + } + } + + @Test + void pagedCursorPinsConnectionUntilItIsClosed() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("cursor_pin"); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try (MultiSessionJsonRpcServer server = server(url, physicalOpens, 1)) { + openSession(server, requestIds, "cursor-owner"); + openSession(server, requestIds, "waiting-session"); + + JsonObject pageParams = sessionParams("cursor-owner"); + pageParams.addProperty("sql", "SELECT X FROM SYSTEM_RANGE(1, 3)"); + pageParams.addProperty("pageSize", 1); + JsonObject firstPage = result(request( + server, + requestIds, + AgentProtocol.METHOD_EXECUTE_QUERY_PAGE, + pageParams + )); + assertTrue(firstPage.get("has_more").getAsBoolean()); + String querySessionId = firstPage.get("session_id").getAsString(); + + Future waiting = worker.submit( + () -> query(server, requestIds, "waiting-session", "SELECT 2", null) + ); + Thread.sleep(200L); + assertFalse(waiting.isDone()); + assertEquals(1, physicalOpens.get()); + + JsonObject closeParams = sessionParams("cursor-owner"); + closeParams.addProperty("sessionId", querySessionId); + assertTrue(request( + server, + requestIds, + AgentProtocol.METHOD_CLOSE_QUERY_SESSION, + closeParams + ).get("result").getAsBoolean()); + + assertEquals( + 2, + waiting.get(2, TimeUnit.SECONDS) + .getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsInt() + ); + assertEquals(1, physicalOpens.get()); + } finally { + worker.shutdownNow(); + } + } + + @Test + void maintenanceExpiresAbandonedCursorAndReturnsItsConnection() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("cursor_expire"); + try (MultiSessionJsonRpcServer server = server(url, physicalOpens, 1)) { + openSession(server, requestIds, "abandoned-cursor"); + openSession(server, requestIds, "next-session"); + + JsonObject pageParams = sessionParams("abandoned-cursor"); + pageParams.addProperty("sql", "SELECT X FROM SYSTEM_RANGE(1, 3)"); + pageParams.addProperty("pageSize", 1); + JsonObject firstPage = result(request( + server, + requestIds, + AgentProtocol.METHOD_EXECUTE_QUERY_PAGE, + pageParams + )); + assertTrue(firstPage.get("has_more").getAsBoolean()); + + server.runMaintenance(System.currentTimeMillis() + 1_000L, 0L); + JsonObject result = query(server, requestIds, "next-session", "SELECT 9", null); + + assertEquals(9, result.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsInt()); + assertEquals(1, physicalOpens.get()); + } + } + + @Test + void maintenanceSkipsBusySessionWithoutWaiting() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + CountDownLatch requestStarted = new CountDownLatch(1); + CountDownLatch releaseRequest = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + String url = h2Url("maintenance_busy_session"); + try (MultiSessionJsonRpcServer server = new MultiSessionJsonRpcServer( + () -> new H2TestAgent(url, physicalOpens) { + @Override + public List listDatabases() { + requestStarted.countDown(); + try { + releaseRequest.await(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new RuntimeException(error); + } + return Collections.emptyList(); + } + }, + poolSettings(2) + )) { + openSession(server, requestIds, "busy-session"); + Future busyRequest = workers.submit(() -> request( + server, + requestIds, + AgentProtocol.METHOD_LIST_DATABASES, + sessionParams("busy-session") + )); + assertTrue(requestStarted.await(2, TimeUnit.SECONDS)); + + Future maintenance = workers.submit(() -> server.runMaintenance()); + maintenance.get(500, TimeUnit.MILLISECONDS); + assertFalse(busyRequest.isDone()); + + releaseRequest.countDown(); + busyRequest.get(2, TimeUnit.SECONDS); + } finally { + releaseRequest.countDown(); + workers.shutdownNow(); + } + } + + @Test + void independentSessionsExecuteConcurrentlyThroughSharedPool() throws Exception { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + CountDownLatch bothRequestsStarted = new CountDownLatch(2); + CountDownLatch releaseRequests = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + String url = h2Url("independent_sessions"); + try (MultiSessionJsonRpcServer server = new MultiSessionJsonRpcServer( + () -> new H2TestAgent(url, physicalOpens) { + @Override + public List listDatabases() { + bothRequestsStarted.countDown(); + try { + releaseRequests.await(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new RuntimeException(error); + } + return Collections.emptyList(); + } + }, + poolSettings(2) + )) { + openSession(server, requestIds, "metadata-a"); + openSession(server, requestIds, "metadata-b"); + + Future first = workers.submit(() -> request( + server, + requestIds, + AgentProtocol.METHOD_LIST_DATABASES, + sessionParams("metadata-a") + )); + Future second = workers.submit(() -> request( + server, + requestIds, + AgentProtocol.METHOD_LIST_DATABASES, + sessionParams("metadata-b") + )); + + assertTrue(bothRequestsStarted.await(2, TimeUnit.SECONDS)); + assertFalse(first.isDone()); + assertFalse(second.isDone()); + assertEquals(2, physicalOpens.get()); + + releaseRequests.countDown(); + first.get(2, TimeUnit.SECONDS); + second.get(2, TimeUnit.SECONDS); + } finally { + releaseRequests.countDown(); + workers.shutdownNow(); + } + } + + @Test + void statefulSqlKeepsSessionAffinityAndEvictsOnClose() { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("stateful_evict"); + try (MultiSessionJsonRpcServer server = server(url, physicalOpens, 2)) { + openSession(server, requestIds, "stateful"); + query(server, requestIds, "stateful", "CREATE LOCAL TEMPORARY TABLE TEMP_SESSION(ID INT)", null); + query(server, requestIds, "stateful", "INSERT INTO TEMP_SESSION VALUES (7)", null); + JsonObject sameSession = query(server, requestIds, "stateful", "SELECT ID FROM TEMP_SESSION", null); + assertEquals(7, sameSession.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsInt()); + closeSession(server, requestIds, "stateful"); + + openSession(server, requestIds, "fresh"); + JsonObject freshSession = query( + server, + requestIds, + "fresh", + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'TEMP_SESSION'", + null + ); + assertEquals(0, freshSession.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsInt()); + assertEquals(2, physicalOpens.get()); + } + } + + @Test + void schemaContextDoesNotLeakAcrossLogicalSessions() { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("schema_reset"); + try (MultiSessionJsonRpcServer server = server(url, physicalOpens, 2)) { + openSession(server, requestIds, "schema-owner"); + query(server, requestIds, "schema-owner", "CREATE SCHEMA IF NOT EXISTS APP", null); + JsonObject appResult = query(server, requestIds, "schema-owner", "SELECT CURRENT_SCHEMA", "APP"); + assertEquals("APP", appResult.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsString()); + closeSession(server, requestIds, "schema-owner"); + + openSession(server, requestIds, "fresh-schema"); + JsonObject publicResult = query(server, requestIds, "fresh-schema", "SELECT CURRENT_SCHEMA", null); + assertEquals("PUBLIC", publicResult.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsString()); + assertEquals(1, physicalOpens.get()); + } + } + + @Test + void explicitSessionSchemaIsPreservedUntilStatefulSessionCloses() { + AtomicInteger physicalOpens = new AtomicInteger(); + AtomicInteger requestIds = new AtomicInteger(); + String url = h2Url("stateful_schema"); + try (MultiSessionJsonRpcServer server = server(url, physicalOpens, 2)) { + openSession(server, requestIds, "stateful-schema"); + query(server, requestIds, "stateful-schema", "CREATE SCHEMA IF NOT EXISTS APP", null); + query(server, requestIds, "stateful-schema", "CREATE SCHEMA IF NOT EXISTS OTHER", null); + query(server, requestIds, "stateful-schema", "SET SCHEMA OTHER", "APP"); + + JsonObject retained = query(server, requestIds, "stateful-schema", "SELECT CURRENT_SCHEMA", null); + assertEquals("OTHER", retained.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsString()); + closeSession(server, requestIds, "stateful-schema"); + + openSession(server, requestIds, "fresh-after-stateful"); + JsonObject fresh = query(server, requestIds, "fresh-after-stateful", "SELECT CURRENT_SCHEMA", null); + assertEquals("PUBLIC", fresh.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsString()); + assertEquals(2, physicalOpens.get()); + } + } + + @Test + void affinityDetectionCoversCommonSessionStateStatements() { + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("SET search_path TO app")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("-- comment\nUSE sales")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("CREATE TEMPORARY TABLE t(id int)")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("CREATE TABLE #session_rows(id int)")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("SELECT * INTO #session_rows FROM users")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity( + "WITH source_rows AS (SELECT * FROM users) SELECT * INTO #session_rows FROM source_rows" + )); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity( + "WITH source_rows AS (SELECT * FROM users) SELECT * INTO [#session_rows] FROM source_rows" + )); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("CREATE MULTISET VOLATILE TABLE vt(id int)")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("DATABASE analytics")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("UNSET current_namespace")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("ADD JAR '/tmp/session-udf.jar'")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("SELECT 1; /* switch */ SET ROLE app_user")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("SELECT 1; # switch\nSET sql_mode = 'ANSI'")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE */")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("/*M!100100 SET @OLD_TIME_ZONE=@@TIME_ZONE */")); + assertTrue(JdbcConnectionAffinity.requiresSessionAffinity("SELECT GET_LOCK('dbx', 5)")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("SELECT * FROM users")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("CREATE TABLE users(id int)")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("SELECT 'SET ROLE app_user; CREATE TEMP TABLE t(id int)'")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("SELECT $$SET ROLE app_user$$")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("SELECT '# SET ROLE app_user'")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("SELECT '#session_rows'")); + assertFalse(JdbcConnectionAffinity.requiresSessionAffinity("SELECT payload #>> '{path}' FROM events")); + } + + private static MultiSessionJsonRpcServer server(String url, AtomicInteger physicalOpens, int maximumPoolSize) { + return new MultiSessionJsonRpcServer( + () -> new H2TestAgent(url, physicalOpens), + poolSettings(maximumPoolSize) + ); + } + + private static void openSession( + MultiSessionJsonRpcServer server, + AtomicInteger requestIds, + String sessionId + ) { + JsonObject params = sessionParams(sessionId); + params.addProperty("database", "pooling"); + params.addProperty("username", "sa"); + params.addProperty("password", ""); + result(request(server, requestIds, AgentProtocol.METHOD_OPEN_SESSION, params)); + } + + private static void closeSession( + MultiSessionJsonRpcServer server, + AtomicInteger requestIds, + String sessionId + ) { + result(request( + server, + requestIds, + AgentProtocol.METHOD_CLOSE_SESSION, + sessionParams(sessionId) + )); + } + + private static JsonObject query( + MultiSessionJsonRpcServer server, + AtomicInteger requestIds, + String sessionId, + String sql, + String schema + ) { + JsonObject params = sessionParams(sessionId); + params.addProperty("sql", sql); + if (schema != null) { + params.addProperty("schema", schema); + } + return result(request(server, requestIds, AgentProtocol.METHOD_EXECUTE_QUERY, params)); + } + + private static JsonObject request( + MultiSessionJsonRpcServer server, + AtomicInteger requestIds, + String method, + JsonObject params + ) { + JsonObject request = new JsonObject(); + request.addProperty("jsonrpc", "2.0"); + request.addProperty("id", requestIds.incrementAndGet()); + request.addProperty("method", method); + request.add("params", params); + JsonObject response = JsonParser.parseString(server.handleRequest(GSON.toJson(request))).getAsJsonObject(); + assertFalse(response.has("error"), () -> response.toString()); + return response; + } + + private static JsonObject result(JsonObject response) { + assertNotNull(response.get("result")); + return response.getAsJsonObject("result"); + } + + private static JsonObject sessionParams(String sessionId) { + JsonObject params = new JsonObject(); + params.addProperty("agentSessionId", sessionId); + return params; + } + + private static JdbcConnectionPoolRegistry.PoolSettings poolSettings(int maximumPoolSize) { + return new JdbcConnectionPoolRegistry.PoolSettings( + maximumPoolSize, + 0, + 2_000L, + 1_000L, + 10_000L, + 30_000L, + 60_000L + ); + } + + private static Connection openH2(String url, AtomicInteger physicalOpens) throws Exception { + physicalOpens.incrementAndGet(); + return DriverManager.getConnection(url, "sa", ""); + } + + private static String h2Url(String prefix) { + return "jdbc:h2:mem:" + prefix + "_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1"; + } + + private static class H2TestAgent extends AbstractJdbcAgent { + private final String url; + private final AtomicInteger physicalOpens; + + private H2TestAgent(String url, AtomicInteger physicalOpens) { + this.url = url; + this.physicalOpens = physicalOpens; + } + + @Override + protected String driverClass() { + return "org.h2.Driver"; + } + + @Override + protected String buildJdbcUrl(ConnectParams params) { + return url; + } + + @Override + protected Connection openConnection(ConnectParams params) throws Exception { + return openH2(url, physicalOpens); + } + + @Override + public List listDatabases() { + return Collections.emptyList(); + } + + @Override + public List listSchemas() { + return Collections.emptyList(); + } + + @Override + public List listTables(String schema) { + return Collections.emptyList(); + } + + @Override + public List getColumns(String schema, String table) { + return Collections.emptyList(); + } + + @Override + public List listIndexes(String schema, String table) { + return Collections.emptyList(); + } + + @Override + public List listForeignKeys(String schema, String table) { + return Collections.emptyList(); + } + + @Override + public List listTriggers(String schema, String table) { + return Collections.emptyList(); + } + } +} diff --git a/agents/docs/agent-protocol-v2.md b/agents/docs/agent-protocol-v2.md index 87b6da4690..bd7da125eb 100644 --- a/agents/docs/agent-protocol-v2.md +++ b/agents/docs/agent-protocol-v2.md @@ -8,7 +8,7 @@ Protocol v2 allows one Agent process to serve multiple isolated database session - Every connection-scoped RPC contains `agentSessionId`. - `validate_session` validates and, where supported, reconnects only that session. - `cancel_session` cancels active statements and cursor fetches for only that session; other sessions in the runtime continue normally. -- `close_session` closes the session connection, query cursors, and table-read cursors without affecting other sessions. +- `close_session` closes the session resources, query cursors, and table-read cursors without affecting other sessions. - `shutdown` closes all sessions and terminates the runtime. `agentSessionId` identifies a logical database connection. Existing `sessionId` fields remain pagination cursor identifiers and must not be used as logical connection identifiers. @@ -17,6 +17,10 @@ Protocol v2 allows one Agent process to serve multiple isolated database session Requests for different sessions may execute concurrently. Requests for the same session are serialized because connection state, transactions, schema changes, and driver connections are not generally safe for concurrent use. JSON-RPC responses may be returned out of order and are correlated by request `id`. +All Java JDBC runtimes share HikariCP pools by immutable connection identity through `AbstractJdbcAgent`. Stateless requests borrow and return connections. Paged cursors and explicit session-state SQL pin a connection to the logical session, and stateful connections are evicted when that session closes so state cannot leak across sessions. Custom URL construction, transport fallback, connection initialization, and native-driver access remain behind shared lifecycle hooks. + +DBX uses unique short-lived logical sessions for independent Agent metadata tasks so they do not queue behind editor execution. These sessions close after completion, and cancellation-safe cleanup closes them when the caller is dropped. + ## Runtime compatibility Runtime reuse keys include the Agent driver key, executable or JAR path, launch arguments, working directory, JRE selection, JVM options, classpath-affecting options, and native executable version boundary. Host, account, schema, and credentials belong to sessions and are not part of the runtime key. @@ -29,6 +33,6 @@ A runtime accepts at most 256 logical sessions. Closing the final session starts ## Driver author guidance -Use `MultiSessionJsonRpcServer(YourAgent::new)` for Java SQL Agents so each logical session receives a new `DatabaseAgent` and JDBC connection. Do not store connection, statement, cursor, transaction, or schema state in static mutable fields. Use the session execution context for paged query resources. Native Agents must provide equivalent per-session state and synchronized stdout writes. +Use `MultiSessionJsonRpcServer(YourAgent::new)` for Java SQL Agents so each logical session receives a new `DatabaseAgent` with isolated connection state. The shared runtime owns the physical JDBC pools. Do not store connection, statement, cursor, transaction, or schema state in static mutable fields. Use the session execution context for paged query resources. Native Agents must provide equivalent per-session state and synchronized stdout writes. The Xugu native Agent keeps one database connection per logical session plus one shared control connection per database endpoint. Because `go-xugu-driver` does not interrupt network reads through `context.Context`, cancellation records the server-side session ID and calls `DBMS_DBA.KILL_SESSION_TRANS` through the shared control connection. diff --git a/agents/docs/examples/jdbc-agent-template/build.gradle b/agents/docs/examples/jdbc-agent-template/build.gradle index 3443ed7e21..4c64b5f309 100644 --- a/agents/docs/examples/jdbc-agent-template/build.gradle +++ b/agents/docs/examples/jdbc-agent-template/build.gradle @@ -28,6 +28,8 @@ configure([project(':common'), project(':test-support')]) { dependencies { implementation project(':common') + implementation 'com.zaxxer:HikariCP:7.0.2' + runtimeOnly 'org.slf4j:slf4j-nop:2.0.17' // Bundled driver example: // implementation 'com.example:example-jdbc:1.2.3' diff --git a/agents/docs/examples/jdbc-agent-template/src/main/java/com/dbx/agent/template/TemplateAgent.java b/agents/docs/examples/jdbc-agent-template/src/main/java/com/dbx/agent/template/TemplateAgent.java index bca4d45a08..20b9229fa1 100644 --- a/agents/docs/examples/jdbc-agent-template/src/main/java/com/dbx/agent/template/TemplateAgent.java +++ b/agents/docs/examples/jdbc-agent-template/src/main/java/com/dbx/agent/template/TemplateAgent.java @@ -3,7 +3,7 @@ package com.dbx.agent.template; import com.dbx.agent.ConfiguredJdbcAgent; import com.dbx.agent.JdbcAgentProfile; import com.dbx.agent.JdbcIdentifiers; -import com.dbx.agent.JsonRpcServer; +import com.dbx.agent.MultiSessionJsonRpcServer; public final class TemplateAgent extends ConfiguredJdbcAgent { public static final JdbcAgentProfile TEMPLATE_PROFILE = new JdbcAgentProfile( @@ -22,6 +22,6 @@ public final class TemplateAgent extends ConfiguredJdbcAgent { } public static void main(String[] args) { - new JsonRpcServer(new TemplateAgent()).run(); + new MultiSessionJsonRpcServer(TemplateAgent::new).run(); } } diff --git a/agents/drivers/access/src/main/java/com/dbx/agent/access/AccessAgent.java b/agents/drivers/access/src/main/java/com/dbx/agent/access/AccessAgent.java index a60ebe7c75..c723571ecf 100644 --- a/agents/drivers/access/src/main/java/com/dbx/agent/access/AccessAgent.java +++ b/agents/drivers/access/src/main/java/com/dbx/agent/access/AccessAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.access; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -29,39 +29,32 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -public final class AccessAgent extends BaseDatabaseAgent { +public final class AccessAgent extends AbstractJdbcAgent { private static final String ENCRYPTED_ACCESS_OPENER = EncryptedAccessOpener.class.getName(); - private Connection connection; private String databaseFile = ""; @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "net.ucanaccess.jdbc.UcanaccessDriver"; } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - Class.forName("net.ucanaccess.jdbc.UcanaccessDriver"); - String file = databasePath(params); - String url = jdbcUrl(params, true); - connection = DriverManager.getConnection(url, params.getUsername(), params.getPassword()); - databaseFile = file; - }); + protected String buildJdbcUrl(ConnectParams params) { + return jdbcUrl(params, true); } @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - Class.forName("net.ucanaccess.jdbc.UcanaccessDriver"); - String file = databasePath(params); - if (file.isBlank() || !Files.exists(Path.of(file))) { - return false; - } - try (Connection conn = DriverManager.getConnection(jdbcUrl(params, false), params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); + protected Connection openTestConnection(ConnectParams params) throws Exception { + String file = databasePath(params); + if (file.isBlank() || !Files.exists(Path.of(file))) { + return null; + } + return DriverManager.getConnection(jdbcUrl(params, false), params.getUsername(), params.getPassword()); + } + + @Override + protected void afterConnect(ConnectParams params, Connection connection) { + databaseFile = databasePath(params); } @Override @@ -234,16 +227,6 @@ public final class AccessAgent extends BaseDatabaseAgent { return ""; } - @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - private Set primaryKeyColumns(DatabaseMetaData meta, String table) throws Exception { Set result = new LinkedHashSet<>(); try (ResultSet rs = meta.getPrimaryKeys(null, null, table)) { diff --git a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java index 03cd233b6b..455fe90b8b 100644 --- a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java +++ b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.dameng; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -40,7 +40,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -public final class DamengAgent extends BaseDatabaseAgent { +public final class DamengAgent extends AbstractJdbcAgent { private static final String AGENT_VERSION = "9999.06.04.1-fix-default"; private static final String DAMENG_CLASSIFIED_OBJECT_TYPE_SQL = "CASE WHEN o.OBJECT_TYPE = 'MATERIALIZED VIEW' OR (o.OBJECT_TYPE = 'VIEW' AND mv.MVIEW_NAME IS NOT NULL) " @@ -72,35 +72,33 @@ public final class DamengAgent extends BaseDatabaseAgent { ON schema_object.OBJECT_ID = m.SCHID AND schema_object.OBJECT_TYPE = 'SCH' ) mv ON mv.OWNER = o.OWNER AND mv.MVIEW_NAME = o.OBJECT_NAME """.stripIndent().trim(); - private Connection connection; private String connectedUsername; @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "dm.jdbc.driver.DmDriver"; } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - withSuppressedStdout(() -> { - Class.forName("dm.jdbc.driver.DmDriver"); - connection = DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword()); - connectedUsername = params.getUsername(); - }); - }); + protected String buildJdbcUrl(ConnectParams params) { + return buildUrl(params); } @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - return withSuppressedStdout(() -> { - Class.forName("dm.jdbc.driver.DmDriver"); - try (Connection conn = DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); - }); + protected void loadDriver(ConnectParams params) throws Exception { + withSuppressedStdout(() -> super.loadDriver(params)); + } + + @Override + protected Connection openConnection(ConnectParams params) throws Exception { + return withSuppressedStdout( + () -> DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword()) + ); + } + + @Override + protected void afterConnect(ConnectParams params, Connection connection) { + connectedUsername = params.getUsername(); } /** @@ -883,7 +881,7 @@ public final class DamengAgent extends BaseDatabaseAgent { options.getMaxRows(), options.getFetchSize(), options.getTimeoutSecs(), - this::stringResultValue + this::resultValue ); } @@ -979,7 +977,7 @@ public final class DamengAgent extends BaseDatabaseAgent { schema, this::setSchemaSQL, options, - this::stringResultValue + this::resultValue ); } @@ -991,7 +989,7 @@ public final class DamengAgent extends BaseDatabaseAgent { schema, this::setSchemaSQL, options, - this::stringResultValue + this::resultValue ); } @@ -1001,16 +999,7 @@ public final class DamengAgent extends BaseDatabaseAgent { } @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - - private Object stringResultValue(ResultSet rs, int index, int sqlType) { + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value = switch (sqlType) { case Types.BIGINT -> rs.getLong(index); @@ -1464,9 +1453,10 @@ public final class DamengAgent extends BaseDatabaseAgent { } try { Class dmConnClass = Class.forName("dm.jdbc.driver.DmdbConnection"); - if (dmConnClass.isInstance(conn)) { + Object dmConnection = unwrapConnection(conn, dmConnClass); + if (dmConnection != null) { Method m = dmConnClass.getMethod("getExplainInfo", Statement.class); - planText = (String) m.invoke(dmConnClass.cast(conn), stmt); + planText = (String) m.invoke(dmConnection, stmt); } } catch (Exception ignored) {} } finally { @@ -1479,9 +1469,10 @@ public final class DamengAgent extends BaseDatabaseAgent { } else { try { Class dmConnClass = Class.forName("dm.jdbc.driver.DmdbConnection"); - if (dmConnClass.isInstance(conn)) { + Object dmConnection = unwrapConnection(conn, dmConnClass); + if (dmConnection != null) { Method m = dmConnClass.getMethod("getExplainInfo", String.class); - planText = (String) m.invoke(dmConnClass.cast(conn), sql); + planText = (String) m.invoke(dmConnection, sql); } } catch (Exception ignored) {} } diff --git a/agents/drivers/db2/src/main/java/com/dbx/agent/db2/Db2Agent.java b/agents/drivers/db2/src/main/java/com/dbx/agent/db2/Db2Agent.java index f43c4fdff6..11f468e2fa 100644 --- a/agents/drivers/db2/src/main/java/com/dbx/agent/db2/Db2Agent.java +++ b/agents/drivers/db2/src/main/java/com/dbx/agent/db2/Db2Agent.java @@ -1,6 +1,6 @@ package com.dbx.agent.db2; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -17,8 +17,6 @@ import com.dbx.agent.ObjectSource; import com.dbx.agent.QueryResult; import com.dbx.agent.TableInfo; import com.dbx.agent.TriggerInfo; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.ArrayList; @@ -26,18 +24,16 @@ import java.util.List; import java.util.Locale; import java.util.Set; -public final class Db2Agent extends BaseDatabaseAgent { +public final class Db2Agent extends AbstractJdbcAgent { private static final Set NUMERIC_PRECISION_TYPES = Set.of( "DECIMAL", "NUMERIC", "INTEGER", "SMALLINT", "BIGINT", "REAL", "DOUBLE", "FLOAT" ); private static final Set NUMERIC_SCALE_TYPES = Set.of("DECIMAL", "NUMERIC"); private static final Set CHARACTER_LENGTH_TYPES = Set.of("VARCHAR", "CHAR", "CLOB", "GRAPHIC", "VARGRAPHIC"); - private Connection connection; - @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "com.ibm.db2.jcc.DB2Driver"; } @Override @@ -46,21 +42,8 @@ public final class Db2Agent extends BaseDatabaseAgent { } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - Class.forName("com.ibm.db2.jcc.DB2Driver"); - connection = DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword()); - }); - } - - @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - Class.forName("com.ibm.db2.jcc.DB2Driver"); - try (Connection conn = DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); + protected String buildJdbcUrl(ConnectParams params) { + return buildUrl(params); } @Override @@ -395,21 +378,12 @@ public final class Db2Agent extends BaseDatabaseAgent { options.getMaxRows(), options.getFetchSize(), options.getTimeoutSecs(), - this::stringResultValue + this::resultValue ); } @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - - private Object stringResultValue(ResultSet rs, int index, int sqlType) { + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value = rs.getObject(index); return rs.wasNull() ? null : value == null ? null : value.toString(); diff --git a/agents/drivers/gbase8s/src/main/java/com/dbx/agent/gbase8s/Gbase8sAgent.java b/agents/drivers/gbase8s/src/main/java/com/dbx/agent/gbase8s/Gbase8sAgent.java index f64acdad18..e547406e55 100644 --- a/agents/drivers/gbase8s/src/main/java/com/dbx/agent/gbase8s/Gbase8sAgent.java +++ b/agents/drivers/gbase8s/src/main/java/com/dbx/agent/gbase8s/Gbase8sAgent.java @@ -90,9 +90,8 @@ public final class Gbase8sAgent extends ConfiguredJdbcAgent { } @Override - public void disconnect() { + protected void afterDisconnect() { clearMetadataCache(); - super.disconnect(); } @Override diff --git a/agents/drivers/goldendb/src/main/java/com/dbx/agent/goldendb/GoldendbAgent.java b/agents/drivers/goldendb/src/main/java/com/dbx/agent/goldendb/GoldendbAgent.java index 3df05ecad4..80360c4bcc 100644 --- a/agents/drivers/goldendb/src/main/java/com/dbx/agent/goldendb/GoldendbAgent.java +++ b/agents/drivers/goldendb/src/main/java/com/dbx/agent/goldendb/GoldendbAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.goldendb; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -17,8 +17,6 @@ import com.dbx.agent.ObjectSource; import com.dbx.agent.QueryResult; import com.dbx.agent.TableInfo; import com.dbx.agent.TriggerInfo; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Types; import java.util.ArrayList; @@ -30,30 +28,15 @@ import java.util.Set; import java.util.TreeMap; import java.util.HashSet; -public final class GoldendbAgent extends BaseDatabaseAgent { - private Connection connection; - +public final class GoldendbAgent extends AbstractJdbcAgent { @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "com.mysql.cj.jdbc.Driver"; } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - Class.forName("com.mysql.cj.jdbc.Driver"); - connection = DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword()); - }); - } - - @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - Class.forName("com.mysql.cj.jdbc.Driver"); - try (Connection conn = DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); + protected String buildJdbcUrl(ConnectParams params) { + return buildUrl(params); } @Override @@ -424,16 +407,7 @@ public final class GoldendbAgent extends BaseDatabaseAgent { } @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - - private Object resultValue(ResultSet rs, int index, int sqlType) { + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value; switch (sqlType) { diff --git a/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java b/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java index 03abc6d71c..2001ccc32f 100644 --- a/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java +++ b/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java @@ -144,9 +144,7 @@ public final class HiveAgent extends AbstractJdbcAgent { } private void useSchema(String schema) throws Exception { - try (java.sql.Statement stmt = requireConnected().createStatement()) { - stmt.execute(setSchemaSQL(schema)); - } + applySchemaContext(requireConnected(), schema); } private List getColumnsFromDescribe(String schema, String table) throws Exception { diff --git a/agents/drivers/informix/src/main/java/com/dbx/agent/informix/InformixAgent.java b/agents/drivers/informix/src/main/java/com/dbx/agent/informix/InformixAgent.java index 5850d1266d..5c175f6280 100644 --- a/agents/drivers/informix/src/main/java/com/dbx/agent/informix/InformixAgent.java +++ b/agents/drivers/informix/src/main/java/com/dbx/agent/informix/InformixAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.informix; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -28,15 +28,18 @@ import java.util.List; import java.util.Locale; import java.util.Set; -public final class InformixAgent extends BaseDatabaseAgent { - private Connection connection; - +public final class InformixAgent extends AbstractJdbcAgent { @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "com.informix.jdbc.IfxDriver"; } - public static String buildJdbcUrl(ConnectParams params) { + @Override + protected String buildJdbcUrl(ConnectParams params) { + return jdbcUrl(params); + } + + public static String jdbcUrl(ConnectParams params) { String rawUrlParams = params.getUrl_params(); String extraParams = rawUrlParams == null ? "" : trimEnd(trimStart(rawUrlParams.trim(), ':', ';'), ';'); String rawDatabase = params.getDatabase(); @@ -138,30 +141,16 @@ public final class InformixAgent extends BaseDatabaseAgent { } @Override - public void connect(ConnectParams params) { - String url = buildJdbcUrl(params); - uncheckedVoid(() -> { - Class.forName("com.informix.jdbc.IfxDriver"); - try { - connection = DriverManager.getConnection(url, params.getUsername(), params.getPassword()); - } catch (SQLException e) { - throw new SQLException( - "Informix connection failed.\nURL: " + url.replaceAll("//[^@]+@", "//***@") + "\nError: " + e.getMessage(), - e.getSQLState(), e.getErrorCode() - ); - } - }); - } - - @Override - public boolean testConnection(ConnectParams params) { - String url = buildJdbcUrl(params); - return unchecked(() -> { - Class.forName("com.informix.jdbc.IfxDriver"); - try (Connection conn = DriverManager.getConnection(url, params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); + protected Connection openConnection(ConnectParams params) throws SQLException { + String url = jdbcUrl(params); + try { + return DriverManager.getConnection(url, params.getUsername(), params.getPassword()); + } catch (SQLException error) { + throw new SQLException( + "Informix connection failed.\nURL: " + url.replaceAll("//[^@]+@", "//***@") + "\nError: " + error.getMessage(), + error.getSQLState(), error.getErrorCode() + ); + } } @Override @@ -467,7 +456,7 @@ public final class InformixAgent extends BaseDatabaseAgent { options.getMaxRows(), options.getFetchSize(), options.getTimeoutSecs(), - this::stringResultValue + this::resultValue ); } @@ -477,16 +466,7 @@ public final class InformixAgent extends BaseDatabaseAgent { } @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - - private Object stringResultValue(ResultSet rs, int index, int sqlType) { + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value = rs.getObject(index); return rs.wasNull() ? null : value == null ? null : value.toString(); diff --git a/agents/drivers/informix/src/test/java/com/dbx/agent/informix/InformixAgentTest.java b/agents/drivers/informix/src/test/java/com/dbx/agent/informix/InformixAgentTest.java index ea99fbba31..5bdc2ce509 100644 --- a/agents/drivers/informix/src/test/java/com/dbx/agent/informix/InformixAgentTest.java +++ b/agents/drivers/informix/src/test/java/com/dbx/agent/informix/InformixAgentTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.api.Test; class InformixAgentTest { @Test void buildsJdbcUrlWithExplicitInformixServerAndLocaleParameters() { - String url = InformixAgent.buildJdbcUrl( + String url = InformixAgent.jdbcUrl( new ConnectParams( "172.26.128.159", 20013, @@ -34,7 +34,7 @@ class InformixAgentTest { @Test void fallsBackToHostAsInformixServerWhenNoExplicitServerIsConfigured() { - String url = InformixAgent.buildJdbcUrl( + String url = InformixAgent.jdbcUrl( new ConnectParams( "informix-host", 9088, @@ -55,7 +55,7 @@ class InformixAgentTest { @Test void fallsBackToInformixServerNameWhenHostIsAnIpAddress() { - String url = InformixAgent.buildJdbcUrl( + String url = InformixAgent.jdbcUrl( new ConnectParams( "172.26.128.159", 20013, @@ -88,7 +88,7 @@ class InformixAgentTest { ); params.setInformix_server("ol_informix1410"); - String url = InformixAgent.buildJdbcUrl(params); + String url = InformixAgent.jdbcUrl(params); Assertions.assertEquals( "jdbc:informix-sqli://172.26.128.159:20013/testdb:INFORMIXSERVER=ol_informix1410;CLIENT_LOCALE=en_US.utf8;DB_LOCALE=en_US.utf8", @@ -98,7 +98,7 @@ class InformixAgentTest { @Test void usesSysmasterWhenDatabaseIsBlank() { - String url = InformixAgent.buildJdbcUrl( + String url = InformixAgent.jdbcUrl( new ConnectParams( "informix-host", 9088, diff --git a/agents/drivers/neo4j/src/main/java/com/dbx/agent/neo4j/Neo4jAgent.java b/agents/drivers/neo4j/src/main/java/com/dbx/agent/neo4j/Neo4jAgent.java index 5dd93ca7cd..9165351ed8 100644 --- a/agents/drivers/neo4j/src/main/java/com/dbx/agent/neo4j/Neo4jAgent.java +++ b/agents/drivers/neo4j/src/main/java/com/dbx/agent/neo4j/Neo4jAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.neo4j; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -16,7 +16,6 @@ import com.dbx.agent.TriggerInfo; import java.sql.Connection; import java.sql.DatabaseMetaData; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; @@ -27,12 +26,10 @@ import java.util.List; import java.util.Locale; import java.util.Set; -public class Neo4jAgent extends BaseDatabaseAgent { - private Connection connection; - +public class Neo4jAgent extends AbstractJdbcAgent { @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "org.neo4j.jdbc.Neo4jDriver"; } @Override @@ -41,23 +38,8 @@ public class Neo4jAgent extends BaseDatabaseAgent { } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - Class.forName("org.neo4j.jdbc.Neo4jDriver"); - String url = buildNeo4jUrl(params); - connection = DriverManager.getConnection(url, params.getUsername(), params.getPassword()); - }); - } - - @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - Class.forName("org.neo4j.jdbc.Neo4jDriver"); - String url = buildNeo4jUrl(params); - try (Connection conn = DriverManager.getConnection(url, params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); + protected String buildJdbcUrl(ConnectParams params) { + return buildNeo4jUrl(params); } private static String buildNeo4jUrl(ConnectParams params) { @@ -287,21 +269,12 @@ public class Neo4jAgent extends BaseDatabaseAgent { options.getMaxRows(), options.getFetchSize(), options.getTimeoutSecs(), - this::stringResultValue + this::resultValue ); } @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - - private Object stringResultValue(ResultSet rs, int index, int sqlType) { + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value = rs.getObject(index); return rs.wasNull() ? null : value == null ? null : value.toString(); diff --git a/agents/drivers/neo4j/src/test/java/com/dbx/agent/neo4j/Neo4jAgentTest.java b/agents/drivers/neo4j/src/test/java/com/dbx/agent/neo4j/Neo4jAgentTest.java index 0573ea3d69..1f3f87df62 100644 --- a/agents/drivers/neo4j/src/test/java/com/dbx/agent/neo4j/Neo4jAgentTest.java +++ b/agents/drivers/neo4j/src/test/java/com/dbx/agent/neo4j/Neo4jAgentTest.java @@ -1,5 +1,6 @@ package com.dbx.agent.neo4j; +import com.dbx.agent.ConnectParams; import com.dbx.agent.QueryResult; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Proxy; @@ -18,9 +19,8 @@ import org.junit.jupiter.api.Test; class Neo4jAgentTest { @Test void executesCypherWritesWithReturnThroughStatementExecute() throws Exception { - Neo4jAgent agent = new Neo4jAgent(); List calls = new ArrayList<>(); - setConnectionForTest(agent, fakeConnection(calls)); + Neo4jAgent agent = connectedAgent(fakeConnection(calls)); QueryResult result = agent.executeQuery("CREATE (n:Person {name: 'Ada'}) RETURN n", null); @@ -35,9 +35,8 @@ class Neo4jAgentTest { @Test void executesTransactionsThroughStatementExecute() throws Exception { - Neo4jAgent agent = new Neo4jAgent(); List calls = new ArrayList<>(); - setConnectionForTest(agent, fakeConnection(calls)); + Neo4jAgent agent = connectedAgent(fakeConnection(calls)); QueryResult result = agent.executeTransaction( Arrays.asList( @@ -55,10 +54,19 @@ class Neo4jAgentTest { Assertions.assertFalse(calls.contains("executeUpdate")); } - private static void setConnectionForTest(Neo4jAgent agent, Connection connection) throws Exception { - java.lang.reflect.Field field = Neo4jAgent.class.getDeclaredField("connection"); - field.setAccessible(true); - field.set(agent, connection); + private static Neo4jAgent connectedAgent(Connection connection) { + Neo4jAgent agent = new Neo4jAgent() { + @Override + protected void loadDriver(ConnectParams params) { + } + + @Override + protected Connection openConnection(ConnectParams params) { + return connection; + } + }; + agent.connect(new ConnectParams()); + return agent; } private static Connection fakeConnection(List calls) { diff --git a/agents/drivers/oceanbase-oracle/src/main/java/com/dbx/agent/oceanbaseoracle/OceanBaseOracleAgent.java b/agents/drivers/oceanbase-oracle/src/main/java/com/dbx/agent/oceanbaseoracle/OceanBaseOracleAgent.java index 0170de552a..30a7b2b3b0 100644 --- a/agents/drivers/oceanbase-oracle/src/main/java/com/dbx/agent/oceanbaseoracle/OceanBaseOracleAgent.java +++ b/agents/drivers/oceanbase-oracle/src/main/java/com/dbx/agent/oceanbaseoracle/OceanBaseOracleAgent.java @@ -43,6 +43,7 @@ public final class OceanBaseOracleAgent extends ConfiguredJdbcAgent { "REMOTE_SCHEDULER_AGENT", "PDBADMIN", "DGPDB_INT", "OPS$ORACLE", "GGSYS", "FLOWS_FILES", "APEX_PUBLIC_USER", "GSMROOTUSER", "SYSRAC" ); + private boolean queryTimeoutChanged; public static final JdbcAgentProfile OCEANBASE_ORACLE_PROFILE = new JdbcAgentProfile( "com.oceanbase.jdbc.Driver", @@ -77,6 +78,18 @@ public final class OceanBaseOracleAgent extends ConfiguredJdbcAgent { // session variable, so synchronize both limits before every execution. try (var stmt = connection.createStatement()) { stmt.execute(queryTimeoutSql(timeoutSecs)); + queryTimeoutChanged = true; + } + } + + @Override + protected void beforePooledConnectionReturn(Connection connection) throws SQLException { + if (!queryTimeoutChanged) { + return; + } + try (var stmt = connection.createStatement()) { + stmt.execute(queryTimeoutSql(0)); + queryTimeoutChanged = false; } } diff --git a/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java b/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java index 35acd03b25..e134b680aa 100644 --- a/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java +++ b/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java @@ -78,6 +78,11 @@ public final class SparkAgent extends AbstractJdbcAgent { protected void afterConnect(ConnectParams params, Connection connection) throws Exception { String catalog = extractCatalogParam(params.getUrl_params()); configuredCatalog = catalog; + } + + @Override + protected void afterPhysicalConnect(ConnectParams params, Connection connection) throws Exception { + String catalog = extractCatalogParam(params.getUrl_params()); if (catalog != null && !catalog.isEmpty()) { try (java.sql.Statement stmt = connection.createStatement()) { stmt.execute("USE " + JdbcIdentifiers.INSTANCE.backtick(catalog)); @@ -224,9 +229,7 @@ public final class SparkAgent extends AbstractJdbcAgent { } private void useSchema(String schema) throws Exception { - try (java.sql.Statement stmt = requireConnected().createStatement()) { - stmt.execute(setSchemaSQL(schema)); - } + applySchemaContext(requireConnected(), schema); } private List getColumnsFromDescribe(String schema, String table) throws Exception { diff --git a/agents/drivers/sundb/src/main/java/com/dbx/agent/sundb/SundbAgent.java b/agents/drivers/sundb/src/main/java/com/dbx/agent/sundb/SundbAgent.java index ec462801d1..af21947170 100644 --- a/agents/drivers/sundb/src/main/java/com/dbx/agent/sundb/SundbAgent.java +++ b/agents/drivers/sundb/src/main/java/com/dbx/agent/sundb/SundbAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.sundb; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -18,8 +18,6 @@ import com.dbx.agent.QueryResult; import com.dbx.agent.TableInfo; import com.dbx.agent.TriggerInfo; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Types; import java.util.ArrayList; @@ -32,32 +30,15 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -public final class SundbAgent extends BaseDatabaseAgent { - private Connection connection; - +public final class SundbAgent extends AbstractJdbcAgent { @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return "com.sundb.jdbc.SundbDriver"; } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - Class.forName("com.sundb.jdbc.SundbDriver"); - String url = buildUrl(params); - connection = DriverManager.getConnection(url, params.getUsername(), params.getPassword()); - }); - } - - @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - Class.forName("com.sundb.jdbc.SundbDriver"); - String url = buildUrl(params); - try (Connection conn = DriverManager.getConnection(url, params.getUsername(), params.getPassword())) { - return conn.isValid(5); - } - }); + protected String buildJdbcUrl(ConnectParams params) { + return buildUrl(params); } @Override @@ -415,7 +396,7 @@ public final class SundbAgent extends BaseDatabaseAgent { options.getMaxRows(), options.getFetchSize(), options.getTimeoutSecs(), - this::getResultValue + this::resultValue ); } @@ -424,16 +405,6 @@ public final class SundbAgent extends BaseDatabaseAgent { return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema); } - @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - }); - } - private static String buildUrl(ConnectParams params) { return "jdbc:sundb://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase(); } @@ -504,7 +475,8 @@ public final class SundbAgent extends BaseDatabaseAgent { return value == null || value.isEmpty() ? null : value; } - private Object getResultValue(ResultSet rs, int index, int sqlType) { + @Override + protected Object resultValue(ResultSet rs, int index, int sqlType) { try { Object value = switch (sqlType) { case Types.BIGINT -> rs.getLong(index); diff --git a/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java b/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java index efb917f2d2..7f8b0672f2 100644 --- a/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java +++ b/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java @@ -1,6 +1,6 @@ package com.dbx.agent.tdengine; -import com.dbx.agent.BaseDatabaseAgent; +import com.dbx.agent.AbstractJdbcAgent; import com.dbx.agent.ColumnInfo; import com.dbx.agent.ConnectParams; import com.dbx.agent.DatabaseInfo; @@ -46,7 +46,7 @@ import java.util.concurrent.TimeoutException; import java.util.regex.Matcher; import java.util.regex.Pattern; -public final class TDengineAgent extends BaseDatabaseAgent { +public final class TDengineAgent extends AbstractJdbcAgent { private static final long TABLE_CACHE_TTL_MILLIS = 10_000L; private static final DateTimeFormatter TDENGINE_TIMESTAMP_FORMAT = new DateTimeFormatterBuilder() @@ -62,32 +62,63 @@ public final class TDengineAgent extends BaseDatabaseAgent { private static final Pattern COMPOSITE_KEY_PATTERN = Pattern.compile("(?i)\\bCOMPOSITE\\s+KEY\\b"); - private Connection connection; private final Object tableCacheLock = new Object(); private String tableCacheSchema = ""; private long tableCacheTimeMillis; private List tableCache = Collections.emptyList(); + private Connection restfulStateConnection; + private String restfulOriginalCatalog; + private String restfulOriginalDatabase; @Override - public Connection getConnection() { - return connection; + protected String driverClass() { + return TDengineTransport.WEBSOCKET.driverClass(); } @Override - public void connect(ConnectParams params) { - uncheckedVoid(() -> { - connection = TDengineConnectionFactory.open(params); - clearTableCache(); - }); + protected String buildJdbcUrl(ConnectParams params) { + String connectionString = params.getConnection_string() == null ? "" : params.getConnection_string().trim(); + if (!connectionString.isBlank()) { + return TDengineJdbcUrl.sanitizeConnectionString(connectionString); + } + TDengineJdbcUrl.TransportPreference preference = TDengineJdbcUrl.transportPreference(params.getUrl_params()); + TDengineTransport transport = preference == TDengineJdbcUrl.TransportPreference.REST + ? TDengineTransport.REST + : TDengineTransport.WEBSOCKET; + return TDengineJdbcUrl.from(params, transport); } @Override - public boolean testConnection(ConnectParams params) { - return unchecked(() -> { - try (Connection conn = TDengineConnectionFactory.open(params)) { - return conn.isValid(5); - } - }); + protected void loadDriver(ConnectParams params) { + } + + @Override + protected Connection openConnection(ConnectParams params) throws Exception { + return TDengineConnectionFactory.open(params); + } + + @Override + protected void afterConnect(ConnectParams params, Connection connection) { + clearTableCache(); + clearRestfulState(); + } + + @Override + protected void beforePooledConnectionReturn(Connection connection) throws Exception { + if (restfulStateConnection != connection) { + return; + } + try { + restoreRestfulConnectionState(connection, restfulOriginalCatalog, restfulOriginalDatabase); + } finally { + clearRestfulState(); + } + } + + @Override + protected void afterDisconnect() { + clearTableCache(); + clearRestfulState(); } @Override @@ -215,7 +246,7 @@ public final class TDengineAgent extends BaseDatabaseAgent { options.getMaxRows(), options.getFetchSize(), options.getTimeoutSecs(), - this::tdengineResultValue + this::resultValue ); if (mayChangeMetadata(sql)) { clearTableCache(); @@ -231,7 +262,7 @@ public final class TDengineAgent extends BaseDatabaseAgent { prepareExecutionSchema(schema), this::setSchemaSQL, options, - this::tdengineResultValue + this::resultValue ); } @@ -243,7 +274,7 @@ public final class TDengineAgent extends BaseDatabaseAgent { prepareExecutionSchema(schema), this::setSchemaSQL, options, - this::tdengineResultValue + this::resultValue ); } @@ -271,11 +302,22 @@ public final class TDengineAgent extends BaseDatabaseAgent { } private String prepareExecutionSchema(String schema) { - return unchecked(() -> prepareExecutionSchema(requireConnected(), schema)); + return unchecked(() -> { + Connection connection = requireConnected(); + if (schema != null + && !schema.trim().isEmpty() + && unwrapConnection(connection, RestfulConnection.class) != null + && restfulStateConnection != connection) { + restfulStateConnection = connection; + restfulOriginalCatalog = connection.getCatalog(); + restfulOriginalDatabase = connection.getClientInfo(TSDBDriver.PROPERTY_KEY_DBNAME); + } + return prepareExecutionSchema(connection, schema); + }); } static String prepareExecutionSchema(Connection connection, String schema) throws SQLException { - if (!(connection instanceof RestfulConnection) || schema == null || schema.trim().isEmpty()) { + if (unwrapConnection(connection, RestfulConnection.class) == null || schema == null || schema.trim().isEmpty()) { return schema; } @@ -286,15 +328,13 @@ public final class TDengineAgent extends BaseDatabaseAgent { return null; } - @Override - public void disconnect() { - uncheckedVoid(() -> { - if (connection != null) { - connection.close(); - } - connection = null; - clearTableCache(); - }); + static void restoreRestfulConnectionState(Connection connection, String catalog, String database) throws SQLException { + connection.setCatalog(catalog); + if (database == null) { + connection.getClientInfo().remove(TSDBDriver.PROPERTY_KEY_DBNAME); + } else { + connection.setClientInfo(TSDBDriver.PROPERTY_KEY_DBNAME, database); + } } private List queryTables(String sql, String tableType, boolean includesStableName) throws Exception { @@ -342,6 +382,12 @@ public final class TDengineAgent extends BaseDatabaseAgent { } } + private void clearRestfulState() { + restfulStateConnection = null; + restfulOriginalCatalog = null; + restfulOriginalDatabase = null; + } + private static boolean cacheFresh(long cachedAtMillis) { return cachedAtMillis > 0L && System.currentTimeMillis() - cachedAtMillis <= TABLE_CACHE_TTL_MILLIS; } @@ -452,7 +498,8 @@ public final class TDengineAgent extends BaseDatabaseAgent { } } - private Object tdengineResultValue(ResultSet rs, int index, int sqlType) { + @Override + protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { Object value = switch (sqlType) { case Types.BIGINT -> rs.getLong(index); diff --git a/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java b/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java index 6e2072d817..e1f17f56ec 100644 --- a/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java +++ b/agents/drivers/tdengine/src/test/java/com/dbx/agent/tdengine/TDengineAgentTest.java @@ -12,6 +12,8 @@ import com.dbx.agent.test.JdbcMetadataSqlFake; import com.dbx.agent.test.TestSupport; import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.rs.RestfulConnection; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Proxy; import java.nio.charset.StandardCharsets; @@ -22,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Properties; +import javax.sql.DataSource; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -354,6 +357,49 @@ class TDengineAgentMetadataTest { } } + @Test + void restExecutionSupportsPooledWrappersAndRestoresOriginalDatabase() throws Exception { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_DBNAME, "original"); + try (RestfulConnection physical = restfulConnection(properties)) { + physical.setCatalog("original"); + Connection pooled = wrapperConnection(physical); + + Assertions.assertNull(TDengineAgent.prepareExecutionSchema(pooled, "power")); + Assertions.assertEquals("power", pooled.getCatalog()); + Assertions.assertEquals("power", pooled.getClientInfo(TSDBDriver.PROPERTY_KEY_DBNAME)); + + TDengineAgent.restoreRestfulConnectionState(pooled, "original", "original"); + Assertions.assertEquals("original", pooled.getCatalog()); + Assertions.assertEquals("original", pooled.getClientInfo(TSDBDriver.PROPERTY_KEY_DBNAME)); + } + } + + @Test + void hikariCanManageRestfulConnectionsWithoutLosingNativeAccess() throws Exception { + RestfulConnection physical = restfulConnection(); + DataSource dataSource = proxy(DataSource.class, (proxy, method, args) -> { + if ("getConnection".equals(method.getName())) { + return physical; + } + return defaultValue(method.getReturnType()); + }); + HikariConfig config = new HikariConfig(); + config.setPoolName("tdengine-rest-test"); + config.setDataSource(dataSource); + config.setMaximumPoolSize(1); + config.setMinimumIdle(0); + config.setInitializationFailTimeout(-1L); + + try (HikariDataSource pool = new HikariDataSource(config); + Connection connection = pool.getConnection()) { + Assertions.assertTrue(connection.isWrapperFor(RestfulConnection.class)); + Assertions.assertSame(physical, connection.unwrap(RestfulConnection.class)); + Assertions.assertNull(TDengineAgent.prepareExecutionSchema(connection, "power")); + Assertions.assertEquals("power", connection.getCatalog()); + } + } + @Test void websocketExecutionKeepsSchemaForUseSqlSwitching() throws Exception { Connection connection = JdbcAgentFake.connection(); @@ -441,10 +487,14 @@ class TDengineAgentMetadataTest { } private static RestfulConnection restfulConnection() { + return restfulConnection(new Properties()); + } + + private static RestfulConnection restfulConnection(Properties properties) { return new RestfulConnection( "127.0.0.1", "6041", - new Properties(), + properties, "", "jdbc:TAOS-RS://127.0.0.1:6041/", null, @@ -453,4 +503,20 @@ class TDengineAgentMetadataTest { null ); } + + private static Connection wrapperConnection(Connection physical) { + return proxy(Connection.class, (proxy, method, args) -> { + if ("isWrapperFor".equals(method.getName())) { + return ((Class) args[0]).isInstance(physical); + } + if ("unwrap".equals(method.getName())) { + return ((Class) args[0]).cast(physical); + } + try { + return method.invoke(physical, args); + } catch (java.lang.reflect.InvocationTargetException error) { + throw error.getCause(); + } + }); + } } diff --git a/agents/scripts/validate_agents.py b/agents/scripts/validate_agents.py index 195fb7fa84..fc030bdc40 100644 --- a/agents/scripts/validate_agents.py +++ b/agents/scripts/validate_agents.py @@ -21,15 +21,10 @@ NATIVE_ONLY_AGENT_MODULES = { AUTO_VERSIONED_NATIVE_MODULES = {"duckdb"} JDBC_ARCHITECTURE_ALLOWLIST = { "h2-legacy": "reuses the H2 agent implementation with an isolated legacy driver version", - "access": "custom Access metadata and URL behavior pending migration", - "dameng": "custom Dameng metadata and DDL pending migration", - "db2": "custom DB2 metadata pending migration", - "gbase8s": "custom GBase 8s metadata pending migration", - "goldendb": "custom GoldenDB metadata pending migration", - "informix": "custom Informix metadata pending migration", - "neo4j": "custom Neo4j transaction/query behavior pending migration", - "sundb": "custom SunDB metadata pending migration", - "tdengine": "custom TDengine WebSocket JDBC behavior pending migration", + "access": "shared lifecycle with a test-only non-creating Access URL", + "dameng": "shared lifecycle with protocol-safe driver loading and native explain access", + "informix": "shared lifecycle with contextual connection error reporting", + "tdengine": "shared lifecycle with WebSocket-to-REST transport fallback", } APPROVED_JDBC_BASES = { "AbstractJdbcAgent", @@ -189,6 +184,25 @@ def validate_jdbc_architecture(root: Path, modules: set[str]) -> list[str]: return problems +def validate_jdbc_pool_coverage(root: Path, modules: set[str]) -> list[str]: + build_file = root / "build.gradle" + if not build_file.exists(): + return [] + match = re.search( + r"\bdef\s+pooledJdbcProjects\s*=\s*\[(.*?)\]\s*as\s+Set", + build_file.read_text(encoding="utf-8"), + flags=re.S, + ) + if match is None: + return ["build.gradle: missing pooledJdbcProjects set"] + pooled = set(re.findall(r"['\"]([^'\"]+)['\"]", match.group(1))) + expected = modules - NON_JDBC_AGENT_MODULES - set(NATIVE_ONLY_AGENT_MODULES) + return ( + [f"JDBC module missing pooled runtime dependency: {module}" for module in sorted(expected - pooled)] + + [f"non-JDBC module listed as pooled JDBC runtime: {module}" for module in sorted(pooled - expected)] + ) + + def validate_authoring_template(root: Path) -> list[str]: template = root / "docs/examples/jdbc-agent-template/src/main/java/com/dbx/agent/template/TemplateAgent.java" if not template.exists(): @@ -294,6 +308,7 @@ def validate(root: Path) -> list[str]: + validate_source_patterns(root) + validate_manifest_fields(root, modules) + validate_jdbc_architecture(root, modules) + + validate_jdbc_pool_coverage(root, modules) + validate_authoring_template(root) + validate_release_runtime_keys(root) + validate_no_kotlin_residue(root) diff --git a/agents/scripts/validate_agents_test.py b/agents/scripts/validate_agents_test.py index 2d48788a64..8c438d06c7 100644 --- a/agents/scripts/validate_agents_test.py +++ b/agents/scripts/validate_agents_test.py @@ -174,6 +174,32 @@ class ValidateAgentsTest(unittest.TestCase): validate_agents.validate_authoring_template(root), ) + def test_jdbc_pool_coverage_requires_every_jdbc_module(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "build.gradle").write_text( + "def pooledJdbcProjects = ['h2'] as Set\n", + encoding="utf-8", + ) + + self.assertEqual( + ["JDBC module missing pooled runtime dependency: access"], + validate_agents.validate_jdbc_pool_coverage(root, {"access", "h2", "mongodb"}), + ) + + def test_jdbc_pool_coverage_rejects_non_jdbc_modules(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "build.gradle").write_text( + "def pooledJdbcProjects = ['h2', 'mongodb'] as Set\n", + encoding="utf-8", + ) + + self.assertEqual( + ["non-JDBC module listed as pooled JDBC runtime: mongodb"], + validate_agents.validate_jdbc_pool_coverage(root, {"h2", "mongodb"}), + ) + def test_manifest_validation_requires_registry_fields(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/crates/dbx-core/src/connection.rs b/crates/dbx-core/src/connection.rs index 87eda46c5f..c4d83ccadb 100644 --- a/crates/dbx-core/src/connection.rs +++ b/crates/dbx-core/src/connection.rs @@ -278,6 +278,15 @@ pub struct PoolActivityTouch { task_supervisor: TaskSupervisor, } +pub(crate) struct ClientSessionPoolCleanupGuard { + pool_key: String, + connections: Arc>>, + pool_activity: Arc>>, + postgres_cancel_contexts: Arc>>, + task_supervisor: TaskSupervisor, + armed: bool, +} + impl Drop for PoolActivityTouch { fn drop(&mut self) { let pool_key = self.pool_key.clone(); @@ -296,6 +305,34 @@ impl Drop for PoolActivityTouch { } } +impl ClientSessionPoolCleanupGuard { + pub(crate) fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for ClientSessionPoolCleanupGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + let pool_key = self.pool_key.clone(); + let connections = self.connections.clone(); + let pool_activity = self.pool_activity.clone(); + let postgres_cancel_contexts = self.postgres_cancel_contexts.clone(); + let task_supervisor = self.task_supervisor.clone(); + task_supervisor.stop(&format!("keepalive:{pool_key}")); + task_supervisor.spawn_once(format!("client-session-cleanup:{pool_key}"), move |_| async move { + pool_activity.write().await.remove(&pool_key); + postgres_cancel_contexts.write().await.remove(&pool_key); + let removed = connections.write().await.remove(&pool_key); + if let Some(pool) = removed { + close_pool_kind_with_timeout(pool_key, pool).await; + } + }); + } +} + pub fn metadata_connection_config(config: &ConnectionConfig) -> ConnectionConfig { let mut db_config = config.canonicalized(); if database_capabilities::is_metadata_connection_scoped(&db_config.db_type) { @@ -2418,7 +2455,10 @@ impl AppState { PoolKind::Agent(client) => { let client = client.clone(); drop(connections); - let mut agent = client.lock().await; + let Ok(mut agent) = client.try_lock() else { + log::debug!("Agent connection pool '{pool_key}' is busy; skipping health probe"); + return false; + }; let timeout = crate::db::connection_timeout(); match agent.validate_connection(Some(timeout)).await { Ok(_) => false, @@ -2515,6 +2555,32 @@ impl AppState { Ok(true) } + pub(crate) async fn client_session_pool_cleanup_guard( + &self, + connection_id: &str, + database: Option<&str>, + client_session_id: &str, + ) -> Option { + let config = { + let configs = self.configs.read().await; + configs.get(connection_id).cloned() + }; + let db_type = config.as_ref().map(|config| config.db_type); + let base_pool_key = base_pool_key_for(db_type, connection_id, database, false); + let pool_key = session_scoped_pool_key_for(config.as_ref(), base_pool_key.clone(), Some(client_session_id)); + if pool_key == base_pool_key { + return None; + } + Some(ClientSessionPoolCleanupGuard { + pool_key, + connections: self.connections.clone(), + pool_activity: self.pool_activity.clone(), + postgres_cancel_contexts: self.postgres_cancel_contexts.clone(), + task_supervisor: self.task_supervisor.clone(), + armed: true, + }) + } + /// Removes a session-scoped pool immediately and schedules the potentially slow driver /// shutdown on the supervised background task set. pub async fn detach_client_session_pool( @@ -5332,6 +5398,22 @@ for line in sys.stdin: let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn busy_agent_pool_skips_health_probe_without_waiting() { + let (state, dir) = test_app_state().await; + let client = + std::sync::Arc::new(tokio::sync::Mutex::new(crate::db::agent_driver::AgentDriverClient::test_stub())); + state.connections.write().await.insert("conn".to_string(), PoolKind::Agent(client.clone())); + let _busy = client.lock().await; + + let started = Instant::now(); + assert!(!state.remove_stale_connection_pool("conn").await); + assert!(started.elapsed() < Duration::from_millis(100)); + assert!(state.connections.read().await.contains_key("conn")); + + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn detects_database_connection_slot_exhaustion_errors() { assert!(super::is_connection_slot_exhausted_error( @@ -5448,6 +5530,41 @@ for line in sys.stdin: let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn client_session_cleanup_guard_detaches_pool_when_request_is_dropped() { + let (state, dir) = test_app_state().await; + let mut config = mysql_config(None); + config.id = "conn".to_string(); + state.configs.write().await.insert(config.id.clone(), config); + + let pool_key = "conn:session:completion-objects_request-1"; + let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap(); + state.connections.write().await.insert(pool_key.to_string(), PoolKind::Sqlite(pool)); + state.pool_activity.write().await.insert(pool_key.to_string(), super::PoolActivity::now()); + + let guard = + state.client_session_pool_cleanup_guard("conn", None, "completion-objects:request-1").await.unwrap(); + drop(guard); + + for _ in 0..100 { + if !state.connections.read().await.contains_key(pool_key) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!state.connections.read().await.contains_key(pool_key)); + assert!(!state.pool_activity.read().await.contains_key(pool_key)); + for _ in 0..100 { + if state.supervised_task_count() == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(state.supervised_task_count(), 0); + + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn closing_oracle_table_tab_keeps_connection_scoped_pool() { let (state, dir) = test_app_state().await; diff --git a/crates/dbx-core/src/db/agent_driver.rs b/crates/dbx-core/src/db/agent_driver.rs index 13a6e56665..f0d6c0a762 100644 --- a/crates/dbx-core/src/db/agent_driver.rs +++ b/crates/dbx-core/src/db/agent_driver.rs @@ -81,14 +81,25 @@ impl AgentRuntimeClient { handshake: AgentHandshake { protocol_version: 0, agent_protocol_version: 0, capabilities: Vec::new() }, }); runtime.start_response_reader(stdout); - let handshake = runtime + let handshake = match runtime .call::( AgentMethod::Handshake.as_str(), agent_handshake_params(app_version), Some(Duration::from_secs(RPC_TIMEOUT_SECS)), None, ) - .await?; + .await + { + Ok(handshake) => handshake, + Err(error) if is_unsupported_handshake_error(&error) => { + runtime.kill(); + return Err("Agent runtime does not support multi_session protocol v2".to_string()); + } + Err(error) => { + runtime.kill(); + return Err(error); + } + }; if handshake.protocol_version < 2 || !handshake.supports(AgentCapability::MultiSession) { runtime.kill(); return Err("Agent runtime does not support multi_session protocol v2".to_string()); @@ -2633,6 +2644,42 @@ for line in sys.stdin: let _ = std::fs::remove_file(script_path); } + #[tokio::test] + async fn shared_runtime_routes_agents_without_handshake_to_legacy_fallback() { + let script_path = + std::env::temp_dir().join(format!("dbx-agent-runtime-no-handshake-test-{}.py", uuid::Uuid::new_v4())); + std::fs::write( + &script_path, + r#"import json, sys +print(json.dumps({'ready': True}), flush=True) +for line in sys.stdin: + req = json.loads(line) + print(json.dumps({ + 'jsonrpc': '2.0', + 'id': req['id'], + 'error': {'code': -1, 'message': 'Unknown method: ' + req['method']} + }), flush=True) +"#, + ) + .unwrap(); + + let error = match AgentRuntimeClient::spawn( + AgentLaunchSpec::new("python3").with_args([script_path.to_string_lossy().to_string()]), + "test", + ) + .await + { + Ok(runtime) => { + runtime.kill(); + panic!("agent without handshake unexpectedly started as a shared runtime"); + } + Err(error) => error, + }; + + assert_eq!(error, "Agent runtime does not support multi_session protocol v2"); + let _ = std::fs::remove_file(script_path); + } + #[tokio::test] async fn canceling_one_runtime_request_keeps_other_requests_alive() { let script_path = std::env::temp_dir().join(format!("dbx-agent-cancel-test-{}.py", uuid::Uuid::new_v4())); diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema.rs index c8e17f0932..5a30142304 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema.rs @@ -1,4 +1,6 @@ -use crate::connection::{connection_url_for_endpoint, database_connection_config, AppState, MysqlMode, PoolKind}; +use crate::connection::{ + connection_url_for_endpoint, database_connection_config, task_client_session_id, AppState, MysqlMode, PoolKind, +}; use crate::db; use crate::models::connection::{ConnectionConfig, DatabaseType}; use crate::query::{agent_execute_query_params, should_discard_pool_after_error, QueryExecutionOptions}; @@ -29,6 +31,37 @@ macro_rules! dispatch_mysql { }; } +struct EphemeralAgentMetadataSession { + client_session_id: Option, + cleanup_guard: Option, +} + +impl EphemeralAgentMetadataSession { + async fn open(state: &AppState, connection_id: &str, database: Option<&str>, task_kind: &str) -> Self { + let db_config = connection_config(state, connection_id).await; + let client_session_id = ephemeral_agent_metadata_session_id(db_config.as_ref(), task_kind); + let cleanup_guard = match client_session_id.as_deref() { + Some(client_session_id) => { + state.client_session_pool_cleanup_guard(connection_id, database, client_session_id).await + } + None => None, + }; + Self { client_session_id, cleanup_guard } + } + + fn client_session_id(&self) -> Option<&str> { + self.client_session_id.as_deref() + } + + async fn finish(mut self, state: &AppState, connection_id: &str, database: Option<&str>) { + if close_ephemeral_agent_metadata_session(state, connection_id, database, self.client_session_id()).await { + if let Some(cleanup_guard) = self.cleanup_guard.as_mut() { + cleanup_guard.disarm(); + } + } + } +} + macro_rules! try_sqlserver { ($connections:expr, $pool_key:expr, $method:ident $(, $arg:expr)*) => { if let Some(client) = extract_pool!(&$connections, $pool_key, SqlServer) { @@ -775,10 +808,30 @@ pub async fn list_tables_core( object_types: Option<&[String]>, table_name_filter: Option<&TableNameFilter>, ) -> Result, String> { - retry_metadata_connection(state, connection_id, Some(database), || { - list_tables_once(state, connection_id, database, schema, filter, limit, offset, object_types, table_name_filter) - }) - .await + let metadata_session = EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "tables").await; + let result = retry_metadata_connection_for_session( + state, + connection_id, + Some(database), + metadata_session.client_session_id(), + || { + list_tables_once( + state, + connection_id, + database, + schema, + filter, + limit, + offset, + object_types, + table_name_filter, + metadata_session.client_session_id(), + ) + }, + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + result } /// List vector database collections, returning structured info (name, id, dimension). @@ -830,8 +883,31 @@ pub async fn get_table_comment_core( return Err("Table comments are not available for linked server tables".to_string()); } - retry_metadata_connection(state, connection_id, Some(database), || async { - let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; + let metadata_session = + EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "table-comment").await; + let result = get_table_comment_core_for_session( + state, + connection_id, + database, + schema, + table, + metadata_session.client_session_id(), + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + result +} + +async fn get_table_comment_core_for_session( + state: &AppState, + connection_id: &str, + database: &str, + schema: &str, + table: &str, + client_session_id: Option<&str>, +) -> Result, String> { + retry_metadata_connection_for_session(state, connection_id, Some(database), client_session_id, || async { + let pool_key = state.get_or_create_pool_for_session(connection_id, Some(database), client_session_id).await?; let db_config = connection_config(state, connection_id).await; { @@ -1649,8 +1725,9 @@ async fn list_tables_once( offset: Option, object_types: Option<&[String]>, table_name_filter: Option<&TableNameFilter>, + client_session_id: Option<&str>, ) -> Result, String> { - let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; + let pool_key = state.get_or_create_pool_for_session(connection_id, Some(database), client_session_id).await?; let db_config = connection_config(state, connection_id).await; { @@ -2350,11 +2427,12 @@ mod tests { use super::{ clickhouse_metadata_database, dameng_object_statistics_dba_segments_sql, dameng_object_statistics_rows_only_sql, dameng_object_statistics_user_segments_sql, deduplicate_column_infos, - filter_mysql_system_databases_for_config, filter_object_infos, filter_table_infos, filter_visible_schema_names, - gbase8a_object_statistics_sql, is_agent_postgres_metadata_fallback_config, is_retryable_metadata_error, - metadata_name_or_comment_matches, mysql_object_source_ddl_column_index, mysql_object_source_sql, - mysql_table_metadata_catalog, normalize_information_schema_table_type, oracle_columns_from_query_result, - oracle_columns_sql, oracle_object_statistics_dba_segments_sql, oracle_object_statistics_from_query_result, + ephemeral_agent_metadata_session_id, filter_mysql_system_databases_for_config, filter_object_infos, + filter_table_infos, filter_visible_schema_names, gbase8a_object_statistics_sql, + is_agent_postgres_metadata_fallback_config, is_retryable_metadata_error, metadata_name_or_comment_matches, + mysql_object_source_ddl_column_index, mysql_object_source_sql, mysql_table_metadata_catalog, + normalize_information_schema_table_type, oracle_columns_from_query_result, oracle_columns_sql, + oracle_object_statistics_dba_segments_sql, oracle_object_statistics_from_query_result, oracle_object_statistics_rows_only_sql, oracle_object_statistics_sql, oracle_object_statistics_user_segments_sql, oracle_table_comment_from_query_result, oracle_table_comment_sql, oracle_table_comments_sql, presto_like_columns_from_query_result, presto_like_information_schema_columns_sql, @@ -2440,6 +2518,20 @@ mod tests { } } + #[test] + fn agent_metadata_uses_unique_ephemeral_sessions_only_for_agents() { + let oracle = test_connection_config(DatabaseType::Oracle); + let first = ephemeral_agent_metadata_session_id(Some(&oracle), "completion-objects").unwrap(); + let second = ephemeral_agent_metadata_session_id(Some(&oracle), "completion-objects").unwrap(); + + assert_ne!(first, second); + assert!(first.starts_with("completion-objects:")); + + let postgres = test_connection_config(DatabaseType::Postgres); + assert!(ephemeral_agent_metadata_session_id(Some(&postgres), "completion-objects").is_none()); + assert!(ephemeral_agent_metadata_session_id(None, "completion-objects").is_none()); + } + #[test] fn mysql_table_child_metadata_prefers_schema_when_present() { assert_eq!(mysql_table_metadata_catalog("app_db", ""), "app_db"); @@ -3521,8 +3613,24 @@ pub async fn list_objects_core( }); let use_oracle_agent_paging = db_config.as_ref().is_some_and(is_default_oracle_agent_config) && !filter_locally_after_oracle_comments; - retry_metadata_connection(state, connection_id, Some(database), || async { - let objects = list_objects_once(state, connection_id, database, schema, filter, limit, offset, object_types) + let metadata_session = EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "objects").await; + let result = retry_metadata_connection_for_session( + state, + connection_id, + Some(database), + metadata_session.client_session_id(), + || async { + let objects = list_objects_once( + state, + connection_id, + database, + schema, + filter, + limit, + offset, + object_types, + metadata_session.client_session_id(), + ) .await .map(|outcome| { let final_offset = if outcome.paging_applied @@ -3534,9 +3642,12 @@ pub async fn list_objects_core( }; filter_object_infos(outcome.objects, filter, limit, final_offset, object_types) })?; - Ok(objects) - }) - .await + Ok(objects) + }, + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + result } pub async fn list_object_statistics_core( @@ -3557,10 +3668,44 @@ pub async fn list_completion_objects_core( database: &str, schema: &str, ) -> Result, String> { - retry_metadata_connection(state, connection_id, Some(database), || { - list_completion_objects_once(state, connection_id, database, schema) - }) - .await + let metadata_session = + EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "completion-objects").await; + let result = retry_metadata_connection_for_session( + state, + connection_id, + Some(database), + metadata_session.client_session_id(), + || list_completion_objects_once(state, connection_id, database, schema, metadata_session.client_session_id()), + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + result +} + +fn ephemeral_agent_metadata_session_id(config: Option<&ConnectionConfig>, task_kind: &str) -> Option { + config + .filter(|config| crate::database_capabilities::is_agent_type(&config.db_type)) + .map(|_| task_client_session_id(task_kind, &uuid::Uuid::new_v4().to_string())) +} + +async fn close_ephemeral_agent_metadata_session( + state: &AppState, + connection_id: &str, + database: Option<&str>, + client_session_id: Option<&str>, +) -> bool { + let Some(client_session_id) = client_session_id else { + return true; + }; + match state.close_client_session_pool(connection_id, database, client_session_id).await { + Ok(_) => true, + Err(error) => { + log::warn!( + "Failed to close ephemeral Agent metadata session '{client_session_id}' for '{connection_id}': {error}" + ); + false + } + } } pub async fn completion_assistant_search_core( @@ -3899,8 +4044,9 @@ async fn list_objects_once( limit: Option, offset: Option, object_types: Option<&[String]>, + client_session_id: Option<&str>, ) -> Result { - let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; + let pool_key = state.get_or_create_pool_for_session(connection_id, Some(database), client_session_id).await?; let db_config = connection_config(state, connection_id).await; let (mysql_limit, mysql_offset) = if filter.is_none_or(|value| value.trim().is_empty()) { (limit, offset) } else { (None, None) }; @@ -4095,8 +4241,9 @@ async fn list_completion_objects_once( connection_id: &str, database: &str, schema: &str, + client_session_id: Option<&str>, ) -> Result, String> { - let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; + let pool_key = state.get_or_create_pool_for_session(connection_id, Some(database), client_session_id).await?; let db_config = connection_config(state, connection_id).await; let connections = state.connections.read().await; @@ -4187,7 +4334,8 @@ async fn list_completion_objects_once( PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(filter_completion_objects), PoolKind::SqlServer(_) => { drop(connections); - let outcome = list_objects_once(state, connection_id, database, schema, None, None, None, None).await?; + let outcome = + list_objects_once(state, connection_id, database, schema, None, None, None, None, None).await?; Ok(filter_completion_objects(outcome.objects)) } _ => Ok(Vec::new()), @@ -4286,6 +4434,37 @@ pub async fn get_columns_core_for_session( table: &str, client_session_id: Option<&str>, ) -> Result, String> { + if client_session_id.is_none() { + let metadata_session = + EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "columns").await; + if metadata_session.client_session_id().is_some() { + let result = get_columns_core_for_session_inner( + state, + connection_id, + database, + schema, + table, + metadata_session.client_session_id(), + false, + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + return result; + } + } + get_columns_core_for_session_inner(state, connection_id, database, schema, table, client_session_id, true).await +} + +async fn get_columns_core_for_session_inner( + state: &AppState, + connection_id: &str, + database: &str, + schema: &str, + table: &str, + client_session_id: Option<&str>, + use_client_session_context: bool, +) -> Result, String> { + let context_session_id = if use_client_session_context { client_session_id } else { None }; retry_metadata_connection_for_session(state, connection_id, Some(database), client_session_id, || async { let pool_key = state .get_or_create_pool_for_session(connection_id, Some(database), client_session_id) @@ -4302,7 +4481,7 @@ pub async fn get_columns_core_for_session( return external_driver_presto_like_columns(session, config.as_ref(), database, schema, table).await; } let query_oracle_columns_first = - should_query_oracle_columns_via_sql_first(&config.db_type, schema, client_session_id); + should_query_oracle_columns_via_sql_first(&config.db_type, schema, context_session_id); if query_oracle_columns_first { match external_driver_oracle_columns_via_sql( session.clone(), @@ -4404,7 +4583,7 @@ pub async fn get_columns_core_for_session( drop(connections); let mut client = client.lock().await; let oracle_sql_config = fallback_config.as_ref().filter(|config| { - should_query_oracle_columns_via_sql_first(&config.db_type, schema, client_session_id) + should_query_oracle_columns_via_sql_first(&config.db_type, schema, context_session_id) }); let query_oracle_columns_first = oracle_sql_config.is_some(); if let Some(config) = oracle_sql_config { @@ -4629,8 +4808,30 @@ pub async fn list_indexes_core( if crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema).is_some() { return Ok(vec![]); } - retry_metadata_connection(state, connection_id, Some(database), || async { - let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; + let metadata_session = EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "indexes").await; + let result = list_indexes_core_for_session( + state, + connection_id, + database, + schema, + table, + metadata_session.client_session_id(), + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + result +} + +async fn list_indexes_core_for_session( + state: &AppState, + connection_id: &str, + database: &str, + schema: &str, + table: &str, + client_session_id: Option<&str>, +) -> Result, String> { + retry_metadata_connection_for_session(state, connection_id, Some(database), client_session_id, || async { + let pool_key = state.get_or_create_pool_for_session(connection_id, Some(database), client_session_id).await?; let db_config = connection_config(state, connection_id).await; { @@ -4685,8 +4886,31 @@ pub async fn list_foreign_keys_core( if crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema).is_some() { return Ok(vec![]); } - retry_metadata_connection(state, connection_id, Some(database), || async { - let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; + let metadata_session = + EphemeralAgentMetadataSession::open(state, connection_id, Some(database), "foreign-keys").await; + let result = list_foreign_keys_core_for_session( + state, + connection_id, + database, + schema, + table, + metadata_session.client_session_id(), + ) + .await; + metadata_session.finish(state, connection_id, Some(database)).await; + result +} + +async fn list_foreign_keys_core_for_session( + state: &AppState, + connection_id: &str, + database: &str, + schema: &str, + table: &str, + client_session_id: Option<&str>, +) -> Result, String> { + retry_metadata_connection_for_session(state, connection_id, Some(database), client_session_id, || async { + let pool_key = state.get_or_create_pool_for_session(connection_id, Some(database), client_session_id).await?; let db_config = connection_config(state, connection_id).await; {