feat(export): add table read cursor sessions
This commit is contained in:
1 parent
d5c806f484
commit
0c0e90ab1a
22 files changed
+1562
-54
No files matched your search
@@ -138,6 +138,28 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
|
||||
return JdbcExecutor.INSTANCE.closeQuerySession(sessionId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||
return JdbcExecutor.INSTANCE.startTableRead(
|
||||
requireConnected(),
|
||||
sql,
|
||||
schema,
|
||||
this::setSchemaSQL,
|
||||
options,
|
||||
this::resultValue
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) {
|
||||
return JdbcExecutor.INSTANCE.fetchTableReadPage(sessionId, pageSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean closeTableReadSession(String sessionId) {
|
||||
return JdbcExecutor.INSTANCE.closeTableReadSession(sessionId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult executeTransaction(List<String> statements, String schema) {
|
||||
return TransactionExecutor.executeUpdateStatements(requireConnected(), statements, schema, this::setSchemaSQL);
|
||||
|
||||
@@ -26,6 +26,9 @@ public final class AgentProtocol {
|
||||
public static final String METHOD_EXECUTE_QUERY_PAGE = "execute_query_page";
|
||||
public static final String METHOD_FETCH_QUERY_PAGE = "fetch_query_page";
|
||||
public static final String METHOD_CLOSE_QUERY_SESSION = "close_query_session";
|
||||
public static final String METHOD_START_TABLE_READ = "start_table_read";
|
||||
public static final String METHOD_FETCH_TABLE_READ_PAGE = "fetch_table_read_page";
|
||||
public static final String METHOD_CLOSE_TABLE_READ_SESSION = "close_table_read_session";
|
||||
public static final String METHOD_GET_EXPLAIN_INFO = "get_explain_info";
|
||||
public static final String METHOD_EXECUTE_TRANSACTION = "execute_transaction";
|
||||
public static final String METHOD_DISCONNECT = "disconnect";
|
||||
@@ -93,6 +96,9 @@ public final class AgentProtocol {
|
||||
METHOD_EXECUTE_QUERY_PAGE,
|
||||
METHOD_FETCH_QUERY_PAGE,
|
||||
METHOD_CLOSE_QUERY_SESSION,
|
||||
METHOD_START_TABLE_READ,
|
||||
METHOD_FETCH_TABLE_READ_PAGE,
|
||||
METHOD_CLOSE_TABLE_READ_SESSION,
|
||||
METHOD_GET_EXPLAIN_INFO,
|
||||
METHOD_EXECUTE_TRANSACTION,
|
||||
METHOD_DISCONNECT,
|
||||
|
||||
@@ -67,6 +67,28 @@ public abstract class BaseDatabaseAgent implements DatabaseAgent {
|
||||
return JdbcExecutor.INSTANCE.closeQuerySession(sessionId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||
return JdbcExecutor.INSTANCE.startTableRead(
|
||||
requireConnected(),
|
||||
sql,
|
||||
schema,
|
||||
this::setSchemaSQL,
|
||||
options,
|
||||
JdbcExecutor.INSTANCE::defaultResultValue
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) {
|
||||
return JdbcExecutor.INSTANCE.fetchTableReadPage(sessionId, pageSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean closeTableReadSession(String sessionId) {
|
||||
return JdbcExecutor.INSTANCE.closeTableReadSession(sessionId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult executeTransaction(List<String> statements, String schema) {
|
||||
return TransactionExecutor.executeUpdateStatements(requireConnected(), statements, schema, this::setSchemaSQL);
|
||||
|
||||
@@ -102,6 +102,29 @@ public interface DatabaseAgent {
|
||||
return JdbcExecutor.INSTANCE.closeQuerySession(sessionId);
|
||||
}
|
||||
|
||||
default QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||
Connection conn = getConnection();
|
||||
if (conn == null) {
|
||||
throw new IllegalStateException("Not connected");
|
||||
}
|
||||
return JdbcExecutor.INSTANCE.startTableRead(
|
||||
conn,
|
||||
sql,
|
||||
schema,
|
||||
this::setSchemaSQL,
|
||||
options,
|
||||
JdbcExecutor.INSTANCE::defaultResultValue
|
||||
);
|
||||
}
|
||||
|
||||
default QueryPageResult fetchTableReadPage(String sessionId, int pageSize) {
|
||||
return JdbcExecutor.INSTANCE.fetchTableReadPage(sessionId, pageSize);
|
||||
}
|
||||
|
||||
default boolean closeTableReadSession(String sessionId) {
|
||||
return JdbcExecutor.INSTANCE.closeTableReadSession(sessionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get DM execution plan. Supports two modes:
|
||||
* mode="explain" (default) — direct plan, no execution
|
||||
|
||||
@@ -23,6 +23,7 @@ public final class JdbcExecutor {
|
||||
public static final long QUERY_SESSION_IDLE_TIMEOUT_MILLIS = 10 * 60 * 1000L;
|
||||
|
||||
private final ConcurrentHashMap<String, QuerySession> sessions = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, QuerySession> tableReadSessions = new ConcurrentHashMap<>();
|
||||
|
||||
private JdbcExecutor() {
|
||||
}
|
||||
@@ -176,9 +177,32 @@ public final class JdbcExecutor {
|
||||
Function<String, String> setSchemaSql,
|
||||
QueryPageOptions options,
|
||||
ResultValueReader valueReader
|
||||
) {
|
||||
return executePage(conn, sql, schema, setSchemaSql, options, valueReader, sessions);
|
||||
}
|
||||
|
||||
public QueryPageResult startTableRead(
|
||||
Connection conn,
|
||||
String sql,
|
||||
String schema,
|
||||
Function<String, String> setSchemaSql,
|
||||
QueryPageOptions options,
|
||||
ResultValueReader valueReader
|
||||
) {
|
||||
return executePage(conn, sql, schema, setSchemaSql, options, valueReader, tableReadSessions);
|
||||
}
|
||||
|
||||
private QueryPageResult executePage(
|
||||
Connection conn,
|
||||
String sql,
|
||||
String schema,
|
||||
Function<String, String> setSchemaSql,
|
||||
QueryPageOptions options,
|
||||
ResultValueReader valueReader,
|
||||
ConcurrentHashMap<String, QuerySession> targetSessions
|
||||
) {
|
||||
return unchecked(() -> {
|
||||
expireIdleQuerySessions();
|
||||
expireIdleSessions(targetSessions, System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS);
|
||||
String trimmedSql = trimSql(sql);
|
||||
String upperSql = trimmedSql.toUpperCase(Locale.ROOT).trim();
|
||||
long start = System.currentTimeMillis();
|
||||
@@ -242,8 +266,8 @@ public final class JdbcExecutor {
|
||||
Math.max(options.getMaxRows(), 1),
|
||||
valueReader
|
||||
);
|
||||
sessions.put(sessionId, session);
|
||||
return readSessionPage(session, options.getPageSize(), elapsed);
|
||||
targetSessions.put(sessionId, session);
|
||||
return readSessionPage(targetSessions, session, options.getPageSize(), elapsed);
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
stmt.close();
|
||||
@@ -256,24 +280,28 @@ public final class JdbcExecutor {
|
||||
|
||||
public QueryPageResult fetchPage(String sessionId, int pageSize) {
|
||||
expireIdleQuerySessions();
|
||||
QuerySession session = sessions.get(sessionId);
|
||||
if (session == null) {
|
||||
throw new IllegalArgumentException("Query session not found");
|
||||
}
|
||||
synchronized (session) {
|
||||
return readSessionPage(session, pageSize, 0L);
|
||||
}
|
||||
return fetchSessionPage(sessions, sessionId, pageSize, "Query session not found");
|
||||
}
|
||||
|
||||
public boolean closeQuerySession(String sessionId) {
|
||||
return closeSession(sessionId);
|
||||
return closeSession(sessions, sessionId);
|
||||
}
|
||||
|
||||
public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) {
|
||||
expireIdleTableReadSessions();
|
||||
return fetchSessionPage(tableReadSessions, sessionId, pageSize, "Table read session not found");
|
||||
}
|
||||
|
||||
public boolean closeTableReadSession(String sessionId) {
|
||||
return closeSession(tableReadSessions, sessionId);
|
||||
}
|
||||
|
||||
public void closeAllQuerySessions() {
|
||||
List<String> ids = new ArrayList<>(sessions.keySet());
|
||||
for (String id : ids) {
|
||||
closeSession(id);
|
||||
}
|
||||
closeAllSessions(sessions);
|
||||
}
|
||||
|
||||
public void closeAllTableReadSessions() {
|
||||
closeAllSessions(tableReadSessions);
|
||||
}
|
||||
|
||||
public int expireIdleQuerySessions() {
|
||||
@@ -281,17 +309,33 @@ public final class JdbcExecutor {
|
||||
}
|
||||
|
||||
public int expireIdleQuerySessions(long nowMillis, long idleTimeoutMillis) {
|
||||
return expireIdleSessions(sessions, nowMillis, idleTimeoutMillis);
|
||||
}
|
||||
|
||||
public int expireIdleTableReadSessions() {
|
||||
return expireIdleTableReadSessions(System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS);
|
||||
}
|
||||
|
||||
public int expireIdleTableReadSessions(long nowMillis, long idleTimeoutMillis) {
|
||||
return expireIdleSessions(tableReadSessions, nowMillis, idleTimeoutMillis);
|
||||
}
|
||||
|
||||
private int expireIdleSessions(
|
||||
ConcurrentHashMap<String, QuerySession> targetSessions,
|
||||
long nowMillis,
|
||||
long idleTimeoutMillis
|
||||
) {
|
||||
if (idleTimeoutMillis < 0) {
|
||||
return 0;
|
||||
}
|
||||
int closed = 0;
|
||||
List<QuerySession> snapshot = new ArrayList<>(sessions.values());
|
||||
List<QuerySession> snapshot = new ArrayList<>(targetSessions.values());
|
||||
for (QuerySession session : snapshot) {
|
||||
boolean expired;
|
||||
synchronized (session) {
|
||||
expired = nowMillis - session.lastAccessedAtMillis >= idleTimeoutMillis;
|
||||
}
|
||||
if (expired && closeSession(session.id)) {
|
||||
if (expired && closeSession(targetSessions, session.id)) {
|
||||
closed += 1;
|
||||
}
|
||||
}
|
||||
@@ -409,7 +453,27 @@ public final class JdbcExecutor {
|
||||
return value == null ? null : value.getString();
|
||||
}
|
||||
|
||||
private QueryPageResult readSessionPage(QuerySession session, int pageSize, long executionTimeMs) {
|
||||
private QueryPageResult fetchSessionPage(
|
||||
ConcurrentHashMap<String, QuerySession> targetSessions,
|
||||
String sessionId,
|
||||
int pageSize,
|
||||
String missingMessage
|
||||
) {
|
||||
QuerySession session = targetSessions.get(sessionId);
|
||||
if (session == null) {
|
||||
throw new IllegalArgumentException(missingMessage);
|
||||
}
|
||||
synchronized (session) {
|
||||
return readSessionPage(targetSessions, session, pageSize, 0L);
|
||||
}
|
||||
}
|
||||
|
||||
private QueryPageResult readSessionPage(
|
||||
ConcurrentHashMap<String, QuerySession> targetSessions,
|
||||
QuerySession session,
|
||||
int pageSize,
|
||||
long executionTimeMs
|
||||
) {
|
||||
return unchecked(() -> {
|
||||
session.lastAccessedAtMillis = System.currentTimeMillis();
|
||||
int effectivePageSize = Math.max(pageSize, 1);
|
||||
@@ -422,7 +486,7 @@ public final class JdbcExecutor {
|
||||
|
||||
while (rows.size() < effectivePageSize && session.rowsRead < session.maxRows) {
|
||||
if (!session.resultSet.next()) {
|
||||
closeSession(session.id);
|
||||
closeSession(targetSessions, session.id);
|
||||
return new QueryPageResult(session.columns, session.columnTypes, rows, 0L, executionTimeMs, false, null, false);
|
||||
}
|
||||
rows.add(rowValues(session.resultSet, session.valueReader, session.typeNameByIndex));
|
||||
@@ -431,13 +495,13 @@ public final class JdbcExecutor {
|
||||
|
||||
if (session.rowsRead >= session.maxRows) {
|
||||
boolean truncated = session.resultSet.next();
|
||||
closeSession(session.id);
|
||||
closeSession(targetSessions, session.id);
|
||||
return new QueryPageResult(session.columns, session.columnTypes, rows, 0L, executionTimeMs, truncated, null, false);
|
||||
}
|
||||
|
||||
boolean hasMore = session.resultSet.next();
|
||||
if (!hasMore) {
|
||||
closeSession(session.id);
|
||||
closeSession(targetSessions, session.id);
|
||||
return new QueryPageResult(session.columns, session.columnTypes, rows, 0L, executionTimeMs, false, null, false);
|
||||
}
|
||||
|
||||
@@ -447,8 +511,15 @@ public final class JdbcExecutor {
|
||||
});
|
||||
}
|
||||
|
||||
private boolean closeSession(String sessionId) {
|
||||
QuerySession session = sessions.remove(sessionId);
|
||||
private void closeAllSessions(ConcurrentHashMap<String, QuerySession> targetSessions) {
|
||||
List<String> ids = new ArrayList<>(targetSessions.keySet());
|
||||
for (String id : ids) {
|
||||
closeSession(targetSessions, id);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean closeSession(ConcurrentHashMap<String, QuerySession> targetSessions, String sessionId) {
|
||||
QuerySession session = targetSessions.remove(sessionId);
|
||||
if (session == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -179,6 +179,27 @@ public final class JsonRpcServer {
|
||||
if (AgentProtocol.METHOD_CLOSE_QUERY_SESSION.equals(method)) {
|
||||
return agent.closeQuerySession(params.get("sessionId").getAsString());
|
||||
}
|
||||
if (AgentProtocol.METHOD_START_TABLE_READ.equals(method)) {
|
||||
return agent.startTableRead(
|
||||
params.get("sql").getAsString(),
|
||||
stringOrNull(params, "schema"),
|
||||
new QueryPageOptions(
|
||||
intOrDefault(params, "pageSize", 100),
|
||||
intOrNull(params, "fetchSize"),
|
||||
intOrDefault(params, "maxRows", JdbcExecutor.DEFAULT_MAX_ROWS),
|
||||
intOrDefault(params, "timeoutSecs", 0)
|
||||
)
|
||||
);
|
||||
}
|
||||
if (AgentProtocol.METHOD_FETCH_TABLE_READ_PAGE.equals(method)) {
|
||||
return agent.fetchTableReadPage(
|
||||
params.get("sessionId").getAsString(),
|
||||
intOrDefault(params, "pageSize", 100)
|
||||
);
|
||||
}
|
||||
if (AgentProtocol.METHOD_CLOSE_TABLE_READ_SESSION.equals(method)) {
|
||||
return agent.closeTableReadSession(params.get("sessionId").getAsString());
|
||||
}
|
||||
if (AgentProtocol.METHOD_GET_EXPLAIN_INFO.equals(method)) {
|
||||
String plan = agent.getExplainInfo(
|
||||
params.get("sql").getAsString(),
|
||||
@@ -199,12 +220,14 @@ public final class JsonRpcServer {
|
||||
}
|
||||
if (AgentProtocol.METHOD_DISCONNECT.equals(method)) {
|
||||
JdbcExecutor.INSTANCE.closeAllQuerySessions();
|
||||
JdbcExecutor.INSTANCE.closeAllTableReadSessions();
|
||||
agent.disconnect();
|
||||
lastConnectParams = null;
|
||||
return Collections.singletonMap("ok", true);
|
||||
}
|
||||
if (AgentProtocol.METHOD_SHUTDOWN.equals(method)) {
|
||||
JdbcExecutor.INSTANCE.closeAllQuerySessions();
|
||||
JdbcExecutor.INSTANCE.closeAllTableReadSessions();
|
||||
agent.disconnect();
|
||||
lastConnectParams = null;
|
||||
System.exit(0);
|
||||
@@ -254,6 +277,7 @@ public final class JsonRpcServer {
|
||||
}
|
||||
|
||||
JdbcExecutor.INSTANCE.closeAllQuerySessions();
|
||||
JdbcExecutor.INSTANCE.closeAllTableReadSessions();
|
||||
try {
|
||||
agent.disconnect();
|
||||
} catch (Exception ignored) {
|
||||
@@ -269,6 +293,8 @@ public final class JsonRpcServer {
|
||||
&& !AgentProtocol.METHOD_VALIDATE_CONNECTION.equals(method)
|
||||
&& !AgentProtocol.METHOD_FETCH_QUERY_PAGE.equals(method)
|
||||
&& !AgentProtocol.METHOD_CLOSE_QUERY_SESSION.equals(method)
|
||||
&& !AgentProtocol.METHOD_FETCH_TABLE_READ_PAGE.equals(method)
|
||||
&& !AgentProtocol.METHOD_CLOSE_TABLE_READ_SESSION.equals(method)
|
||||
&& !AgentProtocol.METHOD_DISCONNECT.equals(method)
|
||||
&& !AgentProtocol.METHOD_SHUTDOWN.equals(method);
|
||||
}
|
||||
|
||||
@@ -54,6 +54,18 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent {
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||
return JdbcExecutor.INSTANCE.startTableRead(
|
||||
requireConnected(),
|
||||
sql,
|
||||
schema,
|
||||
this::setSchemaSQL,
|
||||
options,
|
||||
geometryAwareResolver()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap {@link AbstractJdbcAgent#resultValue} so PostGIS-style {@code geometry}
|
||||
* and {@code geography} columns are decoded into WKT (matching the native
|
||||
|
||||
@@ -54,6 +54,9 @@
|
||||
"execute_query_page",
|
||||
"fetch_query_page",
|
||||
"close_query_session",
|
||||
"start_table_read",
|
||||
"fetch_table_read_page",
|
||||
"close_table_read_session",
|
||||
"get_explain_info",
|
||||
"execute_transaction",
|
||||
"disconnect",
|
||||
|
||||
Reference in new issue
Block a user