From 0c0e90ab1a5ba0c6f21905929ad82adfb0d1c699 Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Thu, 25 Jun 2026 18:36:34 +0800 Subject: [PATCH] feat(export): add table read cursor sessions --- .../java/com/dbx/agent/AbstractJdbcAgent.java | 22 + .../java/com/dbx/agent/AgentProtocol.java | 6 + .../java/com/dbx/agent/BaseDatabaseAgent.java | 22 + .../java/com/dbx/agent/DatabaseAgent.java | 23 + .../main/java/com/dbx/agent/JdbcExecutor.java | 117 ++- .../java/com/dbx/agent/JsonRpcServer.java | 26 + .../java/com/dbx/agent/PostgresLikeAgent.java | 12 + .../src/main/resources/agent-protocol-v1.json | 3 + .../agent/CommonJavaCompatibilityTest.java | 85 +++ .../com/dbx/agent/dameng/DamengAgent.java | 12 + .../dbx/agent/oracle10g/Oracle10gAgent.java | 12 + agents/drivers/oracle-go/main.go | 102 ++- agents/drivers/oracle-go/main_test.go | 34 + .../com/dbx/agent/oracle/OracleAgent.java | 12 + .../com/dbx/agent/tdengine/TDengineAgent.java | 12 + .../agent/test/JdbcExecutionBehaviorTest.java | 26 + crates/dbx-core/assets/agent-protocol-v1.json | 3 + crates/dbx-core/src/db/agent_driver.rs | 114 ++- crates/dbx-core/src/db/mysql.rs | 88 +++ crates/dbx-core/src/db/postgres.rs | 90 +++ crates/dbx-core/src/table_export.rs | 717 +++++++++++++++++- crates/dbx-core/src/transfer.rs | 78 +- 22 files changed, 1562 insertions(+), 54 deletions(-) diff --git a/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java b/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java index 8cd794e497..f059e7c600 100644 --- a/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/AbstractJdbcAgent.java @@ -138,6 +138,28 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent { return JdbcExecutor.INSTANCE.closeQuerySession(sessionId); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + sql, + schema, + this::setSchemaSQL, + options, + this::resultValue + ); + } + + @Override + public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) { + return JdbcExecutor.INSTANCE.fetchTableReadPage(sessionId, pageSize); + } + + @Override + public boolean closeTableReadSession(String sessionId) { + return JdbcExecutor.INSTANCE.closeTableReadSession(sessionId); + } + @Override public QueryResult executeTransaction(List statements, String schema) { return TransactionExecutor.executeUpdateStatements(requireConnected(), statements, schema, this::setSchemaSQL); diff --git a/agents/common/src/main/java/com/dbx/agent/AgentProtocol.java b/agents/common/src/main/java/com/dbx/agent/AgentProtocol.java index 23ba9bcfb5..d38063ccd2 100644 --- a/agents/common/src/main/java/com/dbx/agent/AgentProtocol.java +++ b/agents/common/src/main/java/com/dbx/agent/AgentProtocol.java @@ -26,6 +26,9 @@ public final class AgentProtocol { public static final String METHOD_EXECUTE_QUERY_PAGE = "execute_query_page"; public static final String METHOD_FETCH_QUERY_PAGE = "fetch_query_page"; public static final String METHOD_CLOSE_QUERY_SESSION = "close_query_session"; + public static final String METHOD_START_TABLE_READ = "start_table_read"; + public static final String METHOD_FETCH_TABLE_READ_PAGE = "fetch_table_read_page"; + public static final String METHOD_CLOSE_TABLE_READ_SESSION = "close_table_read_session"; public static final String METHOD_GET_EXPLAIN_INFO = "get_explain_info"; public static final String METHOD_EXECUTE_TRANSACTION = "execute_transaction"; public static final String METHOD_DISCONNECT = "disconnect"; @@ -93,6 +96,9 @@ public final class AgentProtocol { METHOD_EXECUTE_QUERY_PAGE, METHOD_FETCH_QUERY_PAGE, METHOD_CLOSE_QUERY_SESSION, + METHOD_START_TABLE_READ, + METHOD_FETCH_TABLE_READ_PAGE, + METHOD_CLOSE_TABLE_READ_SESSION, METHOD_GET_EXPLAIN_INFO, METHOD_EXECUTE_TRANSACTION, METHOD_DISCONNECT, diff --git a/agents/common/src/main/java/com/dbx/agent/BaseDatabaseAgent.java b/agents/common/src/main/java/com/dbx/agent/BaseDatabaseAgent.java index e441ed3de2..685cb824b1 100644 --- a/agents/common/src/main/java/com/dbx/agent/BaseDatabaseAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/BaseDatabaseAgent.java @@ -67,6 +67,28 @@ public abstract class BaseDatabaseAgent implements DatabaseAgent { return JdbcExecutor.INSTANCE.closeQuerySession(sessionId); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + sql, + schema, + this::setSchemaSQL, + options, + JdbcExecutor.INSTANCE::defaultResultValue + ); + } + + @Override + public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) { + return JdbcExecutor.INSTANCE.fetchTableReadPage(sessionId, pageSize); + } + + @Override + public boolean closeTableReadSession(String sessionId) { + return JdbcExecutor.INSTANCE.closeTableReadSession(sessionId); + } + @Override public QueryResult executeTransaction(List statements, String schema) { return TransactionExecutor.executeUpdateStatements(requireConnected(), statements, schema, this::setSchemaSQL); diff --git a/agents/common/src/main/java/com/dbx/agent/DatabaseAgent.java b/agents/common/src/main/java/com/dbx/agent/DatabaseAgent.java index 28172d4312..18f72b4d2c 100644 --- a/agents/common/src/main/java/com/dbx/agent/DatabaseAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/DatabaseAgent.java @@ -102,6 +102,29 @@ public interface DatabaseAgent { return JdbcExecutor.INSTANCE.closeQuerySession(sessionId); } + default QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + Connection conn = getConnection(); + if (conn == null) { + throw new IllegalStateException("Not connected"); + } + return JdbcExecutor.INSTANCE.startTableRead( + conn, + sql, + schema, + this::setSchemaSQL, + options, + JdbcExecutor.INSTANCE::defaultResultValue + ); + } + + default QueryPageResult fetchTableReadPage(String sessionId, int pageSize) { + return JdbcExecutor.INSTANCE.fetchTableReadPage(sessionId, pageSize); + } + + default boolean closeTableReadSession(String sessionId) { + return JdbcExecutor.INSTANCE.closeTableReadSession(sessionId); + } + /** * Get DM execution plan. Supports two modes: * mode="explain" (default) — direct plan, no execution diff --git a/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java b/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java index 73f7c36cf3..5c911cd95e 100644 --- a/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java +++ b/agents/common/src/main/java/com/dbx/agent/JdbcExecutor.java @@ -23,6 +23,7 @@ public final class JdbcExecutor { public static final long QUERY_SESSION_IDLE_TIMEOUT_MILLIS = 10 * 60 * 1000L; private final ConcurrentHashMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentHashMap tableReadSessions = new ConcurrentHashMap<>(); private JdbcExecutor() { } @@ -176,9 +177,32 @@ public final class JdbcExecutor { Function setSchemaSql, QueryPageOptions options, ResultValueReader valueReader + ) { + return executePage(conn, sql, schema, setSchemaSql, options, valueReader, sessions); + } + + public QueryPageResult startTableRead( + Connection conn, + String sql, + String schema, + Function setSchemaSql, + QueryPageOptions options, + ResultValueReader valueReader + ) { + return executePage(conn, sql, schema, setSchemaSql, options, valueReader, tableReadSessions); + } + + private QueryPageResult executePage( + Connection conn, + String sql, + String schema, + Function setSchemaSql, + QueryPageOptions options, + ResultValueReader valueReader, + ConcurrentHashMap targetSessions ) { return unchecked(() -> { - expireIdleQuerySessions(); + expireIdleSessions(targetSessions, System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS); String trimmedSql = trimSql(sql); String upperSql = trimmedSql.toUpperCase(Locale.ROOT).trim(); long start = System.currentTimeMillis(); @@ -242,8 +266,8 @@ public final class JdbcExecutor { Math.max(options.getMaxRows(), 1), valueReader ); - sessions.put(sessionId, session); - return readSessionPage(session, options.getPageSize(), elapsed); + targetSessions.put(sessionId, session); + return readSessionPage(targetSessions, session, options.getPageSize(), elapsed); } catch (Exception e) { try { stmt.close(); @@ -256,24 +280,28 @@ public final class JdbcExecutor { public QueryPageResult fetchPage(String sessionId, int pageSize) { expireIdleQuerySessions(); - QuerySession session = sessions.get(sessionId); - if (session == null) { - throw new IllegalArgumentException("Query session not found"); - } - synchronized (session) { - return readSessionPage(session, pageSize, 0L); - } + return fetchSessionPage(sessions, sessionId, pageSize, "Query session not found"); } public boolean closeQuerySession(String sessionId) { - return closeSession(sessionId); + return closeSession(sessions, sessionId); + } + + public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) { + expireIdleTableReadSessions(); + return fetchSessionPage(tableReadSessions, sessionId, pageSize, "Table read session not found"); + } + + public boolean closeTableReadSession(String sessionId) { + return closeSession(tableReadSessions, sessionId); } public void closeAllQuerySessions() { - List ids = new ArrayList<>(sessions.keySet()); - for (String id : ids) { - closeSession(id); - } + closeAllSessions(sessions); + } + + public void closeAllTableReadSessions() { + closeAllSessions(tableReadSessions); } public int expireIdleQuerySessions() { @@ -281,17 +309,33 @@ public final class JdbcExecutor { } public int expireIdleQuerySessions(long nowMillis, long idleTimeoutMillis) { + return expireIdleSessions(sessions, nowMillis, idleTimeoutMillis); + } + + public int expireIdleTableReadSessions() { + return expireIdleTableReadSessions(System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS); + } + + public int expireIdleTableReadSessions(long nowMillis, long idleTimeoutMillis) { + return expireIdleSessions(tableReadSessions, nowMillis, idleTimeoutMillis); + } + + private int expireIdleSessions( + ConcurrentHashMap targetSessions, + long nowMillis, + long idleTimeoutMillis + ) { if (idleTimeoutMillis < 0) { return 0; } int closed = 0; - List snapshot = new ArrayList<>(sessions.values()); + List snapshot = new ArrayList<>(targetSessions.values()); for (QuerySession session : snapshot) { boolean expired; synchronized (session) { expired = nowMillis - session.lastAccessedAtMillis >= idleTimeoutMillis; } - if (expired && closeSession(session.id)) { + if (expired && closeSession(targetSessions, session.id)) { closed += 1; } } @@ -409,7 +453,27 @@ public final class JdbcExecutor { return value == null ? null : value.getString(); } - private QueryPageResult readSessionPage(QuerySession session, int pageSize, long executionTimeMs) { + private QueryPageResult fetchSessionPage( + ConcurrentHashMap targetSessions, + String sessionId, + int pageSize, + String missingMessage + ) { + QuerySession session = targetSessions.get(sessionId); + if (session == null) { + throw new IllegalArgumentException(missingMessage); + } + synchronized (session) { + return readSessionPage(targetSessions, session, pageSize, 0L); + } + } + + private QueryPageResult readSessionPage( + ConcurrentHashMap targetSessions, + QuerySession session, + int pageSize, + long executionTimeMs + ) { return unchecked(() -> { session.lastAccessedAtMillis = System.currentTimeMillis(); int effectivePageSize = Math.max(pageSize, 1); @@ -422,7 +486,7 @@ public final class JdbcExecutor { while (rows.size() < effectivePageSize && session.rowsRead < session.maxRows) { if (!session.resultSet.next()) { - closeSession(session.id); + closeSession(targetSessions, session.id); return new QueryPageResult(session.columns, session.columnTypes, rows, 0L, executionTimeMs, false, null, false); } rows.add(rowValues(session.resultSet, session.valueReader, session.typeNameByIndex)); @@ -431,13 +495,13 @@ public final class JdbcExecutor { if (session.rowsRead >= session.maxRows) { boolean truncated = session.resultSet.next(); - closeSession(session.id); + closeSession(targetSessions, session.id); return new QueryPageResult(session.columns, session.columnTypes, rows, 0L, executionTimeMs, truncated, null, false); } boolean hasMore = session.resultSet.next(); if (!hasMore) { - closeSession(session.id); + closeSession(targetSessions, session.id); return new QueryPageResult(session.columns, session.columnTypes, rows, 0L, executionTimeMs, false, null, false); } @@ -447,8 +511,15 @@ public final class JdbcExecutor { }); } - private boolean closeSession(String sessionId) { - QuerySession session = sessions.remove(sessionId); + private void closeAllSessions(ConcurrentHashMap targetSessions) { + List ids = new ArrayList<>(targetSessions.keySet()); + for (String id : ids) { + closeSession(targetSessions, id); + } + } + + private boolean closeSession(ConcurrentHashMap targetSessions, String sessionId) { + QuerySession session = targetSessions.remove(sessionId); if (session == null) { return false; } diff --git a/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java b/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java index 4a8ec201d0..72d5c644eb 100644 --- a/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java +++ b/agents/common/src/main/java/com/dbx/agent/JsonRpcServer.java @@ -179,6 +179,27 @@ public final class JsonRpcServer { if (AgentProtocol.METHOD_CLOSE_QUERY_SESSION.equals(method)) { return agent.closeQuerySession(params.get("sessionId").getAsString()); } + if (AgentProtocol.METHOD_START_TABLE_READ.equals(method)) { + return agent.startTableRead( + params.get("sql").getAsString(), + stringOrNull(params, "schema"), + new QueryPageOptions( + intOrDefault(params, "pageSize", 100), + intOrNull(params, "fetchSize"), + intOrDefault(params, "maxRows", JdbcExecutor.DEFAULT_MAX_ROWS), + intOrDefault(params, "timeoutSecs", 0) + ) + ); + } + if (AgentProtocol.METHOD_FETCH_TABLE_READ_PAGE.equals(method)) { + return agent.fetchTableReadPage( + params.get("sessionId").getAsString(), + intOrDefault(params, "pageSize", 100) + ); + } + if (AgentProtocol.METHOD_CLOSE_TABLE_READ_SESSION.equals(method)) { + return agent.closeTableReadSession(params.get("sessionId").getAsString()); + } if (AgentProtocol.METHOD_GET_EXPLAIN_INFO.equals(method)) { String plan = agent.getExplainInfo( params.get("sql").getAsString(), @@ -199,12 +220,14 @@ public final class JsonRpcServer { } if (AgentProtocol.METHOD_DISCONNECT.equals(method)) { JdbcExecutor.INSTANCE.closeAllQuerySessions(); + JdbcExecutor.INSTANCE.closeAllTableReadSessions(); agent.disconnect(); lastConnectParams = null; return Collections.singletonMap("ok", true); } if (AgentProtocol.METHOD_SHUTDOWN.equals(method)) { JdbcExecutor.INSTANCE.closeAllQuerySessions(); + JdbcExecutor.INSTANCE.closeAllTableReadSessions(); agent.disconnect(); lastConnectParams = null; System.exit(0); @@ -254,6 +277,7 @@ public final class JsonRpcServer { } JdbcExecutor.INSTANCE.closeAllQuerySessions(); + JdbcExecutor.INSTANCE.closeAllTableReadSessions(); try { agent.disconnect(); } catch (Exception ignored) { @@ -269,6 +293,8 @@ public final class JsonRpcServer { && !AgentProtocol.METHOD_VALIDATE_CONNECTION.equals(method) && !AgentProtocol.METHOD_FETCH_QUERY_PAGE.equals(method) && !AgentProtocol.METHOD_CLOSE_QUERY_SESSION.equals(method) + && !AgentProtocol.METHOD_FETCH_TABLE_READ_PAGE.equals(method) + && !AgentProtocol.METHOD_CLOSE_TABLE_READ_SESSION.equals(method) && !AgentProtocol.METHOD_DISCONNECT.equals(method) && !AgentProtocol.METHOD_SHUTDOWN.equals(method); } diff --git a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java index 43d6e37f51..977f2ffd34 100644 --- a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java @@ -54,6 +54,18 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { ); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + sql, + schema, + this::setSchemaSQL, + options, + geometryAwareResolver() + ); + } + /** * Wrap {@link AbstractJdbcAgent#resultValue} so PostGIS-style {@code geometry} * and {@code geography} columns are decoded into WKT (matching the native diff --git a/agents/common/src/main/resources/agent-protocol-v1.json b/agents/common/src/main/resources/agent-protocol-v1.json index 7e30f9c9f6..77d03637fe 100644 --- a/agents/common/src/main/resources/agent-protocol-v1.json +++ b/agents/common/src/main/resources/agent-protocol-v1.json @@ -54,6 +54,9 @@ "execute_query_page", "fetch_query_page", "close_query_session", + "start_table_read", + "fetch_table_read_page", + "close_table_read_session", "get_explain_info", "execute_transaction", "disconnect", diff --git a/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java b/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java index 3ec366cd9e..304fc9ec04 100644 --- a/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java +++ b/agents/common/src/test/java/com/dbx/agent/CommonJavaCompatibilityTest.java @@ -122,6 +122,42 @@ class CommonJavaCompatibilityTest { assertEquals("app", agent.lastSchema); } + @Test + void jsonRpcServerDispatchesTableReadSessionMethods() { + TableReadDispatchAgent agent = new TableReadDispatchAgent(); + JsonRpcServer server = new JsonRpcServer(agent); + + String startResponse = server.handleRequest( + "{\"jsonrpc\":\"2.0\",\"id\":11,\"method\":\"" + AgentProtocol.METHOD_START_TABLE_READ + "\",\"params\":{\"sql\":\"select * from orders\",\"schema\":\"public\",\"pageSize\":2,\"fetchSize\":8,\"maxRows\":20,\"timeoutSecs\":3}}" + ); + JsonObject startJson = JsonParser.parseString(startResponse).getAsJsonObject(); + + assertTrue(startJson.has("result")); + assertEquals("select * from orders", agent.lastSql); + assertEquals("public", agent.lastSchema); + assertEquals(new QueryPageOptions(2, 8, 20, 3), agent.lastOptions); + assertEquals("table-session", startJson.getAsJsonObject("result").get("session_id").getAsString()); + + String fetchResponse = server.handleRequest( + "{\"jsonrpc\":\"2.0\",\"id\":12,\"method\":\"" + AgentProtocol.METHOD_FETCH_TABLE_READ_PAGE + "\",\"params\":{\"sessionId\":\"table-session\",\"pageSize\":4}}" + ); + JsonObject fetchJson = JsonParser.parseString(fetchResponse).getAsJsonObject(); + + assertTrue(fetchJson.has("result")); + assertEquals("table-session", agent.fetchedSessionId); + assertEquals(4, agent.fetchedPageSize); + assertFalse(fetchJson.getAsJsonObject("result").get("has_more").getAsBoolean()); + + String closeResponse = server.handleRequest( + "{\"jsonrpc\":\"2.0\",\"id\":13,\"method\":\"" + AgentProtocol.METHOD_CLOSE_TABLE_READ_SESSION + "\",\"params\":{\"sessionId\":\"table-session\"}}" + ); + JsonObject closeJson = JsonParser.parseString(closeResponse).getAsJsonObject(); + + assertTrue(closeJson.get("result").getAsBoolean()); + assertEquals("table-session", agent.closedSessionId); + } + + @Test void exposesJavaFriendlyDefaultsAndModels() { ConnectParams params = new ConnectParams("localhost", 5432, "demo", "user", "secret", "ssl=false", "", false); @@ -170,6 +206,9 @@ class CommonJavaCompatibilityTest { assertThrows(IllegalStateException.class, () -> agent.executeQueryPage("select 1", "public") ); + assertThrows(IllegalStateException.class, () -> + agent.startTableRead("select 1", "public", new QueryPageOptions()) + ); assertEquals(0L, agent.executeQuery("select 1", "public").getAffected_rows()); String ddl = DatabaseAgent.buildTableDdl( @@ -393,6 +432,52 @@ class CommonJavaCompatibilityTest { } } + private static final class TableReadDispatchAgent extends MinimalAgent { + private String lastSql; + private String lastSchema; + private QueryPageOptions lastOptions; + private String fetchedSessionId; + private int fetchedPageSize; + private String closedSessionId; + + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + lastSql = sql; + lastSchema = schema; + lastOptions = options; + return new QueryPageResult( + Collections.singletonList("id"), + Collections.singletonList(Collections.singletonList(1)), + 0L, + 5L, + false, + "table-session", + true + ); + } + + @Override + public QueryPageResult fetchTableReadPage(String sessionId, int pageSize) { + fetchedSessionId = sessionId; + fetchedPageSize = pageSize; + return new QueryPageResult( + Collections.singletonList("id"), + Collections.singletonList(Collections.singletonList(2)), + 0L, + 0L, + false, + null, + false + ); + } + + @Override + public boolean closeTableReadSession(String sessionId) { + closedSessionId = sessionId; + return "table-session".equals(sessionId); + } + } + private static Connection nonTransactionalConnection(List calls) { return proxy(Connection.class, (method, args) -> { String name = method.getName(); diff --git a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java index 3ff1919edd..959f8f26de 100644 --- a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java +++ b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java @@ -421,6 +421,18 @@ public final class DamengAgent extends BaseDatabaseAgent { ); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + sql, + schema, + this::setSchemaSQL, + options, + this::stringResultValue + ); + } + @Override public String setSchemaSQL(String schema) { return "SET SCHEMA " + JdbcIdentifiers.INSTANCE.doubleQuote(schema); diff --git a/agents/drivers/oracle-10g/src/main/java/com/dbx/agent/oracle10g/Oracle10gAgent.java b/agents/drivers/oracle-10g/src/main/java/com/dbx/agent/oracle10g/Oracle10gAgent.java index 49422c3a14..5650701598 100644 --- a/agents/drivers/oracle-10g/src/main/java/com/dbx/agent/oracle10g/Oracle10gAgent.java +++ b/agents/drivers/oracle-10g/src/main/java/com/dbx/agent/oracle10g/Oracle10gAgent.java @@ -470,6 +470,18 @@ public final class Oracle10gAgent extends BaseDatabaseAgent { ); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + prepareExecutableSql(sql), + schema, + this::setSchemaSQL, + options, + this::stringResultValue + ); + } + @Override public void disconnect() { uncheckedVoid(() -> { diff --git a/agents/drivers/oracle-go/main.go b/agents/drivers/oracle-go/main.go index 7435578eb7..cb85b5e41a 100644 --- a/agents/drivers/oracle-go/main.go +++ b/agents/drivers/oracle-go/main.go @@ -238,10 +238,12 @@ type triggerInfo struct { } type server struct { - db *sql.DB - params connectParams - sessions map[string]*querySession - nextSessionID int64 + db *sql.DB + params connectParams + sessions map[string]*querySession + tableReadSessions map[string]*querySession + nextSessionID int64 + nextTableReadSessionID int64 } func main() { @@ -271,7 +273,7 @@ func main() { } func newServer() *server { - return &server{sessions: map[string]*querySession{}} + return &server{sessions: map[string]*querySession{}, tableReadSessions: map[string]*querySession{}} } func (s *server) handleLine(line string) (response, bool) { @@ -369,6 +371,18 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any return result, false, err case "close_query_session": return s.closeQuerySession(stringParam(params, "sessionId")), false, nil + case "start_table_read": + var opts queryOptions + if err := decodeParams(params, &opts); err != nil { + return nil, false, err + } + result, err := s.startTableRead(opts, intParam(params, "pageSize")) + return result, false, err + case "fetch_table_read_page": + result, err := s.fetchTableReadPage(stringParam(params, "sessionId"), intParam(params, "pageSize")) + return result, false, err + case "close_table_read_session": + return s.closeTableReadSession(stringParam(params, "sessionId")), false, nil case "list_indexes": schema := stringParam(params, "schema") table := stringParam(params, "table") @@ -1241,6 +1255,71 @@ func (s *server) storeQuerySession(session *querySession) string { return sessionID } +func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResult, error) { + start := time.Now() + if strings.TrimSpace(opts.Schema) != "" { + if err := s.setSchema(opts.Schema); err != nil { + return queryPageResult{}, err + } + } + sqlText := trimStatementSQL(opts.SQL) + if !isQuerySQL(sqlText) { + return queryPageResult{}, errors.New("table read requires a SELECT query") + } + rows, err := s.queryRows(sqlText, nil) + if err != nil { + return queryPageResult{}, err + } + columns, err := rows.Columns() + if err != nil { + rows.Close() + return queryPageResult{}, err + } + maxRows := opts.MaxRows + if maxRows <= 0 { + maxRows = defaultMaxRows + } + session := &querySession{rows: rows, columns: columns, remaining: maxRows} + result, err := readQuerySessionPage(session, pageSize) + result.ExecutionTimeMS = time.Since(start).Milliseconds() + if err != nil { + rows.Close() + return queryPageResult{}, err + } + if result.HasMore { + sessionID := s.storeTableReadSession(session) + result.SessionID = &sessionID + } else { + rows.Close() + } + return result, nil +} + +func (s *server) fetchTableReadPage(sessionID string, pageSize int) (queryPageResult, error) { + session := s.tableReadSessions[sessionID] + if session == nil { + return queryPageResult{Columns: []string{}, Rows: [][]any{}, SessionID: nil, HasMore: false}, nil + } + result, err := readQuerySessionPage(session, pageSize) + if err != nil { + s.closeTableReadSession(sessionID) + return queryPageResult{}, err + } + if result.HasMore { + result.SessionID = &sessionID + } else { + s.closeTableReadSession(sessionID) + } + return result, nil +} + +func (s *server) storeTableReadSession(session *querySession) string { + s.nextTableReadSessionID++ + sessionID := fmt.Sprintf("oracle-go-table-%d", s.nextTableReadSessionID) + s.tableReadSessions[sessionID] = session + return sessionID +} + func (s *server) closeQuerySession(sessionID string) bool { session := s.sessions[sessionID] if session == nil { @@ -1251,10 +1330,23 @@ func (s *server) closeQuerySession(sessionID string) bool { return true } +func (s *server) closeTableReadSession(sessionID string) bool { + session := s.tableReadSessions[sessionID] + if session == nil { + return false + } + session.rows.Close() + delete(s.tableReadSessions, sessionID) + return true +} + func (s *server) closeAllQuerySessions() { for sessionID := range s.sessions { s.closeQuerySession(sessionID) } + for sessionID := range s.tableReadSessions { + s.closeTableReadSession(sessionID) + } } func readQuerySessionPage(session *querySession, pageSize int) (queryPageResult, error) { diff --git a/agents/drivers/oracle-go/main_test.go b/agents/drivers/oracle-go/main_test.go index 58b1803078..53131c5be7 100644 --- a/agents/drivers/oracle-go/main_test.go +++ b/agents/drivers/oracle-go/main_test.go @@ -60,6 +60,40 @@ func TestCloseMissingQuerySessionReturnsFalse(t *testing.T) { } } +func TestMissingTableReadSessionMethodsReturnEmptyOrFalse(t *testing.T) { + s := newServer() + + fetchResp, shutdown := s.handleLine(`{"jsonrpc":"2.0","id":9,"method":"fetch_table_read_page","params":{"sessionId":"missing","pageSize":10}}`) + if shutdown { + t.Fatal("fetch_table_read_page should not shut down the server") + } + if fetchResp.Error != nil { + t.Fatalf("unexpected fetch error: %v", fetchResp.Error) + } + data, err := json.Marshal(fetchResp.Result) + if err != nil { + t.Fatal(err) + } + var page queryPageResult + if err := json.Unmarshal(data, &page); err != nil { + t.Fatal(err) + } + if len(page.Columns) != 0 || len(page.Rows) != 0 || page.HasMore || page.SessionID != nil { + t.Fatalf("missing table read session should return empty page, got %+v", page) + } + + closeResp, shutdown := s.handleLine(`{"jsonrpc":"2.0","id":10,"method":"close_table_read_session","params":{"sessionId":"missing"}}`) + if shutdown { + t.Fatal("close_table_read_session should not shut down the server") + } + if closeResp.Error != nil { + t.Fatalf("unexpected close error: %v", closeResp.Error) + } + if closeResp.Result != false { + t.Fatalf("expected false result, got %#v", closeResp.Result) + } +} + func TestEmptyResultSlicesMarshalAsArrays(t *testing.T) { data, err := json.Marshal(queryResult{}) if err != nil { diff --git a/agents/drivers/oracle/src/main/java/com/dbx/agent/oracle/OracleAgent.java b/agents/drivers/oracle/src/main/java/com/dbx/agent/oracle/OracleAgent.java index def419b7e7..3cacc98c18 100644 --- a/agents/drivers/oracle/src/main/java/com/dbx/agent/oracle/OracleAgent.java +++ b/agents/drivers/oracle/src/main/java/com/dbx/agent/oracle/OracleAgent.java @@ -575,6 +575,18 @@ public class OracleAgent extends BaseDatabaseAgent { ); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + prepareExecutableSql(sql), + schema, + this::setSchemaSQL, + options, + OracleAgent::stringResultValue + ); + } + @Override public void disconnect() { uncheckedVoid(() -> { diff --git a/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java b/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java index fb48ce0821..19839efcf9 100644 --- a/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java +++ b/agents/drivers/tdengine/src/main/java/com/dbx/agent/tdengine/TDengineAgent.java @@ -191,6 +191,18 @@ public final class TDengineAgent extends BaseDatabaseAgent { ); } + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return JdbcExecutor.INSTANCE.startTableRead( + requireConnected(), + sql, + schema, + this::setSchemaSQL, + options, + this::tdengineResultValue + ); + } + @Override public String setSchemaSQL(String schema) { return "USE " + quoteIdentifier(schema); diff --git a/agents/test-support/src/main/java/com/dbx/agent/test/JdbcExecutionBehaviorTest.java b/agents/test-support/src/main/java/com/dbx/agent/test/JdbcExecutionBehaviorTest.java index c15d377dcc..6764a5ffbe 100644 --- a/agents/test-support/src/main/java/com/dbx/agent/test/JdbcExecutionBehaviorTest.java +++ b/agents/test-support/src/main/java/com/dbx/agent/test/JdbcExecutionBehaviorTest.java @@ -85,6 +85,32 @@ public abstract class JdbcExecutionBehaviorTest extends JdbcConnectedAgentTest { }); } + @Test + void fetchesTableReadPagesFromTheSameJdbcSession() { + withAgent("dbx-agent-table-read-session-pages", agent -> { + QueryPageResult first = agent.startTableRead(rowsSql(5), null, new QueryPageOptions(2, 2, 10)); + + assertEquals(Arrays.asList(Collections.singletonList(1L), Collections.singletonList(2L)), first.getRows()); + assertTrue(first.getHas_more()); + assertFalse(first.getTruncated()); + String sessionId = first.getSession_id(); + assertNotNull(sessionId); + + QueryPageResult second = agent.fetchTableReadPage(sessionId, 2); + + assertEquals(Arrays.asList(Collections.singletonList(3L), Collections.singletonList(4L)), second.getRows()); + assertTrue(second.getHas_more()); + assertFalse(second.getTruncated()); + + QueryPageResult third = agent.fetchTableReadPage(sessionId, 2); + + assertEquals(Collections.singletonList(Collections.singletonList(5L)), third.getRows()); + assertFalse(third.getHas_more()); + assertFalse(third.getTruncated()); + assertFalse(agent.closeTableReadSession(sessionId)); + }); + } + @Test void closesJdbcResultSessionsExplicitly() { withAgent("dbx-agent-session-close", agent -> { diff --git a/crates/dbx-core/assets/agent-protocol-v1.json b/crates/dbx-core/assets/agent-protocol-v1.json index 7e30f9c9f6..77d03637fe 100644 --- a/crates/dbx-core/assets/agent-protocol-v1.json +++ b/crates/dbx-core/assets/agent-protocol-v1.json @@ -54,6 +54,9 @@ "execute_query_page", "fetch_query_page", "close_query_session", + "start_table_read", + "fetch_table_read_page", + "close_table_read_session", "get_explain_info", "execute_transaction", "disconnect", diff --git a/crates/dbx-core/src/db/agent_driver.rs b/crates/dbx-core/src/db/agent_driver.rs index fc7c724b51..ca5132368b 100644 --- a/crates/dbx-core/src/db/agent_driver.rs +++ b/crates/dbx-core/src/db/agent_driver.rs @@ -131,6 +131,9 @@ pub enum AgentMethod { ExecuteQueryPage, FetchQueryPage, CloseQuerySession, + StartTableRead, + FetchTableReadPage, + CloseTableReadSession, GetExplainInfo, ExecuteTransaction, Disconnect, @@ -138,7 +141,7 @@ pub enum AgentMethod { } impl AgentMethod { - pub const ALL: [Self; 23] = [ + pub const ALL: [Self; 26] = [ Self::Handshake, Self::Connect, Self::TestConnection, @@ -158,6 +161,9 @@ impl AgentMethod { Self::ExecuteQueryPage, Self::FetchQueryPage, Self::CloseQuerySession, + Self::StartTableRead, + Self::FetchTableReadPage, + Self::CloseTableReadSession, Self::GetExplainInfo, Self::ExecuteTransaction, Self::Disconnect, @@ -185,6 +191,9 @@ impl AgentMethod { Self::ExecuteQueryPage => "execute_query_page", Self::FetchQueryPage => "fetch_query_page", Self::CloseQuerySession => "close_query_session", + Self::StartTableRead => "start_table_read", + Self::FetchTableReadPage => "fetch_table_read_page", + Self::CloseTableReadSession => "close_table_read_session", Self::GetExplainInfo => "get_explain_info", Self::ExecuteTransaction => "execute_transaction", Self::Disconnect => "disconnect", @@ -193,6 +202,33 @@ impl AgentMethod { } } +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentTableReadStartParams { + pub sql: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub database: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub page_size: usize, + pub max_rows: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub fetch_size: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentTableReadPageParams { + pub session_id: String, + pub page_size: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentTableReadCloseParams { + pub session_id: String, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MongoAgentMethod { ListDatabases, @@ -764,6 +800,38 @@ impl AgentDriverClient { self.call_method(AgentMethod::CloseQuerySession, agent_close_query_session_params(session_id)).await } + pub async fn start_table_read( + &mut self, + params: AgentTableReadStartParams, + ) -> Result { + self.call_method(AgentMethod::StartTableRead, serde_json::to_value(params).map_err(|e| e.to_string())?).await + } + + pub async fn fetch_table_read_page( + &mut self, + session_id: &str, + page_size: usize, + ) -> Result { + self.call_method( + AgentMethod::FetchTableReadPage, + serde_json::to_value(AgentTableReadPageParams { session_id: session_id.to_string(), page_size }) + .map_err(|e| e.to_string())?, + ) + .await + } + + pub async fn close_table_read_session( + &mut self, + session_id: &str, + ) -> Result { + self.call_method( + AgentMethod::CloseTableReadSession, + serde_json::to_value(AgentTableReadCloseParams { session_id: session_id.to_string() }) + .map_err(|e| e.to_string())?, + ) + .await + } + pub async fn execute_transaction( &mut self, database: Option<&str>, @@ -1116,7 +1184,8 @@ mod tests { agent_proxy_env_vars, agent_schema_params, agent_schema_table_params, agent_supports_capability, agent_transaction_params, format_agent_process_error, is_unsupported_handshake_error, mongo_collection_params, mongo_database_params, mongo_document_id_params, read_agent_line, AgentCapability, AgentDriverClient, - AgentHandshake, AgentKvMethod, AgentMethod, MongoAgentMethod, StderrTail, AGENT_PROTOCOL_VERSION, + AgentHandshake, AgentKvMethod, AgentMethod, AgentTableReadCloseParams, AgentTableReadPageParams, + AgentTableReadStartParams, MongoAgentMethod, StderrTail, AGENT_PROTOCOL_VERSION, }; use std::io::Cursor; @@ -1280,6 +1349,9 @@ mod tests { assert_eq!(AgentMethod::ExecuteQueryPage.as_str(), "execute_query_page"); assert_eq!(AgentMethod::FetchQueryPage.as_str(), "fetch_query_page"); assert_eq!(AgentMethod::CloseQuerySession.as_str(), "close_query_session"); + assert_eq!(AgentMethod::StartTableRead.as_str(), "start_table_read"); + assert_eq!(AgentMethod::FetchTableReadPage.as_str(), "fetch_table_read_page"); + assert_eq!(AgentMethod::CloseTableReadSession.as_str(), "close_table_read_session"); assert_eq!(AgentMethod::ExecuteTransaction.as_str(), "execute_transaction"); assert_eq!(AgentMethod::Disconnect.as_str(), "disconnect"); assert_eq!(AgentMethod::Shutdown.as_str(), "shutdown"); @@ -1338,6 +1410,44 @@ mod tests { let _call_kv_method = AgentDriverClient::call_kv_method::; } + #[test] + fn exposes_table_read_protocol_wrappers() { + let _start_table_read = AgentDriverClient::start_table_read::; + let _fetch_table_read_page = AgentDriverClient::fetch_table_read_page::; + let _close_table_read_session = AgentDriverClient::close_table_read_session::; + } + + #[test] + fn serializes_table_read_params_with_agent_field_names() { + let start = serde_json::to_value(AgentTableReadStartParams { + sql: "SELECT * FROM users".to_string(), + database: Some("ORCL".to_string()), + schema: Some("APP".to_string()), + page_size: 500, + max_rows: 1000, + fetch_size: Some(500), + }) + .unwrap(); + assert_eq!( + start, + serde_json::json!({ + "sql": "SELECT * FROM users", + "database": "ORCL", + "schema": "APP", + "pageSize": 500, + "maxRows": 1000, + "fetchSize": 500, + }) + ); + + let page = serde_json::to_value(AgentTableReadPageParams { session_id: "table-1".to_string(), page_size: 250 }) + .unwrap(); + assert_eq!(page, serde_json::json!({ "sessionId": "table-1", "pageSize": 250 })); + + let close = serde_json::to_value(AgentTableReadCloseParams { session_id: "table-1".to_string() }).unwrap(); + assert_eq!(close, serde_json::json!({ "sessionId": "table-1" })); + } + #[test] fn agent_query_result_default_column_types_is_empty_vec() { // Old agent JARs predate the column_types field. Rust tolerates the diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index 13087def20..2c7b9d2570 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -7,6 +7,7 @@ use rust_decimal::Decimal; use std::borrow::Cow; use std::collections::HashMap; use std::path::PathBuf; +use std::sync::atomic::AtomicBool; use std::time::Duration; use std::time::Instant; @@ -2044,6 +2045,93 @@ pub async fn execute_query_with_max_rows( execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await } +pub async fn stream_query_rows( + pool: &MySqlPool, + sql: &str, + bare: bool, + max_rows: Option, + dialect: MySqlQueryDialect, + cancelled: &AtomicBool, + mut on_row: impl FnMut(&[serde_json::Value]) -> Result<(), String>, +) -> Result { + let mut conn = get_conn_with_health_check(pool).await?; + let row_limit = max_rows.unwrap_or(usize::MAX); + + if bare || prefers_text_protocol_query(sql, dialect) { + stream_query_rows_text(&mut conn, sql, row_limit, cancelled, &mut on_row).await + } else { + match stream_query_rows_prepared(&mut conn, sql, row_limit, cancelled, &mut on_row).await { + Ok(rows) => Ok(rows), + Err(err) if mysql_error_should_retry_with_text_protocol(&err) => { + stream_query_rows_text(&mut conn, sql, row_limit, cancelled, &mut on_row).await + } + Err(err) => Err(err), + } + } +} + +async fn stream_query_rows_text( + conn: &mut mysql_async::Conn, + sql: &str, + row_limit: usize, + cancelled: &AtomicBool, + on_row: &mut impl FnMut(&[serde_json::Value]) -> Result<(), String>, +) -> Result { + let mut result = conn.query_iter(sql).await.map_err(|e| e.to_string())?; + let mut stream = result + .stream::() + .await + .map_err(|e| e.to_string())? + .ok_or_else(|| "Empty result set stream".to_string())?; + let mut rows_exported = 0_u64; + + while let Some(row) = stream.next().await { + if cancelled.load(std::sync::atomic::Ordering::SeqCst) { + return Err(crate::query::canceled_error()); + } + if rows_exported as usize >= row_limit { + break; + } + let row = row.map_err(|e| e.to_string())?; + let values: Vec = (0..row.len()).map(|i| mysql_value_to_json(&row, i)).collect(); + on_row(&values)?; + rows_exported += 1; + } + + Ok(rows_exported) +} + +async fn stream_query_rows_prepared( + conn: &mut mysql_async::Conn, + sql: &str, + row_limit: usize, + cancelled: &AtomicBool, + on_row: &mut impl FnMut(&[serde_json::Value]) -> Result<(), String>, +) -> Result { + let mut result = conn.exec_iter(sql, ()).await.map_err(|e| e.to_string())?; + let mut stream = result + .stream::() + .await + .map_err(|e| e.to_string())? + .ok_or_else(|| "Empty result set stream".to_string())?; + let mut rows_exported = 0_u64; + + while let Some(row) = stream.next().await { + if cancelled.load(std::sync::atomic::Ordering::SeqCst) { + return Err(crate::query::canceled_error()); + } + if rows_exported as usize >= row_limit { + break; + } + let row = row.map_err(|e| e.to_string())?; + let values: Vec = (0..row.len()).map(|i| mysql_value_to_json(&row, i)).collect(); + on_row(&values)?; + rows_exported += 1; + } + + Ok(rows_exported) +} + pub async fn kill_query(pool: &MySqlPool, connection_id: u32) -> Result<(), String> { let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?; conn.query_drop(format!("KILL QUERY {connection_id}")).await.map_err(|e| e.to_string()) diff --git a/crates/dbx-core/src/db/postgres.rs b/crates/dbx-core/src/db/postgres.rs index 33e397bd73..8d3726ccb1 100644 --- a/crates/dbx-core/src/db/postgres.rs +++ b/crates/dbx-core/src/db/postgres.rs @@ -12,6 +12,7 @@ use std::fs::File; use std::future::Future; use std::io::BufReader; use std::str::FromStr; +use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio_postgres::config::SslMode; @@ -555,6 +556,10 @@ fn pg_pool_error_to_string(err: PoolError) -> String { fn should_retry_postgres_text_query(err: &tokio_postgres::Error) -> bool { let message = err.as_db_error().map(ToString::to_string).unwrap_or_else(|| err.to_string()).to_ascii_lowercase(); + should_retry_postgres_text_query_message(&message) +} + +fn should_retry_postgres_text_query_message(message: &str) -> bool { message.contains("no binary output function") || message.contains("no binary send function") || message.contains("cannot display a value of type") @@ -706,6 +711,91 @@ async fn execute_select_query( } } +pub async fn stream_query_rows( + pool: &Pool, + sql: &str, + max_rows: Option, + cancelled: &AtomicBool, + mut on_row: impl FnMut(&[serde_json::Value]) -> Result<(), String>, +) -> Result { + let client = + pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?; + match stream_query_rows_on_client(&client, sql, max_rows, cancelled, &mut on_row).await { + Ok(rows) => Ok(rows), + Err(error) if should_retry_postgres_text_query_message(&error.to_ascii_lowercase()) => { + stream_query_rows_text_on_client(&client, sql, max_rows, cancelled, &mut on_row).await + } + Err(error) => Err(error), + } +} + +async fn stream_query_rows_on_client( + client: &deadpool_postgres::Client, + sql: &str, + max_rows: Option, + cancelled: &AtomicBool, + on_row: &mut impl FnMut(&[serde_json::Value]) -> Result<(), String>, +) -> Result { + let stmt = client.prepare_cached(sql).await.map_err(pg_error_to_string)?; + let column_types: Vec = stmt.columns().iter().map(|c| c.type_().name().to_string()).collect(); + let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new(); + let stream = client.query_raw(&stmt, params).await.map_err(pg_error_to_string)?; + tokio::pin!(stream); + let row_limit = max_rows.unwrap_or(usize::MAX); + let mut rows_exported = 0_u64; + + while let Some(row_result) = stream.next().await { + if cancelled.load(std::sync::atomic::Ordering::SeqCst) { + return Err(crate::query::canceled_error()); + } + if rows_exported as usize >= row_limit { + break; + } + let row = row_result.map_err(pg_error_to_string)?; + let values: Vec = (0..row.columns().len()) + .map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or(""))) + .collect(); + on_row(&values)?; + rows_exported += 1; + } + + Ok(rows_exported) +} + +async fn stream_query_rows_text_on_client( + client: &deadpool_postgres::Client, + sql: &str, + max_rows: Option, + cancelled: &AtomicBool, + on_row: &mut impl FnMut(&[serde_json::Value]) -> Result<(), String>, +) -> Result { + let messages = client.simple_query(sql).await.map_err(pg_error_to_string)?; + let row_limit = max_rows.unwrap_or(usize::MAX); + let mut rows_exported = 0_u64; + + for message in messages { + if cancelled.load(std::sync::atomic::Ordering::SeqCst) { + return Err(crate::query::canceled_error()); + } + if rows_exported as usize >= row_limit { + break; + } + if let SimpleQueryMessage::Row(row) = message { + let mut values = Vec::with_capacity(row.len()); + for i in 0..row.len() { + values.push(match row.try_get(i).map_err(pg_error_to_string)? { + Some(value) => serde_json::Value::String(value.to_string()), + None => serde_json::Value::Null, + }); + } + on_row(&values)?; + rows_exported += 1; + } + } + + Ok(rows_exported) +} + pub async fn connect(url: &str, fallback_timeout: Duration) -> Result { let postgres_url = postgres_connection_url(url)?; let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index cf7c5d56bd..d1678d5e46 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -1,18 +1,27 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::io::{BufWriter, Write}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio_util::sync::CancellationToken; -use crate::connection::AppState; +use crate::connection::MysqlMode; +use crate::connection::{AppState, PoolKind}; use crate::csv_export::{escape_csv, format_csv, value_to_csv_text}; pub use crate::database_export::ExportStatus; use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions}; +use crate::db::agent_driver::AgentTableReadStartParams; +use crate::models::connection::DatabaseType; use crate::transfer::{ count_sql_with_where, execute_on_pool, execute_on_pool_with_max_rows, keyset_pagination_sql, - pagination_sql_with_filter_order, + pagination_sql_with_filter_order, qualified_table, quote_identifier, }; +use crate::types::QueryResult; use crate::xlsx_export::{finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook}; const DEFAULT_BATCH_SIZE: usize = 10_000; +const SQL_INSERT_BATCH_SIZE: usize = 100; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -116,7 +125,7 @@ fn format_markdown_rows(rows: &[Vec]) -> String { #[allow(clippy::too_many_arguments)] fn table_page_sql( request: &TableExportRequest, - db_type: &crate::models::connection::DatabaseType, + db_type: &DatabaseType, col_names: &[String], primary_keys: &[String], use_keyset: bool, @@ -149,6 +158,581 @@ fn table_page_sql( } } +fn table_cursor_sql( + request: &TableExportRequest, + db_type: &DatabaseType, + col_names: &[String], + primary_keys: &[String], +) -> String { + let full_table = qualified_table(&request.table_name, request.schema.as_deref().unwrap_or(""), db_type); + let col_list = col_names.iter().map(|column| quote_identifier(column, db_type)).collect::>().join(", "); + let predicate = crate::sql_dialect::normalize_where_input(request.where_input.as_deref()); + let where_clause = if predicate.is_empty() { String::new() } else { format!(" WHERE ({predicate})") }; + let order_by = request + .order_by + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| { + if primary_keys.is_empty() { + None + } else { + Some( + primary_keys + .iter() + .map(|column| format!("{} ASC", quote_identifier(column, db_type))) + .collect::>() + .join(", "), + ) + } + }) + .map(|value| format!(" ORDER BY {value}")) + .unwrap_or_default(); + + format!("SELECT {col_list} FROM {full_table}{where_clause}{order_by}") +} + +fn is_agent_table_read_unsupported(error: &str) -> bool { + let lower = error.to_ascii_lowercase(); + lower.contains("unknown method") || lower.contains("method not found") +} + +async fn pool_is_agent(state: &AppState, pool_key: &str) -> bool { + let connections = state.connections.read().await; + matches!(connections.get(pool_key), Some(PoolKind::Agent(_))) +} + +#[allow(clippy::too_many_arguments)] +async fn fetch_table_export_batch( + state: &AppState, + pool_key: &str, + request: &TableExportRequest, + db_type: &DatabaseType, + col_names: &[String], + primary_keys: &[String], + use_keyset: bool, + last_pk_values: &[Value], + offset: u64, + active_batch_size: usize, + table_read_session_id: &mut Option, + table_read_attempted: &mut bool, + table_read_completed: &mut bool, +) -> Result { + if *table_read_completed { + return Ok(QueryResult { + columns: col_names.to_vec(), + column_types: Vec::new(), + column_sortables: Vec::new(), + rows: Vec::new(), + affected_rows: 0, + execution_time_ms: 0, + truncated: false, + session_id: None, + has_more: false, + }); + } + + if !*table_read_attempted && pool_is_agent(state, pool_key).await { + *table_read_attempted = true; + let sql = table_cursor_sql(request, db_type, col_names, primary_keys); + let max_rows = request.row_limit.unwrap_or(i32::MAX as usize); + let params = AgentTableReadStartParams { + sql, + database: Some(request.database.clone()), + schema: request.schema.clone(), + page_size: active_batch_size, + max_rows, + fetch_size: Some(active_batch_size), + }; + let connections = state.connections.read().await; + let Some(PoolKind::Agent(client)) = connections.get(pool_key) else { + drop(connections); + return fetch_paginated_table_export_batch( + state, + pool_key, + request, + db_type, + col_names, + primary_keys, + use_keyset, + last_pk_values, + offset, + active_batch_size, + ) + .await; + }; + let client = client.clone(); + drop(connections); + let mut client = client.lock().await; + match client.start_table_read::(params).await { + Ok(result) => { + *table_read_session_id = result.session_id.clone(); + if result.session_id.is_none() && !result.has_more { + *table_read_completed = true; + } + return Ok(result); + } + Err(error) if is_agent_table_read_unsupported(&error) => { + log::debug!("Agent table-read cursor unsupported, falling back to paginated export: {error}"); + } + Err(error) => return Err(error), + } + } + + if let Some(session_id) = table_read_session_id.as_deref() { + let connections = state.connections.read().await; + let Some(PoolKind::Agent(client)) = connections.get(pool_key) else { + return Err("Table read session requires an agent connection".to_string()); + }; + let client = client.clone(); + drop(connections); + let mut client = client.lock().await; + return match client.fetch_table_read_page::(session_id, active_batch_size).await { + Ok(result) => { + *table_read_session_id = result.session_id.clone().or_else(|| Some(session_id.to_string())); + if !result.has_more { + *table_read_session_id = None; + *table_read_completed = true; + } + Ok(result) + } + Err(error) => { + let _ = client.close_table_read_session::(session_id).await; + *table_read_session_id = None; + Err(error) + } + }; + } + + fetch_paginated_table_export_batch( + state, + pool_key, + request, + db_type, + col_names, + primary_keys, + use_keyset, + last_pk_values, + offset, + active_batch_size, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn fetch_paginated_table_export_batch( + state: &AppState, + pool_key: &str, + request: &TableExportRequest, + db_type: &DatabaseType, + col_names: &[String], + primary_keys: &[String], + use_keyset: bool, + last_pk_values: &[Value], + offset: u64, + active_batch_size: usize, +) -> Result { + let sql = table_page_sql( + request, + db_type, + col_names, + primary_keys, + use_keyset, + last_pk_values, + offset, + active_batch_size, + ); + execute_on_pool_with_max_rows(state, pool_key, &sql, Some(active_batch_size)).await +} + +async fn close_table_read_session_if_open( + state: &AppState, + pool_key: &str, + table_read_session_id: &mut Option, +) { + let Some(session_id) = table_read_session_id.take() else { + return; + }; + let connections = state.connections.read().await; + let Some(PoolKind::Agent(client)) = connections.get(pool_key) else { + return; + }; + let client = client.clone(); + drop(connections); + let mut client = client.lock().await; + let _ = client.close_table_read_session::(&session_id).await; +} + +async fn start_export_cancel_watcher(export_id: String, cancelled: Arc, token: CancellationToken) { + loop { + if is_export_cancelled(&export_id).await { + cancelled.store(true, Ordering::SeqCst); + token.cancel(); + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +async fn stream_native_table_rows( + state: &AppState, + pool_key: &str, + db_type: &DatabaseType, + sql: &str, + row_limit: Option, + cancelled: &AtomicBool, + cancel_token: CancellationToken, + on_row: impl FnMut(&[Value]) -> Result<(), String>, +) -> Result { + let connections = state.connections.read().await; + match connections.get(pool_key) { + Some(PoolKind::Mysql(pool, mode)) => { + let pool = pool.clone(); + let bare = *mode == MysqlMode::Bare; + drop(connections); + crate::db::mysql::stream_query_rows( + &pool, + sql, + bare, + row_limit, + crate::db::mysql::MySqlQueryDialect::for_connection(*db_type, None), + cancelled, + on_row, + ) + .await?; + Ok(true) + } + Some(PoolKind::Postgres(pool)) => { + let pool = pool.clone(); + drop(connections); + crate::db::postgres::stream_query_rows(&pool, sql, row_limit, cancelled, on_row).await?; + Ok(true) + } + Some(PoolKind::SqlServer(client)) => { + let client = client.clone(); + drop(connections); + let mut on_row = on_row; + let mut client = client.lock().await; + crate::db::sqlserver::stream_first_result_set(&mut client, sql, row_limit, Some(cancel_token), |item| { + if let crate::db::sqlserver::SqlServerStreamItem::Row(row) = item { + on_row(row)?; + } + Ok(()) + }) + .await?; + Ok(true) + } + _ => Ok(false), + } +} + +#[allow(clippy::too_many_arguments)] +async fn try_export_native_table_stream( + state: &AppState, + pool_key: &str, + request: &TableExportRequest, + db_type: &DatabaseType, + col_names: &[String], + column_types: &[Option], + primary_keys: &[String], + total_rows: Option, + row_limit: Option, + batch_size: usize, + on_progress: &impl Fn(TableExportProgress), +) -> Result { + let sql = table_cursor_sql(request, db_type, col_names, primary_keys); + let cancelled = Arc::new(AtomicBool::new(false)); + let cancel_token = CancellationToken::new(); + let cancel_watcher = + tokio::spawn(start_export_cancel_watcher(request.export_id.clone(), cancelled.clone(), cancel_token.clone())); + let mut rows_exported = 0_u64; + let progress_interval = batch_size.max(1) as u64; + + let stream_result = match request.format.to_lowercase().as_str() { + "csv" => { + let mut file = BufWriter::new( + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, + ); + file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?; + let header = format_csv(col_names, &[]); + let header = header.strip_suffix('\n').unwrap_or(&header); + file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; + + let result = stream_native_table_rows( + state, + pool_key, + db_type, + &sql, + row_limit, + &cancelled, + cancel_token.clone(), + |row| { + let row_csv = format_csv_rows(&[row.to_vec()]); + write!(file, "\n{row_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?; + rows_exported += 1; + if rows_exported % progress_interval == 0 { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Running, + error_message: None, + }); + } + Ok(()) + }, + ) + .await; + if result.is_ok() { + file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; + } + result + } + "xlsx" => { + let xlsx_file = + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; + let mut writer = + start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some(&request.table_name), col_names)?; + let result = stream_native_table_rows( + state, + pool_key, + db_type, + &sql, + row_limit, + &cancelled, + cancel_token.clone(), + |row| { + writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + rows_exported += 1; + if rows_exported % progress_interval == 0 { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Running, + error_message: None, + }); + } + Ok(()) + }, + ) + .await; + if result.is_ok() { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Writing, + error_message: None, + }); + let mut xlsx_buf = + finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?; + xlsx_buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?; + } + result + } + "json" => { + let mut file = BufWriter::new( + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, + ); + file.write_all(b"[\n").map_err(|e| format!("Failed to write JSON: {e}"))?; + let mut is_first_row = true; + let result = stream_native_table_rows( + state, + pool_key, + db_type, + &sql, + row_limit, + &cancelled, + cancel_token.clone(), + |row| { + if !is_first_row { + file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; + } + write_json_row_object(&mut file, col_names, row)?; + is_first_row = false; + rows_exported += 1; + if rows_exported % progress_interval == 0 { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Running, + error_message: None, + }); + } + Ok(()) + }, + ) + .await; + if result.is_ok() { + file.write_all(b"\n]\n").map_err(|e| format!("Failed to write JSON: {e}"))?; + file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; + } + result + } + "markdown" | "md" => { + let mut file = BufWriter::new( + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, + ); + file.write_all(format_markdown_header(col_names).as_bytes()) + .map_err(|e| format!("Failed to write Markdown: {e}"))?; + let mut wrote_rows = false; + let result = stream_native_table_rows( + state, + pool_key, + db_type, + &sql, + row_limit, + &cancelled, + cancel_token.clone(), + |row| { + let rows_markdown = format_markdown_rows(&[row.to_vec()]); + if !rows_markdown.is_empty() { + if wrote_rows { + file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?; + } + file.write_all(rows_markdown.as_bytes()) + .map_err(|e| format!("Failed to write Markdown: {e}"))?; + wrote_rows = true; + } + rows_exported += 1; + if rows_exported % progress_interval == 0 { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Running, + error_message: None, + }); + } + Ok(()) + }, + ) + .await; + if result.is_ok() { + file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?; + file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; + } + result + } + "sql" => { + let mut file = BufWriter::new( + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, + ); + let mut pending_rows: Vec> = Vec::new(); + let mut wrote_statements = false; + let mut flush_pending = + |file: &mut BufWriter, pending_rows: &mut Vec>| -> Result<(), String> { + if pending_rows.is_empty() { + return Ok(()); + } + let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(*db_type), + schema: request.schema.clone(), + table_name: Some(request.table_name.clone()), + qualified_table_name: None, + columns: col_names.to_vec(), + column_types: column_types.to_vec(), + rows: std::mem::take(pending_rows), + batch_size: Some(SQL_INSERT_BATCH_SIZE), + })?; + if !statements.is_empty() { + if wrote_statements { + file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?; + } + file.write_all(statements.join("\n").as_bytes()) + .map_err(|e| format!("Failed to write SQL: {e}"))?; + wrote_statements = true; + } + Ok(()) + }; + let result = stream_native_table_rows( + state, + pool_key, + db_type, + &sql, + row_limit, + &cancelled, + cancel_token.clone(), + |row| { + pending_rows.push(row.to_vec()); + if pending_rows.len() >= SQL_INSERT_BATCH_SIZE { + flush_pending(&mut file, &mut pending_rows)?; + } + rows_exported += 1; + if rows_exported % progress_interval == 0 { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Running, + error_message: None, + }); + } + Ok(()) + }, + ) + .await; + if result.is_ok() { + flush_pending(&mut file, &mut pending_rows)?; + if wrote_statements { + file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?; + } + file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; + } + result + } + _ => Ok(false), + }; + + cancel_watcher.abort(); + + match stream_result { + Ok(false) => Ok(false), + Ok(true) => { + if rows_exported % progress_interval != 0 { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Running, + error_message: None, + }); + } + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Done, + error_message: None, + }); + Ok(true) + } + Err(error) if cancelled.load(Ordering::SeqCst) || error == crate::query::canceled_error() => { + on_progress(TableExportProgress { + export_id: request.export_id.clone(), + table_name: request.table_name.clone(), + rows_exported, + total_rows, + status: ExportStatus::Cancelled, + error_message: Some("Export cancelled".to_string()), + }); + Ok(true) + } + Err(error) => Err(error), + } +} + fn next_export_batch_size(row_limit: Option, rows_exported: u64, batch_size: usize) -> Option { let remaining = row_limit.map(|limit| limit.saturating_sub(rows_exported as usize)); if matches!(remaining, Some(0)) { @@ -253,6 +837,24 @@ pub async fn export_table_data_core( error_message: None, }); + if try_export_native_table_stream( + state, + &pool_key, + request, + &db_type, + &col_names, + &column_types, + &primary_keys, + total_rows, + row_limit, + request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1), + &on_progress, + ) + .await? + { + return Ok(()); + } + // 8. Create output file let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?; let mut file = BufWriter::new(file); @@ -260,6 +862,9 @@ pub async fn export_table_data_core( let mut rows_exported: u64 = 0; let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); let mut offset: u64 = 0; + let mut table_read_session_id: Option = None; + let mut table_read_attempted = false; + let mut table_read_completed = false; // Track last primary key values for keyset pagination let mut last_pk_values: Vec = Vec::new(); @@ -282,13 +887,16 @@ pub async fn export_table_data_core( status: ExportStatus::Cancelled, error_message: Some("Export cancelled".to_string()), }); + close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await; return Ok(()); } let Some(active_batch_size) = next_export_batch_size(row_limit, rows_exported, batch_size) else { break; }; - let sql = table_page_sql( + let result = fetch_table_export_batch( + state, + &pool_key, request, &db_type, &col_names, @@ -297,9 +905,11 @@ pub async fn export_table_data_core( &last_pk_values, offset, active_batch_size, - ); - - let result = execute_on_pool_with_max_rows(state, &pool_key, &sql, Some(active_batch_size)).await?; + &mut table_read_session_id, + &mut table_read_attempted, + &mut table_read_completed, + ) + .await?; let row_count = result.rows.len(); if row_count == 0 { break; @@ -363,13 +973,16 @@ pub async fn export_table_data_core( status: ExportStatus::Cancelled, error_message: Some("Export cancelled".to_string()), }); + close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await; return Ok(()); } let Some(active_batch_size) = next_export_batch_size(row_limit, rows_exported, batch_size) else { break; }; - let sql = table_page_sql( + let result = fetch_table_export_batch( + state, + &pool_key, request, &db_type, &col_names, @@ -378,9 +991,11 @@ pub async fn export_table_data_core( &last_pk_values, offset, active_batch_size, - ); - - let result = execute_on_pool_with_max_rows(state, &pool_key, &sql, Some(active_batch_size)).await?; + &mut table_read_session_id, + &mut table_read_attempted, + &mut table_read_completed, + ) + .await?; let row_count = result.rows.len(); if row_count == 0 { break; @@ -445,13 +1060,16 @@ pub async fn export_table_data_core( status: ExportStatus::Cancelled, error_message: Some("Export cancelled".to_string()), }); + close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await; return Ok(()); } let Some(active_batch_size) = next_export_batch_size(row_limit, rows_exported, batch_size) else { break; }; - let sql = table_page_sql( + let result = fetch_table_export_batch( + state, + &pool_key, request, &db_type, &col_names, @@ -460,8 +1078,11 @@ pub async fn export_table_data_core( &last_pk_values, offset, active_batch_size, - ); - let result = execute_on_pool_with_max_rows(state, &pool_key, &sql, Some(active_batch_size)).await?; + &mut table_read_session_id, + &mut table_read_attempted, + &mut table_read_completed, + ) + .await?; let row_count = result.rows.len(); if row_count == 0 { break; @@ -515,13 +1136,16 @@ pub async fn export_table_data_core( status: ExportStatus::Cancelled, error_message: Some("Export cancelled".to_string()), }); + close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await; return Ok(()); } let Some(active_batch_size) = next_export_batch_size(row_limit, rows_exported, batch_size) else { break; }; - let sql = table_page_sql( + let result = fetch_table_export_batch( + state, + &pool_key, request, &db_type, &col_names, @@ -530,8 +1154,11 @@ pub async fn export_table_data_core( &last_pk_values, offset, active_batch_size, - ); - let result = execute_on_pool_with_max_rows(state, &pool_key, &sql, Some(active_batch_size)).await?; + &mut table_read_session_id, + &mut table_read_attempted, + &mut table_read_completed, + ) + .await?; let row_count = result.rows.len(); if row_count == 0 { break; @@ -584,13 +1211,16 @@ pub async fn export_table_data_core( status: ExportStatus::Cancelled, error_message: Some("Export cancelled".to_string()), }); + close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await; return Ok(()); } let Some(active_batch_size) = next_export_batch_size(row_limit, rows_exported, batch_size) else { break; }; - let sql = table_page_sql( + let result = fetch_table_export_batch( + state, + &pool_key, request, &db_type, &col_names, @@ -599,8 +1229,11 @@ pub async fn export_table_data_core( &last_pk_values, offset, active_batch_size, - ); - let result = execute_on_pool_with_max_rows(state, &pool_key, &sql, Some(active_batch_size)).await?; + &mut table_read_session_id, + &mut table_read_attempted, + &mut table_read_completed, + ) + .await?; let row_count = result.rows.len(); if row_count == 0 { break; @@ -657,6 +1290,7 @@ pub async fn export_table_data_core( } } + close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await; file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; // 8. Emit Done progress @@ -745,6 +1379,49 @@ mod tests { assert_eq!(next_export_batch_size(Some(15_000), 15_000, 10_000), None); } + #[test] + fn oracle_table_cursor_sql_builds_single_ordered_select() { + let request = TableExportRequest { + export_id: "export-1".to_string(), + connection_id: "conn-1".to_string(), + database: "ORCL".to_string(), + schema: Some("APP".to_string()), + table_name: "events".to_string(), + file_path: "events.csv".to_string(), + format: "csv".to_string(), + columns: None, + column_types: None, + primary_keys: None, + where_input: Some("WHERE status = 'active'".to_string()), + order_by: None, + skip_count: false, + batch_size: Some(500), + row_limit: Some(1000), + }; + + let sql = table_cursor_sql( + &request, + &DatabaseType::Oracle, + &[String::from("id"), String::from("status")], + &[String::from("id")], + ); + + assert_eq!( + sql, + "SELECT \"id\", \"status\" FROM \"APP\".\"events\" WHERE (status = 'active') ORDER BY \"id\" ASC" + ); + assert!(!sql.contains("OFFSET")); + assert!(!sql.contains("FETCH NEXT")); + assert!(!sql.contains("ROWNUM")); + } + + #[test] + fn agent_table_read_unsupported_detects_old_agent_errors() { + assert!(is_agent_table_read_unsupported("Agent RPC error (-1): unknown method: start_table_read")); + assert!(is_agent_table_read_unsupported("Agent RPC error (-32601): Method not found")); + assert!(!is_agent_table_read_unsupported("ORA-00933: SQL command not properly ended")); + } + #[test] fn writes_json_row_without_allocating_object_map() { let mut out = Vec::new(); diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index 60014c9922..4e7bb511f1 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -346,6 +346,16 @@ fn postgres_order_by_expression(columns: &[String], db_type: &DatabaseType) -> O Some(columns.iter().map(|column| quote_identifier(column, db_type)).collect::>().join(", ")) } +fn oracle_rownum_page_sql(col_list: &str, base_sql: String, offset: u64, limit: usize) -> String { + if offset == 0 { + return format!("SELECT {col_list} FROM ({base_sql}) WHERE ROWNUM <= {limit}"); + } + let end = offset + limit as u64; + format!( + "SELECT {col_list} FROM (SELECT dbx_inner.*, ROWNUM AS \"__dbx_row_num\" FROM ({base_sql}) dbx_inner WHERE ROWNUM <= {end}) WHERE \"__dbx_row_num\" > {offset}" + ) +} + fn postgres_index_column_sql(column: &str) -> String { if is_simple_identifier(column) { quote_identifier(column, &DatabaseType::Postgres) @@ -1551,7 +1561,11 @@ pub fn pagination_sql( let col_list = columns.iter().map(|c| quote_identifier(c, db_type)).collect::>().join(", "); match db_type { - DatabaseType::SqlServer | DatabaseType::Oracle | DatabaseType::Dameng => { + DatabaseType::Oracle => { + let base_sql = format!("SELECT {col_list} FROM {full_table}"); + oracle_rownum_page_sql(&col_list, base_sql, offset, limit) + } + DatabaseType::SqlServer | DatabaseType::Dameng => { format!( "SELECT {col_list} FROM {full_table} ORDER BY (SELECT NULL) OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY" ) @@ -1580,7 +1594,12 @@ pub fn pagination_sql_with_order( let order_expression = postgres_order_by_expression(order_by_columns, db_type); match db_type { - DatabaseType::SqlServer | DatabaseType::Oracle | DatabaseType::Dameng => { + DatabaseType::Oracle => { + let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); + let base_sql = format!("SELECT {col_list} FROM {full_table}{order_by}"); + oracle_rownum_page_sql(&col_list, base_sql, offset, limit) + } + DatabaseType::SqlServer | DatabaseType::Dameng => { let order_by = order_expression.unwrap_or_else(|| "(SELECT NULL)".to_string()); format!( "SELECT {col_list} FROM {full_table} ORDER BY {order_by} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY" @@ -1621,7 +1640,12 @@ pub fn pagination_sql_with_filter_order( .or_else(|| postgres_order_by_expression(default_order_columns, db_type)); match db_type { - DatabaseType::SqlServer | DatabaseType::Oracle | DatabaseType::Dameng => { + DatabaseType::Oracle => { + let order_by = order_expression.map(|value| format!(" ORDER BY {value}")).unwrap_or_default(); + let base_sql = format!("SELECT {col_list} FROM {full_table}{where_clause}{order_by}"); + oracle_rownum_page_sql(&col_list, base_sql, offset, limit) + } + DatabaseType::SqlServer | DatabaseType::Dameng => { let order_by = order_expression.unwrap_or_else(|| "(SELECT NULL)".to_string()); format!( "SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order_by} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY" @@ -1667,7 +1691,11 @@ pub fn keyset_pagination_sql( let where_clause = keyset_where_clause(primary_keys, last_pk_values, db_type); match db_type { - DatabaseType::SqlServer | DatabaseType::Oracle | DatabaseType::Dameng => { + DatabaseType::Oracle => { + let base_sql = format!("SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order}"); + oracle_rownum_page_sql(&col_list, base_sql, 0, limit) + } + DatabaseType::SqlServer | DatabaseType::Dameng => { format!( "SELECT {col_list} FROM {full_table}{where_clause} ORDER BY {order} OFFSET 0 ROWS FETCH NEXT {limit} ROWS ONLY" ) @@ -4045,6 +4073,28 @@ mod tests { assert!(!sql.contains(" LIMIT ")); } + #[test] + fn oracle_filtered_pagination_uses_rownum_for_legacy_compatibility() { + let sql = pagination_sql_with_filter_order( + &[String::from("id"), String::from("status")], + "users", + "APP", + &DatabaseType::Oracle, + 10_000, + 2_000, + Some("WHERE status = 'active'"), + Some("\"id\" DESC"), + &[String::from("id")], + ); + + assert_eq!( + sql, + "SELECT \"id\", \"status\" FROM (SELECT dbx_inner.*, ROWNUM AS \"__dbx_row_num\" FROM (SELECT \"id\", \"status\" FROM \"APP\".\"users\" WHERE (status = 'active') ORDER BY \"id\" DESC) dbx_inner WHERE ROWNUM <= 12000) WHERE \"__dbx_row_num\" > 10000" + ); + assert!(!sql.contains("OFFSET")); + assert!(!sql.contains("FETCH NEXT")); + } + #[test] fn filtered_count_preserves_where() { let sql = count_sql_with_where("users", "public", &DatabaseType::SapHana, Some("WHERE status = 'active'")); @@ -4089,6 +4139,26 @@ mod tests { assert!(!sql.contains(" LIMIT ")); } + #[test] + fn oracle_keyset_pagination_uses_rownum_for_legacy_compatibility() { + let sql = keyset_pagination_sql( + &[String::from("id"), String::from("name")], + "users", + "APP", + &DatabaseType::Oracle, + &[String::from("id")], + &[json!(25)], + 100, + ); + + assert_eq!( + sql, + "SELECT \"id\", \"name\" FROM (SELECT \"id\", \"name\" FROM \"APP\".\"users\" WHERE \"id\" > 25 ORDER BY \"id\" ASC) WHERE ROWNUM <= 100" + ); + assert!(!sql.contains("OFFSET")); + assert!(!sql.contains("FETCH NEXT")); + } + #[test] fn composite_keyset_pagination_uses_portable_lexicographic_predicate() { let sql = keyset_pagination_sql(