diff --git a/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java b/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java index 98a285c901..c4cef21180 100644 --- a/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java +++ b/agents/drivers/hive/src/main/java/com/dbx/agent/hive/HiveAgent.java @@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JsonRpcServer; import com.dbx.agent.TableInfo; import com.dbx.agent.TriggerInfo; +import com.dbx.agent.ExecuteQueryOptions; +import com.dbx.agent.QueryPageOptions; +import com.dbx.agent.QueryPageResult; +import com.dbx.agent.QueryResult; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; @@ -102,6 +106,35 @@ public final class HiveAgent extends AbstractJdbcAgent { return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema); } + @Override + public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) { + // Hive JDBC fetches rows via Thrift FetchResults in batches controlled + // by fetchSize. A large fetchSize can fail with "Error retrieving next + // row" when the result contains large binary/struct columns. Use a + // small fetchSize (50) so each Thrift batch stays within limits. + return super.executeQuery(sql, schema, new ExecuteQueryOptions( + options.getMaxRows(), + 50, + options.getTimeoutSecs() + )); + } + + @Override + public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) { + return super.executeQueryPage(sql, schema, withSmallFetchSize(options)); + } + + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return super.startTableRead(sql, schema, withSmallFetchSize(options)); + } + + private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) { + return new QueryPageOptions( + options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs() + ); + } + @Override protected Object resultValue(ResultSet rs, int index, int sqlType) { return unchecked(() -> { diff --git a/agents/drivers/hive/src/test/java/com/dbx/agent/hive/HiveAgentExecutionTest.java b/agents/drivers/hive/src/test/java/com/dbx/agent/hive/HiveAgentExecutionTest.java index 53f7bb7732..3e7e2c5899 100644 --- a/agents/drivers/hive/src/test/java/com/dbx/agent/hive/HiveAgentExecutionTest.java +++ b/agents/drivers/hive/src/test/java/com/dbx/agent/hive/HiveAgentExecutionTest.java @@ -1,7 +1,18 @@ package com.dbx.agent.hive; import com.dbx.agent.DatabaseAgent; +import com.dbx.agent.ExecuteQueryOptions; +import com.dbx.agent.QueryPageOptions; +import com.dbx.agent.QueryResult; +import com.dbx.agent.test.JdbcAgentFake; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; +import com.dbx.agent.test.TestSupport; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest { @Override @@ -13,4 +24,54 @@ class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest { protected String resultSetSql() { return "MSCK REPAIR TABLE sample"; } + + // Override: HiveAgent.executeQuery forces fetchSize=50 to keep Thrift + // FetchResults batches small (prevents "Error retrieving next row" on + // tables with large binary/struct columns). + @Override + @Test + protected void executesNonSelectStatementsThatReturnResultSets() { + DatabaseAgent agent = createAgent(); + TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); + + QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions()); + + assertEquals(Collections.singletonList("VALUE"), result.getColumns()); + assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows()); + assertEquals(Arrays.asList("setMaxRows:10001", "setFetchSize:50", "execute"), JdbcAgentFake.calls); + } + + @Override + @Test + protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() { + DatabaseAgent agent = createAgent(); + TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); + + QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions(1, 25)); + + assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows()); + assertEquals(Arrays.asList("setMaxRows:2", "setFetchSize:50", "execute"), JdbcAgentFake.calls); + } + + @Override + @Test + protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() { + DatabaseAgent agent = createAgent(); + TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); + + agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000)); + + assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls); + } + + @Override + @Test + protected void tableReadPassesCallerFetchSizeToStatement() { + DatabaseAgent agent = createAgent(); + TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); + + agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000)); + + assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls); + } } diff --git a/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java b/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java index fc0d4e9a0b..bbe1570107 100644 --- a/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java +++ b/agents/drivers/spark/src/main/java/com/dbx/agent/spark/SparkAgent.java @@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers; import com.dbx.agent.JsonRpcServer; import com.dbx.agent.TableInfo; import com.dbx.agent.TriggerInfo; +import com.dbx.agent.ExecuteQueryOptions; +import com.dbx.agent.QueryPageOptions; +import com.dbx.agent.QueryPageResult; +import com.dbx.agent.QueryResult; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.ResultSet; @@ -31,6 +35,35 @@ import java.util.Locale; public final class SparkAgent extends AbstractJdbcAgent { private String configuredCatalog; + @Override + public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) { + // Hive JDBC fetches rows via Thrift FetchResults in batches controlled + // by fetchSize. A large fetchSize can fail with "Error retrieving next + // row" when the result contains large binary/struct columns. Use a + // small fetchSize (50) so each Thrift batch stays within limits. + return super.executeQuery(sql, schema, new ExecuteQueryOptions( + options.getMaxRows(), + 50, + options.getTimeoutSecs() + )); + } + + @Override + public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) { + return super.executeQueryPage(sql, schema, withSmallFetchSize(options)); + } + + @Override + public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) { + return super.startTableRead(sql, schema, withSmallFetchSize(options)); + } + + private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) { + return new QueryPageOptions( + options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs() + ); + } + @Override protected String driverClass() { return "org.apache.hive.jdbc.HiveDriver"; diff --git a/agents/test-support/src/main/java/com/dbx/agent/test/JdbcFakeExecutionBehaviorTest.java b/agents/test-support/src/main/java/com/dbx/agent/test/JdbcFakeExecutionBehaviorTest.java index 0f74d12cb1..18f554d2d0 100644 --- a/agents/test-support/src/main/java/com/dbx/agent/test/JdbcFakeExecutionBehaviorTest.java +++ b/agents/test-support/src/main/java/com/dbx/agent/test/JdbcFakeExecutionBehaviorTest.java @@ -2,6 +2,8 @@ package com.dbx.agent.test; import com.dbx.agent.DatabaseAgent; import com.dbx.agent.ExecuteQueryOptions; +import com.dbx.agent.QueryPageOptions; +import com.dbx.agent.QueryPageResult; import com.dbx.agent.QueryResult; import org.junit.jupiter.api.Test; @@ -15,7 +17,7 @@ public abstract class JdbcFakeExecutionBehaviorTest { protected abstract String resultSetSql(); @Test - void executesNonSelectStatementsThatReturnResultSets() { + protected void executesNonSelectStatementsThatReturnResultSets() { DatabaseAgent agent = createAgent(); TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); @@ -27,7 +29,7 @@ public abstract class JdbcFakeExecutionBehaviorTest { } @Test - void appliesExecutionRowAndFetchLimitsToJdbcStatements() { + protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() { DatabaseAgent agent = createAgent(); TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); @@ -37,11 +39,31 @@ public abstract class JdbcFakeExecutionBehaviorTest { assertEquals(asList("setMaxRows:2", "setFetchSize:25", "execute"), JdbcAgentFake.calls); } - private static java.util.List asList(String first, String second) { + @Test + protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() { + DatabaseAgent agent = createAgent(); + TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); + + agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000)); + + assertEquals(asList("setFetchSize:500", "execute"), JdbcAgentFake.calls); + } + + @Test + protected void tableReadPassesCallerFetchSizeToStatement() { + DatabaseAgent agent = createAgent(); + TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); + + agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000)); + + assertEquals(asList("setFetchSize:500", "execute"), JdbcAgentFake.calls); + } + + protected static java.util.List asList(String first, String second) { return java.util.Arrays.asList(first, second); } - private static java.util.List asList(String first, String second, String third) { + protected static java.util.List asList(String first, String second, String third) { return java.util.Arrays.asList(first, second, third); } }