fix(spark,hive): cap Thrift fetch size

This commit is contained in:
jischeng authored and GitHub committed 2026-07-08 23:44:59 +08:00
1 parent b862870e8e
commit ee34a17e82
4 files changed
+153 -4

No files matched your search

@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryPageResult;
import com.dbx.agent.QueryResult;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
@@ -102,6 +106,35 @@ public final class HiveAgent extends AbstractJdbcAgent {
return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
}
@Override
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
// by fetchSize. A large fetchSize can fail with "Error retrieving next
// row" when the result contains large binary/struct columns. Use a
// small fetchSize (50) so each Thrift batch stays within limits.
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
options.getMaxRows(),
50,
options.getTimeoutSecs()
));
}
@Override
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
}
@Override
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
return super.startTableRead(sql, schema, withSmallFetchSize(options));
}
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
return new QueryPageOptions(
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
);
}
@Override
protected Object resultValue(ResultSet rs, int index, int sqlType) {
return unchecked(() -> {
@@ -1,7 +1,18 @@
package com.dbx.agent.hive;
import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryResult;
import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.jupiter.api.Assertions.assertEquals;
class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
@Override
@@ -13,4 +24,54 @@ class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
protected String resultSetSql() {
return "MSCK REPAIR TABLE sample";
}
// Override: HiveAgent.executeQuery forces fetchSize=50 to keep Thrift
// FetchResults batches small (prevents "Error retrieving next row" on
// tables with large binary/struct columns).
@Override
@Test
protected void executesNonSelectStatementsThatReturnResultSets() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions());
assertEquals(Collections.singletonList("VALUE"), result.getColumns());
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
assertEquals(Arrays.asList("setMaxRows:10001", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
@Override
@Test
protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions(1, 25));
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
assertEquals(Arrays.asList("setMaxRows:2", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
@Override
@Test
protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
@Override
@Test
protected void tableReadPassesCallerFetchSizeToStatement() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
}