fix(spark,hive): cap Thrift fetch size

This commit is contained in:
jischeng authored and GitHub committed 2026-07-08 23:44:59 +08:00
1 parent b862870e8e
commit ee34a17e82
4 files changed
+153 -4

No files matched your search

@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo; import com.dbx.agent.TriggerInfo;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryPageResult;
import com.dbx.agent.QueryResult;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DatabaseMetaData; import java.sql.DatabaseMetaData;
import java.sql.ResultSet; import java.sql.ResultSet;
@@ -102,6 +106,35 @@ public final class HiveAgent extends AbstractJdbcAgent {
return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema); return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
} }
@Override
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
// by fetchSize. A large fetchSize can fail with "Error retrieving next
// row" when the result contains large binary/struct columns. Use a
// small fetchSize (50) so each Thrift batch stays within limits.
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
options.getMaxRows(),
50,
options.getTimeoutSecs()
));
}
@Override
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
}
@Override
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
return super.startTableRead(sql, schema, withSmallFetchSize(options));
}
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
return new QueryPageOptions(
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
);
}
@Override @Override
protected Object resultValue(ResultSet rs, int index, int sqlType) { protected Object resultValue(ResultSet rs, int index, int sqlType) {
return unchecked(() -> { return unchecked(() -> {
@@ -1,7 +1,18 @@
package com.dbx.agent.hive; package com.dbx.agent.hive;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryResult;
import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest; import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.jupiter.api.Assertions.assertEquals;
class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest { class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
@Override @Override
@@ -13,4 +24,54 @@ class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
protected String resultSetSql() { protected String resultSetSql() {
return "MSCK REPAIR TABLE sample"; return "MSCK REPAIR TABLE sample";
} }
// Override: HiveAgent.executeQuery forces fetchSize=50 to keep Thrift
// FetchResults batches small (prevents "Error retrieving next row" on
// tables with large binary/struct columns).
@Override
@Test
protected void executesNonSelectStatementsThatReturnResultSets() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions());
assertEquals(Collections.singletonList("VALUE"), result.getColumns());
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
assertEquals(Arrays.asList("setMaxRows:10001", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
@Override
@Test
protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions(1, 25));
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
assertEquals(Arrays.asList("setMaxRows:2", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
@Override
@Test
protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
@Override
@Test
protected void tableReadPassesCallerFetchSizeToStatement() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
}
} }
@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer; import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.TableInfo; import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo; import com.dbx.agent.TriggerInfo;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryPageResult;
import com.dbx.agent.QueryResult;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DatabaseMetaData; import java.sql.DatabaseMetaData;
import java.sql.ResultSet; import java.sql.ResultSet;
@@ -31,6 +35,35 @@ import java.util.Locale;
public final class SparkAgent extends AbstractJdbcAgent { public final class SparkAgent extends AbstractJdbcAgent {
private String configuredCatalog; private String configuredCatalog;
@Override
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
// by fetchSize. A large fetchSize can fail with "Error retrieving next
// row" when the result contains large binary/struct columns. Use a
// small fetchSize (50) so each Thrift batch stays within limits.
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
options.getMaxRows(),
50,
options.getTimeoutSecs()
));
}
@Override
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
}
@Override
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
return super.startTableRead(sql, schema, withSmallFetchSize(options));
}
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
return new QueryPageOptions(
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
);
}
@Override @Override
protected String driverClass() { protected String driverClass() {
return "org.apache.hive.jdbc.HiveDriver"; return "org.apache.hive.jdbc.HiveDriver";
@@ -2,6 +2,8 @@ package com.dbx.agent.test;
import com.dbx.agent.DatabaseAgent; import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.ExecuteQueryOptions; import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryPageResult;
import com.dbx.agent.QueryResult; import com.dbx.agent.QueryResult;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@@ -15,7 +17,7 @@ public abstract class JdbcFakeExecutionBehaviorTest {
protected abstract String resultSetSql(); protected abstract String resultSetSql();
@Test @Test
void executesNonSelectStatementsThatReturnResultSets() { protected void executesNonSelectStatementsThatReturnResultSets() {
DatabaseAgent agent = createAgent(); DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
@@ -27,7 +29,7 @@ public abstract class JdbcFakeExecutionBehaviorTest {
} }
@Test @Test
void appliesExecutionRowAndFetchLimitsToJdbcStatements() { protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
DatabaseAgent agent = createAgent(); DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection()); TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
@@ -37,11 +39,31 @@ public abstract class JdbcFakeExecutionBehaviorTest {
assertEquals(asList("setMaxRows:2", "setFetchSize:25", "execute"), JdbcAgentFake.calls); assertEquals(asList("setMaxRows:2", "setFetchSize:25", "execute"), JdbcAgentFake.calls);
} }
private static java.util.List<String> asList(String first, String second) { @Test
protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
assertEquals(asList("setFetchSize:500", "execute"), JdbcAgentFake.calls);
}
@Test
protected void tableReadPassesCallerFetchSizeToStatement() {
DatabaseAgent agent = createAgent();
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
assertEquals(asList("setFetchSize:500", "execute"), JdbcAgentFake.calls);
}
protected static java.util.List<String> asList(String first, String second) {
return java.util.Arrays.asList(first, second); return java.util.Arrays.asList(first, second);
} }
private static java.util.List<String> asList(String first, String second, String third) { protected static java.util.List<String> asList(String first, String second, String third) {
return java.util.Arrays.asList(first, second, third); return java.util.Arrays.asList(first, second, third);
} }
} }