fix(spark,hive): cap Thrift fetch size
This commit is contained in:
1 parent
b862870e8e
commit
ee34a17e82
4 files changed
+153
-4
No files matched your search
@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
|
||||
import com.dbx.agent.JsonRpcServer;
|
||||
import com.dbx.agent.TableInfo;
|
||||
import com.dbx.agent.TriggerInfo;
|
||||
import com.dbx.agent.ExecuteQueryOptions;
|
||||
import com.dbx.agent.QueryPageOptions;
|
||||
import com.dbx.agent.QueryPageResult;
|
||||
import com.dbx.agent.QueryResult;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.ResultSet;
|
||||
@@ -102,6 +106,35 @@ public final class HiveAgent extends AbstractJdbcAgent {
|
||||
return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
|
||||
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
|
||||
// by fetchSize. A large fetchSize can fail with "Error retrieving next
|
||||
// row" when the result contains large binary/struct columns. Use a
|
||||
// small fetchSize (50) so each Thrift batch stays within limits.
|
||||
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
|
||||
options.getMaxRows(),
|
||||
50,
|
||||
options.getTimeoutSecs()
|
||||
));
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
|
||||
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||
return super.startTableRead(sql, schema, withSmallFetchSize(options));
|
||||
}
|
||||
|
||||
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
|
||||
return new QueryPageOptions(
|
||||
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Object resultValue(ResultSet rs, int index, int sqlType) {
|
||||
return unchecked(() -> {
|
||||
|
||||
@@ -1,7 +1,18 @@
|
||||
package com.dbx.agent.hive;
|
||||
|
||||
import com.dbx.agent.DatabaseAgent;
|
||||
import com.dbx.agent.ExecuteQueryOptions;
|
||||
import com.dbx.agent.QueryPageOptions;
|
||||
import com.dbx.agent.QueryResult;
|
||||
import com.dbx.agent.test.JdbcAgentFake;
|
||||
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
|
||||
import com.dbx.agent.test.TestSupport;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
|
||||
@Override
|
||||
@@ -13,4 +24,54 @@ class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
|
||||
protected String resultSetSql() {
|
||||
return "MSCK REPAIR TABLE sample";
|
||||
}
|
||||
|
||||
// Override: HiveAgent.executeQuery forces fetchSize=50 to keep Thrift
|
||||
// FetchResults batches small (prevents "Error retrieving next row" on
|
||||
// tables with large binary/struct columns).
|
||||
@Override
|
||||
@Test
|
||||
protected void executesNonSelectStatementsThatReturnResultSets() {
|
||||
DatabaseAgent agent = createAgent();
|
||||
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||
|
||||
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions());
|
||||
|
||||
assertEquals(Collections.singletonList("VALUE"), result.getColumns());
|
||||
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
|
||||
assertEquals(Arrays.asList("setMaxRows:10001", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
|
||||
DatabaseAgent agent = createAgent();
|
||||
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||
|
||||
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions(1, 25));
|
||||
|
||||
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
|
||||
assertEquals(Arrays.asList("setMaxRows:2", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() {
|
||||
DatabaseAgent agent = createAgent();
|
||||
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||
|
||||
agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
|
||||
|
||||
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Test
|
||||
protected void tableReadPassesCallerFetchSizeToStatement() {
|
||||
DatabaseAgent agent = createAgent();
|
||||
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||
|
||||
agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
|
||||
|
||||
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user