fix(spark,hive): cap Thrift fetch size

This commit is contained in:
jischeng authored and GitHub committed 2026-07-08 23:44:59 +08:00
1 parent b862870e8e
commit ee34a17e82
4 files changed
+153 -4

No files matched your search

@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.QueryPageOptions;
import com.dbx.agent.QueryPageResult;
import com.dbx.agent.QueryResult;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
@@ -31,6 +35,35 @@ import java.util.Locale;
public final class SparkAgent extends AbstractJdbcAgent {
private String configuredCatalog;
@Override
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
// by fetchSize. A large fetchSize can fail with "Error retrieving next
// row" when the result contains large binary/struct columns. Use a
// small fetchSize (50) so each Thrift batch stays within limits.
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
options.getMaxRows(),
50,
options.getTimeoutSecs()
));
}
@Override
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
}
@Override
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
return super.startTableRead(sql, schema, withSmallFetchSize(options));
}
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
return new QueryPageOptions(
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
);
}
@Override
protected String driverClass() {
return "org.apache.hive.jdbc.HiveDriver";