fix(spark,hive): cap Thrift fetch size
This commit is contained in:
1 parent
b862870e8e
commit
ee34a17e82
4 files changed
+153
-4
No files matched your search
@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
|
||||
import com.dbx.agent.JsonRpcServer;
|
||||
import com.dbx.agent.TableInfo;
|
||||
import com.dbx.agent.TriggerInfo;
|
||||
import com.dbx.agent.ExecuteQueryOptions;
|
||||
import com.dbx.agent.QueryPageOptions;
|
||||
import com.dbx.agent.QueryPageResult;
|
||||
import com.dbx.agent.QueryResult;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.ResultSet;
|
||||
@@ -31,6 +35,35 @@ import java.util.Locale;
|
||||
public final class SparkAgent extends AbstractJdbcAgent {
|
||||
private String configuredCatalog;
|
||||
|
||||
@Override
|
||||
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
|
||||
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
|
||||
// by fetchSize. A large fetchSize can fail with "Error retrieving next
|
||||
// row" when the result contains large binary/struct columns. Use a
|
||||
// small fetchSize (50) so each Thrift batch stays within limits.
|
||||
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
|
||||
options.getMaxRows(),
|
||||
50,
|
||||
options.getTimeoutSecs()
|
||||
));
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
|
||||
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||
return super.startTableRead(sql, schema, withSmallFetchSize(options));
|
||||
}
|
||||
|
||||
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
|
||||
return new QueryPageOptions(
|
||||
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String driverClass() {
|
||||
return "org.apache.hive.jdbc.HiveDriver";
|
||||
|
||||
Reference in new issue
Block a user