fix(spark,hive): cap Thrift fetch size
This commit is contained in:
1 parent
b862870e8e
commit
ee34a17e82
4 files changed
+153
-4
No files matched your search
@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
|
|||||||
import com.dbx.agent.JsonRpcServer;
|
import com.dbx.agent.JsonRpcServer;
|
||||||
import com.dbx.agent.TableInfo;
|
import com.dbx.agent.TableInfo;
|
||||||
import com.dbx.agent.TriggerInfo;
|
import com.dbx.agent.TriggerInfo;
|
||||||
|
import com.dbx.agent.ExecuteQueryOptions;
|
||||||
|
import com.dbx.agent.QueryPageOptions;
|
||||||
|
import com.dbx.agent.QueryPageResult;
|
||||||
|
import com.dbx.agent.QueryResult;
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.DatabaseMetaData;
|
import java.sql.DatabaseMetaData;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
@@ -102,6 +106,35 @@ public final class HiveAgent extends AbstractJdbcAgent {
|
|||||||
return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
|
return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
|
||||||
|
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
|
||||||
|
// by fetchSize. A large fetchSize can fail with "Error retrieving next
|
||||||
|
// row" when the result contains large binary/struct columns. Use a
|
||||||
|
// small fetchSize (50) so each Thrift batch stays within limits.
|
||||||
|
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
|
||||||
|
options.getMaxRows(),
|
||||||
|
50,
|
||||||
|
options.getTimeoutSecs()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
|
||||||
|
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||||
|
return super.startTableRead(sql, schema, withSmallFetchSize(options));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
|
||||||
|
return new QueryPageOptions(
|
||||||
|
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Object resultValue(ResultSet rs, int index, int sqlType) {
|
protected Object resultValue(ResultSet rs, int index, int sqlType) {
|
||||||
return unchecked(() -> {
|
return unchecked(() -> {
|
||||||
|
|||||||
@@ -1,7 +1,18 @@
|
|||||||
package com.dbx.agent.hive;
|
package com.dbx.agent.hive;
|
||||||
|
|
||||||
import com.dbx.agent.DatabaseAgent;
|
import com.dbx.agent.DatabaseAgent;
|
||||||
|
import com.dbx.agent.ExecuteQueryOptions;
|
||||||
|
import com.dbx.agent.QueryPageOptions;
|
||||||
|
import com.dbx.agent.QueryResult;
|
||||||
|
import com.dbx.agent.test.JdbcAgentFake;
|
||||||
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
|
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
|
||||||
|
import com.dbx.agent.test.TestSupport;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
|
class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
|
||||||
@Override
|
@Override
|
||||||
@@ -13,4 +24,54 @@ class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
|
|||||||
protected String resultSetSql() {
|
protected String resultSetSql() {
|
||||||
return "MSCK REPAIR TABLE sample";
|
return "MSCK REPAIR TABLE sample";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Override: HiveAgent.executeQuery forces fetchSize=50 to keep Thrift
|
||||||
|
// FetchResults batches small (prevents "Error retrieving next row" on
|
||||||
|
// tables with large binary/struct columns).
|
||||||
|
@Override
|
||||||
|
@Test
|
||||||
|
protected void executesNonSelectStatementsThatReturnResultSets() {
|
||||||
|
DatabaseAgent agent = createAgent();
|
||||||
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
|
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions());
|
||||||
|
|
||||||
|
assertEquals(Collections.singletonList("VALUE"), result.getColumns());
|
||||||
|
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
|
||||||
|
assertEquals(Arrays.asList("setMaxRows:10001", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Test
|
||||||
|
protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
|
||||||
|
DatabaseAgent agent = createAgent();
|
||||||
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
|
QueryResult result = agent.executeQuery(resultSetSql(), null, new ExecuteQueryOptions(1, 25));
|
||||||
|
|
||||||
|
assertEquals(Collections.singletonList(Collections.singletonList("row-value")), result.getRows());
|
||||||
|
assertEquals(Arrays.asList("setMaxRows:2", "setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Test
|
||||||
|
protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() {
|
||||||
|
DatabaseAgent agent = createAgent();
|
||||||
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
|
agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
|
||||||
|
|
||||||
|
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Test
|
||||||
|
protected void tableReadPassesCallerFetchSizeToStatement() {
|
||||||
|
DatabaseAgent agent = createAgent();
|
||||||
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
|
agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
|
||||||
|
|
||||||
|
assertEquals(Arrays.asList("setFetchSize:50", "execute"), JdbcAgentFake.calls);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -10,6 +10,10 @@ import com.dbx.agent.JdbcIdentifiers;
|
|||||||
import com.dbx.agent.JsonRpcServer;
|
import com.dbx.agent.JsonRpcServer;
|
||||||
import com.dbx.agent.TableInfo;
|
import com.dbx.agent.TableInfo;
|
||||||
import com.dbx.agent.TriggerInfo;
|
import com.dbx.agent.TriggerInfo;
|
||||||
|
import com.dbx.agent.ExecuteQueryOptions;
|
||||||
|
import com.dbx.agent.QueryPageOptions;
|
||||||
|
import com.dbx.agent.QueryPageResult;
|
||||||
|
import com.dbx.agent.QueryResult;
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.DatabaseMetaData;
|
import java.sql.DatabaseMetaData;
|
||||||
import java.sql.ResultSet;
|
import java.sql.ResultSet;
|
||||||
@@ -31,6 +35,35 @@ import java.util.Locale;
|
|||||||
public final class SparkAgent extends AbstractJdbcAgent {
|
public final class SparkAgent extends AbstractJdbcAgent {
|
||||||
private String configuredCatalog;
|
private String configuredCatalog;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryResult executeQuery(String sql, String schema, ExecuteQueryOptions options) {
|
||||||
|
// Hive JDBC fetches rows via Thrift FetchResults in batches controlled
|
||||||
|
// by fetchSize. A large fetchSize can fail with "Error retrieving next
|
||||||
|
// row" when the result contains large binary/struct columns. Use a
|
||||||
|
// small fetchSize (50) so each Thrift batch stays within limits.
|
||||||
|
return super.executeQuery(sql, schema, new ExecuteQueryOptions(
|
||||||
|
options.getMaxRows(),
|
||||||
|
50,
|
||||||
|
options.getTimeoutSecs()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryPageResult executeQueryPage(String sql, String schema, QueryPageOptions options) {
|
||||||
|
return super.executeQueryPage(sql, schema, withSmallFetchSize(options));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public QueryPageResult startTableRead(String sql, String schema, QueryPageOptions options) {
|
||||||
|
return super.startTableRead(sql, schema, withSmallFetchSize(options));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static QueryPageOptions withSmallFetchSize(QueryPageOptions options) {
|
||||||
|
return new QueryPageOptions(
|
||||||
|
options.getPageSize(), 50, options.getMaxRows(), options.getTimeoutSecs()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected String driverClass() {
|
protected String driverClass() {
|
||||||
return "org.apache.hive.jdbc.HiveDriver";
|
return "org.apache.hive.jdbc.HiveDriver";
|
||||||
|
|||||||
+26
-4
@@ -2,6 +2,8 @@ package com.dbx.agent.test;
|
|||||||
|
|
||||||
import com.dbx.agent.DatabaseAgent;
|
import com.dbx.agent.DatabaseAgent;
|
||||||
import com.dbx.agent.ExecuteQueryOptions;
|
import com.dbx.agent.ExecuteQueryOptions;
|
||||||
|
import com.dbx.agent.QueryPageOptions;
|
||||||
|
import com.dbx.agent.QueryPageResult;
|
||||||
import com.dbx.agent.QueryResult;
|
import com.dbx.agent.QueryResult;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
@@ -15,7 +17,7 @@ public abstract class JdbcFakeExecutionBehaviorTest {
|
|||||||
protected abstract String resultSetSql();
|
protected abstract String resultSetSql();
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void executesNonSelectStatementsThatReturnResultSets() {
|
protected void executesNonSelectStatementsThatReturnResultSets() {
|
||||||
DatabaseAgent agent = createAgent();
|
DatabaseAgent agent = createAgent();
|
||||||
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
@@ -27,7 +29,7 @@ public abstract class JdbcFakeExecutionBehaviorTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
|
protected void appliesExecutionRowAndFetchLimitsToJdbcStatements() {
|
||||||
DatabaseAgent agent = createAgent();
|
DatabaseAgent agent = createAgent();
|
||||||
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
@@ -37,11 +39,31 @@ public abstract class JdbcFakeExecutionBehaviorTest {
|
|||||||
assertEquals(asList("setMaxRows:2", "setFetchSize:25", "execute"), JdbcAgentFake.calls);
|
assertEquals(asList("setMaxRows:2", "setFetchSize:25", "execute"), JdbcAgentFake.calls);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static java.util.List<String> asList(String first, String second) {
|
@Test
|
||||||
|
protected void pagedQueryExecutionPassesCallerFetchSizeToStatement() {
|
||||||
|
DatabaseAgent agent = createAgent();
|
||||||
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
|
agent.executeQueryPage(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
|
||||||
|
|
||||||
|
assertEquals(asList("setFetchSize:500", "execute"), JdbcAgentFake.calls);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
protected void tableReadPassesCallerFetchSizeToStatement() {
|
||||||
|
DatabaseAgent agent = createAgent();
|
||||||
|
TestSupport.setPrivateConnection(agent, JdbcAgentFake.connection());
|
||||||
|
|
||||||
|
agent.startTableRead(resultSetSql(), null, new QueryPageOptions(100, 500, 1000));
|
||||||
|
|
||||||
|
assertEquals(asList("setFetchSize:500", "execute"), JdbcAgentFake.calls);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static java.util.List<String> asList(String first, String second) {
|
||||||
return java.util.Arrays.asList(first, second);
|
return java.util.Arrays.asList(first, second);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static java.util.List<String> asList(String first, String second, String third) {
|
protected static java.util.List<String> asList(String first, String second, String third) {
|
||||||
return java.util.Arrays.asList(first, second, third);
|
return java.util.Arrays.asList(first, second, third);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user