feat(agents): move agents into dbx repo

This commit is contained in:
t8y2 committed 2026-06-23 11:38:16 +08:00
1 parent ab35fda299
commit 3a25487dea
215 files changed
+29740 -15

No files matched your search

@@ -0,0 +1,219 @@
package com.dbx.agent.hive;
import com.dbx.agent.AbstractJdbcAgent;
import com.dbx.agent.ColumnInfo;
import com.dbx.agent.ConnectParams;
import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.JsonRpcServer;
import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
public final class HiveAgent extends AbstractJdbcAgent {
@Override
protected String driverClass() {
return "org.apache.hive.jdbc.HiveDriver";
}
@Override
protected String buildJdbcUrl(ConnectParams params) {
return buildUrl(params);
}
@Override
public List<DatabaseInfo> listDatabases() {
return unchecked(() -> {
List<DatabaseInfo> result = new ArrayList<>();
try (java.sql.Statement stmt = requireConnected().createStatement();
ResultSet rs = stmt.executeQuery("SHOW DATABASES")) {
while (rs.next()) {
result.add(new DatabaseInfo(rs.getString(1)));
}
}
result.sort(Comparator.comparing(DatabaseInfo::getName));
return result;
});
}
@Override
public List<String> listSchemas() {
List<String> result = new ArrayList<>();
for (DatabaseInfo database : listDatabases()) {
result.add(database.getName());
}
return result;
}
@Override
public List<TableInfo> listTables(String schema) {
return unchecked(() -> {
List<TableInfo> result = new ArrayList<>();
useSchema(schema);
try (java.sql.Statement stmt = requireConnected().createStatement();
ResultSet rs = stmt.executeQuery("SHOW TABLES")) {
while (rs.next()) {
result.add(new TableInfo(rs.getString(1), "TABLE", null));
}
}
result.sort(Comparator.comparing(TableInfo::getName));
return result;
});
}
@Override
public List<ColumnInfo> getColumns(String schema, String table) {
return unchecked(() -> {
try {
return getColumnsFromDescribe(schema, table);
} catch (Exception ignored) {
return getColumnsFromMetadata(requireConnected(), schema, table);
}
});
}
@Override
public List<IndexInfo> listIndexes(String schema, String table) {
return Collections.emptyList();
}
@Override
public List<ForeignKeyInfo> listForeignKeys(String schema, String table) {
return Collections.emptyList();
}
@Override
public List<TriggerInfo> listTriggers(String schema, String table) {
return Collections.emptyList();
}
@Override
public String setSchemaSQL(String schema) {
return "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
}
@Override
protected Object resultValue(ResultSet rs, int index, int sqlType) {
return unchecked(() -> {
Object value = rs.getObject(index);
return rs.wasNull() ? null : value == null ? null : value.toString();
});
}
private void useSchema(String schema) throws Exception {
try (java.sql.Statement stmt = requireConnected().createStatement()) {
stmt.execute(setSchemaSQL(schema));
}
}
private List<ColumnInfo> getColumnsFromDescribe(String schema, String table) throws Exception {
List<ColumnInfo> result = new ArrayList<>();
useSchema(schema);
try (java.sql.Statement stmt = requireConnected().createStatement();
ResultSet rs = stmt.executeQuery("DESCRIBE " + JdbcIdentifiers.INSTANCE.backtick(table))) {
while (rs.next()) {
String colName = trimToNull(rs.getString(1));
if (colName == null || colName.startsWith("#")) {
continue;
}
result.add(new ColumnInfo(
colName,
trimToEmpty(rs.getString(2)),
true,
null,
false,
null,
optionalComment(rs),
null,
null,
null
));
}
}
return result;
}
private static List<ColumnInfo> getColumnsFromMetadata(Connection conn, String schema, String table) throws Exception {
List<ColumnInfo> result = new ArrayList<>();
DatabaseMetaData meta = conn.getMetaData();
try (ResultSet rs = meta.getColumns(null, trimToNull(schema), table, "%")) {
while (rs.next()) {
String name = rs.getString("COLUMN_NAME");
if (trimToNull(name) == null) {
continue;
}
result.add(new ColumnInfo(
name,
trimToEmpty(rs.getString("TYPE_NAME")),
rs.getInt("NULLABLE") != DatabaseMetaData.columnNoNulls,
rs.getString("COLUMN_DEF"),
false,
null,
rs.getString("REMARKS"),
intOrNull(rs, "COLUMN_SIZE"),
intOrNull(rs, "DECIMAL_DIGITS"),
characterLength(rs)
));
}
}
return result;
}
private static String buildUrl(ConnectParams params) {
return "jdbc:hive2://" + params.getHost() + ":" + params.getPort() + "/" + params.getDatabase();
}
private static String trimToNull(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed;
}
private static String trimToEmpty(String value) {
return value == null ? "" : value.trim();
}
private static Integer intOrNull(ResultSet rs, String column) throws Exception {
Object value = rs.getObject(column);
return value instanceof Number ? ((Number) value).intValue() : null;
}
private static Integer characterLength(ResultSet rs) throws Exception {
String typeName = rs.getString("TYPE_NAME");
if (typeName == null) {
return null;
}
String normalized = typeName.toLowerCase(Locale.ROOT);
if (!normalized.contains("char") && !normalized.contains("text") && !normalized.contains("string")) {
return null;
}
return intOrNull(rs, "COLUMN_SIZE");
}
private static String optionalComment(ResultSet rs) {
return unchecked(() -> {
try {
String comment = trimToNull(rs.getString(3));
return comment;
} catch (Exception e) {
return null;
}
});
}
public static void main(String[] args) {
new JsonRpcServer(new HiveAgent()).run();
}
}
@@ -0,0 +1,16 @@
package com.dbx.agent.hive;
import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
class HiveAgentExecutionTest extends JdbcFakeExecutionBehaviorTest {
@Override
protected DatabaseAgent createAgent() {
return new HiveAgent();
}
@Override
protected String resultSetSql() {
return "MSCK REPAIR TABLE sample";
}
}
@@ -0,0 +1,186 @@
package com.dbx.agent.hive;
import com.dbx.agent.test.TestSupport;
import com.dbx.agent.test.JdbcMetadataSqlFake;
import com.dbx.agent.ColumnInfo;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.util.List;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.LinkedHashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class HiveAgentMetadataTest {
@Test
void quotesSchemaAndTableIdentifiersInMetadataSql() {
HiveAgent agent = new HiveAgent();
Connection fake = JdbcMetadataSqlFake.connection();
TestSupport.setPrivateConnection(agent, fake);
agent.listTables("bad`schema");
agent.getColumns("bad`schema", "bad`table");
Assertions.assertEquals(
List.of(
"USE `bad``schema`",
"SHOW TABLES",
"USE `bad``schema`",
"DESCRIBE `bad``table`"
),
JdbcMetadataSqlFake.statements
);
}
@Test
void fallsBackToJdbcMetadataWhenDescribeCannotCreateResultSet() {
HiveAgent agent = new HiveAgent();
Connection fake = describeFailingMetadataConnection();
TestSupport.setPrivateConnection(agent, fake);
List<ColumnInfo> columns = agent.getColumns("default", "events");
Assertions.assertEquals(2, columns.size());
Assertions.assertEquals("id", columns.get(0).getName());
Assertions.assertEquals("bigint", columns.get(0).getData_type());
Assertions.assertFalse(columns.get(0).getIs_nullable());
Assertions.assertEquals("name", columns.get(1).getName());
Assertions.assertEquals("string", columns.get(1).getData_type());
Assertions.assertEquals(Integer.valueOf(255), columns.get(1).getCharacter_maximum_length());
}
private static Connection describeFailingMetadataConnection() {
DatabaseMetaData meta = proxy(DatabaseMetaData.class, (method, args) -> {
if ("getColumns".equals(method.getName())) {
return rows(
row(
"COLUMN_NAME", "id",
"TYPE_NAME", "bigint",
"NULLABLE", DatabaseMetaData.columnNoNulls,
"COLUMN_DEF", null,
"REMARKS", "identifier",
"COLUMN_SIZE", 19,
"DECIMAL_DIGITS", 0
),
row(
"COLUMN_NAME", "name",
"TYPE_NAME", "string",
"NULLABLE", DatabaseMetaData.columnNullable,
"COLUMN_DEF", null,
"REMARKS", null,
"COLUMN_SIZE", 255,
"DECIMAL_DIGITS", 0
)
);
}
return defaultValue(method.getReturnType());
});
return proxy(Connection.class, (method, args) -> {
String name = method.getName();
if ("createStatement".equals(name)) {
return describeFailingStatement();
}
if ("getMetaData".equals(name)) {
return meta;
}
if ("isClosed".equals(name)) {
return false;
}
return defaultValue(method.getReturnType());
});
}
private static Statement describeFailingStatement() {
return proxy(Statement.class, (method, args) -> {
String name = method.getName();
if ("execute".equals(name)) {
return false;
}
if ("executeQuery".equals(name)) {
throw new SQLException("Could not create ResultSet: Required field 'type' is unset! Struct:TPrimitiveTypeEntry(type:null)");
}
return defaultValue(method.getReturnType());
});
}
@SafeVarargs
private static ResultSet rows(Map<String, Object>... rows) {
return proxy(ResultSet.class, new MethodHandler() {
private int index = -1;
@Override
public Object handle(Method method, Object[] args) {
String name = method.getName();
if ("next".equals(name)) {
index += 1;
return index < rows.length;
}
if ("close".equals(name)) {
return null;
}
Object value = rows[index].get(args[0]);
if ("getString".equals(name)) {
return value == null ? null : String.valueOf(value);
}
if ("getObject".equals(name)) {
return value;
}
if ("getInt".equals(name)) {
return value instanceof Number ? ((Number) value).intValue() : 0;
}
return defaultValue(method.getReturnType());
}
});
}
private static Map<String, Object> row(Object... values) {
Map<String, Object> row = new LinkedHashMap<>();
for (int i = 0; i < values.length; i += 2) {
row.put(String.valueOf(values[i]), values[i + 1]);
}
return row;
}
private static <T> T proxy(Class<T> type, MethodHandler handler) {
InvocationHandler invocationHandler = (proxy, method, args) -> handler.handle(method, args);
return type.cast(Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[]{type}, invocationHandler));
}
private static Object defaultValue(Class<?> type) {
if (Boolean.TYPE.equals(type)) {
return false;
}
if (Byte.TYPE.equals(type)) {
return (byte) 0;
}
if (Short.TYPE.equals(type)) {
return (short) 0;
}
if (Integer.TYPE.equals(type)) {
return 0;
}
if (Long.TYPE.equals(type)) {
return 0L;
}
if (Float.TYPE.equals(type)) {
return 0f;
}
if (Double.TYPE.equals(type)) {
return 0.0d;
}
if (Character.TYPE.equals(type)) {
return '\0';
}
return null;
}
private interface MethodHandler {
Object handle(Method method, Object[] args) throws Throwable;
}
}
@@ -0,0 +1,11 @@
package com.dbx.agent.hive;
import org.junit.jupiter.api.Test;
class HiveAgentTest {
@Test
void hiveJdbcStandaloneRuntimeClassesAreAvailable() throws ClassNotFoundException {
Class.forName("org.apache.hive.jdbc.HiveDriver");
Class.forName("org.apache.hive.org.apache.thrift.protocol.TProtocol");
}
}