feat(transwarp): add Inceptor data source with table operations
This commit is contained in:
1 parent
597b134d21
commit
6fdc338348
101 files changed
+2300
-66
No files matched your search
@@ -0,0 +1,25 @@
|
||||
dependencies {
|
||||
implementation files('libs/inceptor-sdk-transwarp-6.1.0-SNAPSHOT.jar')
|
||||
testImplementation 'com.h2database:h2:2.4.240'
|
||||
}
|
||||
|
||||
tasks.named('shadowJar') {
|
||||
doFirst {
|
||||
if (!file('libs/inceptor-sdk-transwarp-6.1.0-SNAPSHOT.jar').isFile()) {
|
||||
throw new GradleException('Missing bundled Transwarp SDK in libs/inceptor-sdk-transwarp-6.1.0-SNAPSHOT.jar')
|
||||
}
|
||||
}
|
||||
manifest {
|
||||
attributes(
|
||||
'Agent-Label': 'Transwarp Inceptor',
|
||||
'Main-Class': 'com.dbx.agent.transwarp.TranswarpAgent'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tasks.named('test') {
|
||||
dependsOn tasks.named('shadowJar')
|
||||
doFirst {
|
||||
systemProperty 'dbx.transwarp.agent.jar', tasks.named('shadowJar').get().archiveFile.get().asFile.absolutePath
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,443 @@
|
||||
package com.dbx.agent.transwarp;
|
||||
|
||||
import com.dbx.agent.AbstractJdbcAgent;
|
||||
import com.dbx.agent.ColumnInfo;
|
||||
import com.dbx.agent.ConnectParams;
|
||||
import com.dbx.agent.DatabaseInfo;
|
||||
import com.dbx.agent.ForeignKeyInfo;
|
||||
import com.dbx.agent.IndexInfo;
|
||||
import com.dbx.agent.JdbcIdentifiers;
|
||||
import com.dbx.agent.MetadataListConstraints;
|
||||
import com.dbx.agent.MultiSessionJsonRpcServer;
|
||||
import com.dbx.agent.ObjectInfo;
|
||||
import com.dbx.agent.ObjectSource;
|
||||
import com.dbx.agent.PartitionInfo;
|
||||
import com.dbx.agent.QueryResult;
|
||||
import com.dbx.agent.TableInfo;
|
||||
import com.dbx.agent.TriggerInfo;
|
||||
import org.apache.hive.jdbc.HiveConnection;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public final class TranswarpAgent extends AbstractJdbcAgent {
|
||||
private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver";
|
||||
private boolean transactionStateUnknown;
|
||||
|
||||
@Override
|
||||
protected void afterConnect(ConnectParams params, Connection connection) {
|
||||
transactionStateUnknown = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void beforePooledConnectionReturn(Connection connection) throws Exception {
|
||||
if (transactionStateUnknown) {
|
||||
throw new SQLException("Transwarp transaction state is unknown after rollback failure");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String connectionValidationQuery() {
|
||||
return "SELECT 1";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String driverClass() {
|
||||
return DRIVER_CLASS;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String buildJdbcUrl(ConnectParams params) {
|
||||
String supplied = params.getConnection_string();
|
||||
if (supplied != null && !supplied.isBlank()) {
|
||||
return supplied.trim();
|
||||
}
|
||||
String host = params.getHost() == null ? "" : params.getHost().trim();
|
||||
if (host.isEmpty()) {
|
||||
throw new IllegalArgumentException("Transwarp host is required");
|
||||
}
|
||||
int port = params.getPort() > 0 ? params.getPort() : 10000;
|
||||
String database = database(params.getDatabase());
|
||||
StringBuilder url = new StringBuilder("jdbc:inceptor2://")
|
||||
.append(host).append(':').append(port).append('/').append(database);
|
||||
String options = params.getUrl_params();
|
||||
if (options != null && !options.isBlank()) {
|
||||
url.append(';').append(options.replaceFirst("^;+", ""));
|
||||
}
|
||||
if (params.isSsl() && (options == null || !options.toLowerCase(Locale.ROOT).contains("ssl="))) {
|
||||
url.append(";ssl=true");
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
private static String database(String value) {
|
||||
return value == null || value.isBlank() ? "default" : value.trim();
|
||||
}
|
||||
|
||||
private String schema(String value) {
|
||||
return value == null || value.isBlank() ? database(getConfiguredDatabase()) : value.trim();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String setSchemaSQL(String schema) {
|
||||
return schema == null || schema.isBlank() ? "" : "USE " + JdbcIdentifiers.INSTANCE.backtick(schema);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatabaseInfo> listDatabases() {
|
||||
return query("SELECT database_name FROM system.databases_v ORDER BY database_name", List.of(),
|
||||
row -> new DatabaseInfo(row.getString(1)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listSchemas() {
|
||||
return listDatabases().stream().map(DatabaseInfo::getName).toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TableInfo> listTables(String schema) {
|
||||
return listTables(schema, MetadataListConstraints.NONE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
|
||||
String catalog = schema(schema);
|
||||
MetadataListConstraints requested = MetadataListConstraints.orNone(constraints);
|
||||
List<TableInfo> tables = new ArrayList<>();
|
||||
if (!requested.hasObjectTypes() || requested.tableTypeAllowed("TABLE") || requested.tableTypeAllowed("MATERIALIZED_VIEW")) {
|
||||
tables.addAll(query(
|
||||
"SELECT table_name, table_type, commentstring FROM system.tables_v WHERE database_name=? ORDER BY table_name",
|
||||
List.of(catalog), row -> new TableInfo(row.getString(1), tableType(row.getString(2)), row.getString(3))));
|
||||
}
|
||||
if (!requested.hasObjectTypes() || requested.tableTypeAllowed("VIEW")) {
|
||||
tables.addAll(query("SELECT view_name FROM system.views_v WHERE database_name=? ORDER BY view_name",
|
||||
List.of(catalog), row -> new TableInfo(row.getString(1), "VIEW", null)));
|
||||
}
|
||||
tables.sort(Comparator.comparing(TableInfo::getName));
|
||||
return requested.filterTables(tables);
|
||||
}
|
||||
|
||||
private static String tableType(String value) {
|
||||
if (value == null) return "TABLE";
|
||||
String normalized = value.toUpperCase(Locale.ROOT).replace(' ', '_');
|
||||
if (normalized.contains("MATERIALIZED") && normalized.contains("VIEW")) return "MATERIALIZED_VIEW";
|
||||
return normalized.contains("VIEW") ? "VIEW" : "TABLE";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ObjectInfo> listObjects(String schema) {
|
||||
return listObjects(schema, MetadataListConstraints.NONE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
|
||||
String catalog = schema(schema);
|
||||
MetadataListConstraints requested = MetadataListConstraints.orNone(constraints);
|
||||
List<ObjectInfo> objects = new ArrayList<>();
|
||||
if (requested.includesTableLikeTypes()) {
|
||||
for (TableInfo table : listTables(catalog, requested.withoutPaging())) {
|
||||
objects.add(new ObjectInfo(table.getName(), table.getTable_type(), catalog, table.getComment()));
|
||||
}
|
||||
}
|
||||
if (!requested.hasObjectTypes() || requested.objectTypeAllowed("PROCEDURE")) {
|
||||
objects.addAll(namedObjects(catalog, "system.procedures_v", "procedure_name", "PROCEDURE"));
|
||||
}
|
||||
if (!requested.hasObjectTypes() || requested.objectTypeAllowed("FUNCTION")) {
|
||||
objects.addAll(namedObjects(catalog, "system.functions_v", "function_name", "FUNCTION"));
|
||||
}
|
||||
if (!requested.hasObjectTypes() || requested.objectTypeAllowed("PACKAGE") || requested.objectTypeAllowed("PACKAGE_BODY")) {
|
||||
objects.addAll(optionalCatalogQuery(
|
||||
"SELECT package_name, package_body FROM system.packages_v WHERE database_name=? ORDER BY package_name",
|
||||
List.of(catalog), row -> {
|
||||
List<ObjectInfo> entries = new ArrayList<>();
|
||||
String name = row.getString(1);
|
||||
entries.add(new ObjectInfo(name, "PACKAGE", catalog, null));
|
||||
if (row.getString(2) != null) {
|
||||
entries.add(new ObjectInfo(name, "PACKAGE_BODY", catalog, null));
|
||||
}
|
||||
return entries;
|
||||
}).stream().flatMap(List::stream).toList());
|
||||
}
|
||||
objects.sort(Comparator.comparing(ObjectInfo::getName).thenComparing(ObjectInfo::getObject_type));
|
||||
return requested.filterObjects(objects);
|
||||
}
|
||||
|
||||
private List<ObjectInfo> namedObjects(String schema, String view, String nameColumn, String type) {
|
||||
return optionalCatalogQuery("SELECT " + nameColumn + " FROM " + view + " WHERE database_name=? ORDER BY " + nameColumn,
|
||||
List.of(schema), row -> new ObjectInfo(row.getString(1), type, schema, null));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listDataTypes() {
|
||||
return unchecked(() -> {
|
||||
Set<String> types = new LinkedHashSet<>();
|
||||
try (ResultSet rows = requireConnected().getMetaData().getTypeInfo()) {
|
||||
while (rows.next()) {
|
||||
String name = rows.getString("TYPE_NAME");
|
||||
if (name != null && !name.isBlank()) types.add(name);
|
||||
}
|
||||
}
|
||||
return types.stream().sorted().toList();
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ColumnInfo> getColumns(String schema, String table) {
|
||||
String catalog = schema(schema);
|
||||
Set<String> primaryKeys = unchecked(() -> {
|
||||
Set<String> keys = new HashSet<>();
|
||||
DatabaseMetaData metadata = requireConnected().getMetaData();
|
||||
String metadataSchema = metadata.storesUpperCaseIdentifiers() ? catalog.toUpperCase(Locale.ROOT)
|
||||
: metadata.storesLowerCaseIdentifiers() ? catalog.toLowerCase(Locale.ROOT) : catalog;
|
||||
String metadataTable = metadata.storesUpperCaseIdentifiers() ? table.toUpperCase(Locale.ROOT)
|
||||
: metadata.storesLowerCaseIdentifiers() ? table.toLowerCase(Locale.ROOT) : table;
|
||||
try (ResultSet rows = metadata.getPrimaryKeys(null, metadataSchema, metadataTable)) {
|
||||
while (rows.next()) keys.add(rows.getString("COLUMN_NAME").toLowerCase(Locale.ROOT));
|
||||
} catch (SQLException ignored) {
|
||||
// The system catalog still provides column metadata on SDKs without PK metadata.
|
||||
}
|
||||
return keys;
|
||||
});
|
||||
Map<String, int[]> dimensions = unchecked(() -> {
|
||||
Map<String, int[]> sizes = new HashMap<>();
|
||||
DatabaseMetaData metadata = requireConnected().getMetaData();
|
||||
String metadataSchema = metadata.storesUpperCaseIdentifiers() ? catalog.toUpperCase(Locale.ROOT)
|
||||
: metadata.storesLowerCaseIdentifiers() ? catalog.toLowerCase(Locale.ROOT) : catalog;
|
||||
String metadataTable = metadata.storesUpperCaseIdentifiers() ? table.toUpperCase(Locale.ROOT)
|
||||
: metadata.storesLowerCaseIdentifiers() ? table.toLowerCase(Locale.ROOT) : table;
|
||||
try (ResultSet rows = metadata.getColumns(null, metadataSchema, metadataTable, null)) {
|
||||
while (rows.next()) {
|
||||
String column = rows.getString("COLUMN_NAME");
|
||||
int size = rows.getInt("COLUMN_SIZE");
|
||||
int scale = rows.getInt("DECIMAL_DIGITS");
|
||||
sizes.put(column.toLowerCase(Locale.ROOT), new int[]{size, scale});
|
||||
}
|
||||
} catch (SQLException ignored) {
|
||||
// The system catalog remains the source for type names and other attributes.
|
||||
}
|
||||
return sizes;
|
||||
});
|
||||
return query("SELECT column_name, column_type, nullable, default_value, commentstring "
|
||||
+ "FROM system.columns_v WHERE database_name=? AND table_name=? ORDER BY column_id",
|
||||
List.of(catalog, table), row -> {
|
||||
String name = row.getString(1);
|
||||
String type = row.getString(2);
|
||||
int[] size = dimensions.get(name.toLowerCase(Locale.ROOT));
|
||||
boolean numeric = type != null && type.matches("(?i)^(decimal|numeric|number)(\\(.*)?$");
|
||||
boolean character = type != null && type.matches("(?i)^(char|varchar|character)(\\(.*)?$");
|
||||
return new ColumnInfo(name, type, nullable(row.getObject(3)), row.getString(4),
|
||||
primaryKeys.contains(name.toLowerCase(Locale.ROOT)), null, row.getString(5),
|
||||
numeric && size != null ? size[0] : null, numeric && size != null ? size[1] : null,
|
||||
character && size != null ? size[0] : null);
|
||||
});
|
||||
}
|
||||
|
||||
private static boolean nullable(Object value) {
|
||||
if (value == null) return true;
|
||||
if (value instanceof Boolean flag) return flag;
|
||||
if (value instanceof Number number) return number.intValue() != 0;
|
||||
String text = value.toString().trim().toUpperCase(Locale.ROOT);
|
||||
return !text.equals("NO") && !text.equals("N") && !text.equals("FALSE") && !text.equals("0") && !text.equals("NOT NULL");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<IndexInfo> listIndexes(String schema, String table) {
|
||||
// Waterdrop exposes no index list, and the SDK's getIndexInfo throws SQLException.
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ForeignKeyInfo> listForeignKeys(String schema, String table) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TriggerInfo> listTriggers(String schema, String table) {
|
||||
String catalog = schema(schema);
|
||||
return optionalCatalogQuery("SELECT * FROM system.triggers_v WHERE database_name=? AND table_name=?",
|
||||
List.of(catalog, table), row -> new TriggerInfo(
|
||||
firstColumn(row, "trigger_name", "name", "trigger"),
|
||||
firstColumn(row, "event", "event_type", "trigger_event"),
|
||||
firstColumn(row, "timing", "trigger_timing", "action_timing")))
|
||||
.stream()
|
||||
.filter(trigger -> trigger.getName() != null && !trigger.getName().isBlank())
|
||||
.toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PartitionInfo> listPartitions(String schema, String table) {
|
||||
String catalog = schema(schema);
|
||||
List<PartitionInfo> partitions = new ArrayList<>();
|
||||
partitions.addAll(query("SELECT partition_name, partition_value, partition_key FROM system.partitions_v "
|
||||
+ "WHERE database_name=? AND table_name=? ORDER BY partition_id",
|
||||
List.of(catalog, table), row -> new PartitionInfo(row.getString(1), 0, row.getString(2), "LIST", row.getString(3))));
|
||||
partitions.addAll(optionalCatalogQuery("SELECT partition_name, partition_range, partition_key FROM system.range_partitions_v "
|
||||
+ "WHERE database_name=? AND table_name=? ORDER BY partition_id",
|
||||
List.of(catalog, table), row -> new PartitionInfo(row.getString(1), 0, row.getString(2), "RANGE", row.getString(3))));
|
||||
partitions.addAll(optionalCatalogQuery("SELECT * FROM system.buckets_v WHERE database_name=? AND table_name=?",
|
||||
List.of(catalog, table), row -> new PartitionInfo(
|
||||
firstColumn(row, "bucket_name", "name", "bucket"),
|
||||
0,
|
||||
firstColumn(row, "bucket_value", "value", "bucket_count", "count", "bucket_number"),
|
||||
"BUCKET",
|
||||
firstColumn(row, "bucket_key", "bucket_column", "column_name", "key"))));
|
||||
List<PartitionInfo> numbered = new ArrayList<>();
|
||||
for (int index = 0; index < partitions.size(); index++) {
|
||||
PartitionInfo partition = partitions.get(index);
|
||||
String name = partition.name();
|
||||
if (name == null || name.isBlank()) {
|
||||
name = partition.partition_type().equals("BUCKET") ? "bucket-" + (index + 1) : "partition-" + (index + 1);
|
||||
}
|
||||
numbered.add(new PartitionInfo(name, index + 1, partition.value(),
|
||||
partition.partition_type(), partition.partition_key()));
|
||||
}
|
||||
return numbered;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTableDdl(String schema, String table) {
|
||||
String sql = "SHOW CREATE TABLE " + JdbcIdentifiers.INSTANCE.backtick(schema(schema))
|
||||
+ "." + JdbcIdentifiers.INSTANCE.backtick(table);
|
||||
return unchecked(() -> {
|
||||
List<String> lines = new ArrayList<>();
|
||||
try (Statement statement = requireConnected().createStatement(); ResultSet rows = statement.executeQuery(sql)) {
|
||||
while (rows.next()) {
|
||||
String line = rows.getString(1);
|
||||
if (line != null && !line.isBlank()) lines.add(line);
|
||||
}
|
||||
}
|
||||
return String.join("\n", lines);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectSource getObjectSource(String schema, String name, String objectType) {
|
||||
String catalog = schema(schema);
|
||||
String type = objectType.toUpperCase(Locale.ROOT);
|
||||
if (type.equals("TABLE")) {
|
||||
return new ObjectSource(name, type, catalog, getTableDdl(catalog, name));
|
||||
}
|
||||
String view;
|
||||
String nameColumn;
|
||||
String sourceColumn;
|
||||
switch (type) {
|
||||
case "VIEW" -> { view = "system.views_v"; nameColumn = "view_name"; sourceColumn = "origin_text"; }
|
||||
case "PROCEDURE" -> { view = "system.procedures_v"; nameColumn = "procedure_name"; sourceColumn = "full_text"; }
|
||||
case "FUNCTION" -> { view = "system.functions_v"; nameColumn = "function_name"; sourceColumn = "full_text"; }
|
||||
case "PACKAGE" -> { view = "system.packages_v"; nameColumn = "package_name"; sourceColumn = "full_text"; }
|
||||
case "PACKAGE_BODY" -> { view = "system.packages_v"; nameColumn = "package_name"; sourceColumn = "package_body"; }
|
||||
default -> throw new UnsupportedOperationException("Unsupported Transwarp object type: " + type);
|
||||
}
|
||||
List<String> source = query("SELECT " + sourceColumn + " FROM " + view + " WHERE database_name=? AND "
|
||||
+ nameColumn + "=?", List.of(catalog, name), row -> row.getString(1));
|
||||
if (source.isEmpty() || source.stream().allMatch(value -> value == null || value.isBlank())) {
|
||||
throw new IllegalArgumentException(type + " source not found: " + catalog + "." + name);
|
||||
}
|
||||
return new ObjectSource(name, type, catalog,
|
||||
String.join("\n", source.stream().filter(value -> value != null && !value.isBlank()).toList()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult executeTransaction(List<String> statements, String schema) {
|
||||
return unchecked(() -> {
|
||||
long started = System.currentTimeMillis();
|
||||
long affected = 0;
|
||||
Connection connection = requireConnected();
|
||||
applySchemaContext(connection, schema);
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
if (connection instanceof HiveConnection) {
|
||||
statement.execute("SET transaction.type = inceptor");
|
||||
}
|
||||
statement.execute("BEGIN TRANSACTION");
|
||||
try {
|
||||
for (String sql : statements) {
|
||||
if (sql != null && !sql.isBlank()) {
|
||||
statement.execute(sql);
|
||||
affected += Math.max(statement.getUpdateCount(), 0);
|
||||
}
|
||||
}
|
||||
statement.execute("COMMIT");
|
||||
} catch (Exception error) {
|
||||
try {
|
||||
statement.execute("ROLLBACK");
|
||||
} catch (Exception rollbackError) {
|
||||
transactionStateUnknown = true;
|
||||
error.addSuppressed(rollbackError);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return new QueryResult(List.of(), List.of(), affected, System.currentTimeMillis() - started);
|
||||
});
|
||||
}
|
||||
|
||||
private interface RowReader<T> {
|
||||
T read(ResultSet row) throws Exception;
|
||||
}
|
||||
|
||||
private <T> List<T> query(String sql, List<String> parameters, RowReader<T> reader) {
|
||||
return unchecked(() -> {
|
||||
List<T> values = new ArrayList<>();
|
||||
try (PreparedStatement statement = requireConnected().prepareStatement(sql)) {
|
||||
for (int index = 0; index < parameters.size(); index++) {
|
||||
statement.setString(index + 1, parameters.get(index));
|
||||
}
|
||||
try (ResultSet rows = statement.executeQuery()) {
|
||||
while (rows.next()) values.add(reader.read(rows));
|
||||
}
|
||||
}
|
||||
return values;
|
||||
});
|
||||
}
|
||||
|
||||
private <T> List<T> optionalCatalogQuery(String sql, List<String> parameters, RowReader<T> reader) {
|
||||
try {
|
||||
return query(sql, parameters, reader);
|
||||
} catch (RuntimeException error) {
|
||||
Throwable cause = error;
|
||||
while (cause != null) {
|
||||
if (cause instanceof SQLException sqlError) {
|
||||
if ("42S02".equals(sqlError.getSQLState()) || "42P01".equals(sqlError.getSQLState())) {
|
||||
return List.of();
|
||||
}
|
||||
break;
|
||||
}
|
||||
cause = cause.getCause();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static String firstColumn(ResultSet row, String... candidates) throws SQLException {
|
||||
ResultSetMetaData metadata = row.getMetaData();
|
||||
for (String candidate : candidates) {
|
||||
for (int index = 1; index <= metadata.getColumnCount(); index++) {
|
||||
String label = metadata.getColumnLabel(index);
|
||||
String name = metadata.getColumnName(index);
|
||||
if (candidate.equalsIgnoreCase(label) || candidate.equalsIgnoreCase(name)) {
|
||||
Object value = row.getObject(index);
|
||||
return value == null ? null : value.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
new MultiSessionJsonRpcServer(TranswarpAgent::new).run();
|
||||
}
|
||||
}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
package com.dbx.agent.transwarp;
|
||||
|
||||
import com.dbx.agent.ColumnInfo;
|
||||
import com.dbx.agent.ConnectParams;
|
||||
import com.dbx.agent.MetadataListConstraints;
|
||||
import com.dbx.agent.ObjectInfo;
|
||||
import com.dbx.agent.PartitionInfo;
|
||||
import com.dbx.agent.QueryPageOptions;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.sql.Driver;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.Statement;
|
||||
import java.util.List;
|
||||
import java.util.jar.JarFile;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class TranswarpAgentTest {
|
||||
private static final String JDBC_URL = "jdbc:h2:mem:transwarp_agent;DB_CLOSE_DELAY=-1";
|
||||
|
||||
@BeforeAll
|
||||
static void loadTestDatabaseDriver() throws Exception {
|
||||
Class.forName("org.h2.Driver");
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildsVendorUrlForBothProductProfiles() {
|
||||
TranswarpAgent agent = new TranswarpAgent();
|
||||
ConnectParams params = new ConnectParams("quark.example", 10000, "analytics", "user", "", "fetchSize=1000", "", false);
|
||||
params.setSsl(true);
|
||||
assertEquals("jdbc:inceptor2://quark.example:10000/analytics;fetchSize=1000;ssl=true", agent.buildJdbcUrl(params));
|
||||
params.setConnection_string("jdbc:transwarp2://a:10000,b:10000/analytics;serviceDiscoveryMode=zooKeeper");
|
||||
assertEquals(params.getConnection_string(), agent.buildJdbcUrl(params));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedIndexAndForeignKeyMetadataDoesNotCallTheSdk() {
|
||||
TranswarpAgent agent = new TranswarpAgent();
|
||||
assertTrue(agent.listIndexes("demo", "orders").isEmpty());
|
||||
assertTrue(agent.listForeignKeys("demo", "orders").isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void readsWaterdropCatalogViewsWithoutHidingPackagesOrPartitions() throws Exception {
|
||||
try (Connection connection = DriverManager.getConnection(JDBC_URL); Statement sql = connection.createStatement()) {
|
||||
sql.execute("CREATE SCHEMA IF NOT EXISTS system");
|
||||
sql.execute("CREATE SCHEMA IF NOT EXISTS demo");
|
||||
sql.execute("DROP TABLE IF EXISTS demo.orders");
|
||||
sql.execute("CREATE TABLE demo.orders (id INT PRIMARY KEY, amount DECIMAL(12,2))");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.databases_v (database_name VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.tables_v (database_name VARCHAR, table_name VARCHAR, table_type VARCHAR, commentstring VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.views_v (database_name VARCHAR, view_name VARCHAR, origin_text VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.columns_v (database_name VARCHAR, table_name VARCHAR, column_id INT, column_name VARCHAR, column_type VARCHAR, nullable VARCHAR, default_value VARCHAR, commentstring VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.procedures_v (database_name VARCHAR, procedure_name VARCHAR, full_text VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.functions_v (database_name VARCHAR, function_name VARCHAR, full_text VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.packages_v (database_name VARCHAR, package_name VARCHAR, full_text VARCHAR, package_body VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.partitions_v (database_name VARCHAR, table_name VARCHAR, partition_id INT, partition_name VARCHAR, partition_value VARCHAR, partition_key VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.range_partitions_v (database_name VARCHAR, table_name VARCHAR, partition_id INT, partition_name VARCHAR, partition_range VARCHAR, partition_key VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.buckets_v (database_name VARCHAR, table_name VARCHAR, bucket_id INT, bucket_name VARCHAR, bucket_count INT, bucket_column VARCHAR)");
|
||||
sql.execute("CREATE TABLE IF NOT EXISTS system.triggers_v (database_name VARCHAR, table_name VARCHAR, trigger_name VARCHAR, event VARCHAR, timing VARCHAR)");
|
||||
sql.execute("INSERT INTO system.databases_v VALUES ('demo')");
|
||||
sql.execute("INSERT INTO system.tables_v VALUES ('demo', 'orders', 'MANAGED_TABLE', 'sales')");
|
||||
sql.execute("INSERT INTO system.tables_v VALUES ('demo', 'daily_mv', 'MATERIALIZED VIEW', NULL)");
|
||||
sql.execute("INSERT INTO system.views_v VALUES ('demo', 'summary', 'CREATE VIEW summary AS SELECT id FROM orders')");
|
||||
sql.execute("INSERT INTO system.columns_v VALUES ('demo', 'orders', 1, 'id', 'INT', 'false', NULL, 'order key')");
|
||||
sql.execute("INSERT INTO system.columns_v VALUES ('demo', 'orders', 2, 'amount', 'DECIMAL(12,2)', 'true', '0', NULL)");
|
||||
sql.execute("INSERT INTO system.procedures_v VALUES ('demo', 'load_orders', 'CREATE PROCEDURE load_orders() BEGIN END')");
|
||||
sql.execute("INSERT INTO system.functions_v VALUES ('demo', 'total', 'CREATE FUNCTION total() RETURNS INT')");
|
||||
sql.execute("INSERT INTO system.packages_v VALUES ('demo', 'reports', 'CREATE PACKAGE reports', 'CREATE PACKAGE BODY reports')");
|
||||
sql.execute("INSERT INTO system.partitions_v VALUES ('demo', 'orders', 1, 'west', 'W', 'region')");
|
||||
sql.execute("INSERT INTO system.range_partitions_v VALUES ('demo', 'orders', 2, 'recent', '2025', 'year')");
|
||||
sql.execute("INSERT INTO system.buckets_v VALUES ('demo', 'orders', 1, 'bucket_0', 8, 'id')");
|
||||
sql.execute("INSERT INTO system.triggers_v VALUES ('demo', 'orders', 'orders_audit', 'INSERT', 'AFTER')");
|
||||
}
|
||||
|
||||
TranswarpAgent agent = connectedAgent();
|
||||
try {
|
||||
assertEquals(List.of("demo"), agent.listSchemas());
|
||||
assertEquals("H2", agent.getDatabaseInfo().get("productName"));
|
||||
assertEquals(List.of("daily_mv", "orders", "summary"), agent.listTables("demo").stream().map(table -> table.getName()).toList());
|
||||
assertEquals(List.of("daily_mv"), agent.listTables("demo", new MetadataListConstraints(null, null, null, List.of("MATERIALIZED_VIEW")))
|
||||
.stream().map(table -> table.getName()).toList());
|
||||
List<ObjectInfo> objects = agent.listObjects("demo", MetadataListConstraints.NONE);
|
||||
assertTrue(objects.stream().anyMatch(item -> item.getObject_type().equals("PACKAGE_BODY") && item.getName().equals("reports")));
|
||||
assertTrue(objects.stream().anyMatch(item -> item.getObject_type().equals("PROCEDURE")));
|
||||
assertEquals(List.of("reports"), agent.listObjects("demo", new MetadataListConstraints(null, null, null, List.of("PACKAGE")))
|
||||
.stream().map(ObjectInfo::getName).toList());
|
||||
List<ColumnInfo> columns = agent.getColumns("demo", "orders");
|
||||
assertEquals(2, columns.size());
|
||||
assertTrue(columns.get(0).getIs_primary_key());
|
||||
assertFalse(columns.get(0).getIs_nullable());
|
||||
assertEquals("0", columns.get(1).getColumn_default());
|
||||
assertEquals(12, columns.get(1).getNumeric_precision());
|
||||
assertEquals(2, columns.get(1).getNumeric_scale());
|
||||
assertEquals("CREATE PACKAGE BODY reports", agent.getObjectSource("demo", "reports", "PACKAGE_BODY").getSource());
|
||||
List<PartitionInfo> partitions = agent.listPartitions("demo", "orders");
|
||||
assertEquals(3, partitions.size());
|
||||
assertEquals("LIST", partitions.get(0).partition_type());
|
||||
assertEquals("RANGE", partitions.get(1).partition_type());
|
||||
assertEquals("BUCKET", partitions.get(2).partition_type());
|
||||
assertEquals("bucket_0", partitions.get(2).name());
|
||||
assertEquals("8", partitions.get(2).value());
|
||||
assertEquals("id", partitions.get(2).partition_key());
|
||||
assertEquals("orders_audit", agent.listTriggers("demo", "orders").get(0).getName());
|
||||
try (Connection connection = DriverManager.getConnection(JDBC_URL); Statement sql = connection.createStatement()) {
|
||||
sql.execute("DROP TABLE system.range_partitions_v");
|
||||
sql.execute("DROP TABLE system.buckets_v");
|
||||
sql.execute("DROP TABLE system.triggers_v");
|
||||
sql.execute("DROP TABLE system.packages_v");
|
||||
}
|
||||
assertEquals(1, agent.listPartitions("demo", "orders").size());
|
||||
assertTrue(agent.listObjects("demo").stream().anyMatch(item -> item.getName().equals("load_orders")));
|
||||
} finally {
|
||||
agent.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void transactionFailureRollsBackEarlierStatement() throws Exception {
|
||||
try (Connection connection = DriverManager.getConnection(JDBC_URL); Statement sql = connection.createStatement()) {
|
||||
sql.execute("CREATE SCHEMA IF NOT EXISTS demo");
|
||||
sql.execute("DROP TABLE IF EXISTS demo.tx_test");
|
||||
sql.execute("CREATE TABLE demo.tx_test (id INT PRIMARY KEY)");
|
||||
}
|
||||
TranswarpAgent agent = connectedAgent();
|
||||
try {
|
||||
assertThrows(RuntimeException.class, () -> agent.executeTransaction(
|
||||
List.of("INSERT INTO demo.tx_test VALUES (1)", "INSERT INTO demo.tx_test VALUES (1)"), ""));
|
||||
try (Connection connection = DriverManager.getConnection(JDBC_URL);
|
||||
var sql = connection.createStatement();
|
||||
var rows = sql.executeQuery("SELECT COUNT(*) FROM demo.tx_test")) {
|
||||
assertTrue(rows.next());
|
||||
assertEquals(0, rows.getInt(1));
|
||||
}
|
||||
} finally {
|
||||
agent.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void pagesQueryResultsThroughSharedJdbcExecutor() throws Exception {
|
||||
try (Connection connection = DriverManager.getConnection(JDBC_URL); Statement sql = connection.createStatement()) {
|
||||
sql.execute("CREATE SCHEMA IF NOT EXISTS demo");
|
||||
sql.execute("DROP TABLE IF EXISTS demo.page_test");
|
||||
sql.execute("CREATE TABLE demo.page_test (id INT PRIMARY KEY)");
|
||||
sql.execute("INSERT INTO demo.page_test VALUES (1), (2), (3)");
|
||||
}
|
||||
TranswarpAgent agent = connectedAgent();
|
||||
try {
|
||||
var first = agent.executeQueryPage("SELECT id FROM demo.page_test ORDER BY id", "",
|
||||
new QueryPageOptions(2, null, 10));
|
||||
assertEquals(2, first.getRows().size());
|
||||
assertTrue(first.getHas_more());
|
||||
var next = agent.fetchQueryPage(first.getSession_id(), 2);
|
||||
assertEquals(1, next.getRows().size());
|
||||
assertFalse(next.getHas_more());
|
||||
} finally {
|
||||
agent.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void packagedAgentLoadsDefaultDriverWithoutExternalClasspath() throws Exception {
|
||||
Path jar = Path.of(System.getProperty("dbx.transwarp.agent.jar"));
|
||||
assertTrue(Files.isRegularFile(jar));
|
||||
try (JarFile archive = new JarFile(jar.toFile())) {
|
||||
assertFalse(Boolean.parseBoolean(archive.getManifest().getMainAttributes().getValue("Agent-External-Driver")));
|
||||
}
|
||||
try (URLClassLoader loader = new URLClassLoader(new URL[]{jar.toUri().toURL()}, ClassLoader.getPlatformClassLoader())) {
|
||||
Class<?> paramsClass = loader.loadClass("com.dbx.agent.ConnectParams");
|
||||
Object params = paramsClass.getDeclaredConstructor().newInstance();
|
||||
Object agent = loader.loadClass("com.dbx.agent.transwarp.TranswarpAgent").getDeclaredConstructor().newInstance();
|
||||
var loadDriver = loader.loadClass("com.dbx.agent.AbstractJdbcAgent").getDeclaredMethod("loadDriver", paramsClass);
|
||||
loadDriver.setAccessible(true);
|
||||
loadDriver.invoke(agent, params);
|
||||
Driver driver = (Driver) Class.forName("org.apache.hive.jdbc.HiveDriver", true, loader)
|
||||
.getDeclaredConstructor().newInstance();
|
||||
assertTrue(driver.acceptsURL("jdbc:inceptor2://localhost:10000/default"));
|
||||
assertTrue(driver.acceptsURL("jdbc:transwarp2://localhost:10000/default"));
|
||||
assertTrue(driver.acceptsURL("jdbc:hive2://localhost:10000/default"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatesUsingSharedJdbcConnectionLifecycle() {
|
||||
ConnectParams params = new ConnectParams("", 0, "demo", "", "", "", JDBC_URL, false);
|
||||
params.setJdbc_driver_class("org.h2.Driver");
|
||||
var result = new TranswarpAgent().testConnectionWithInfo(params);
|
||||
assertEquals(true, result.get("ok"));
|
||||
assertEquals("SELECT 1", result.get("validation"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void packagedAgentRunsSharedJsonRpcProtocol() throws Exception {
|
||||
String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
|
||||
Process process = new ProcessBuilder(java, "-jar", System.getProperty("dbx.transwarp.agent.jar"))
|
||||
.redirectErrorStream(true).start();
|
||||
try {
|
||||
try (var input = process.getOutputStream()) {
|
||||
input.write("{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"shutdown\",\"params\":{}}\n".getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
assertTrue(process.waitFor(10, TimeUnit.SECONDS), "Packaged Agent did not shut down");
|
||||
String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
|
||||
assertEquals(0, process.exitValue(), output);
|
||||
assertTrue(output.contains("\"ready\":true"), output);
|
||||
assertTrue(output.contains("\"result\":"), output);
|
||||
assertFalse(output.contains("\"error\":"), output);
|
||||
} finally {
|
||||
process.destroyForcibly();
|
||||
process.waitFor(5, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
private static TranswarpAgent connectedAgent() {
|
||||
ConnectParams params = new ConnectParams("", 0, "demo", "", "", "", JDBC_URL, false);
|
||||
params.setJdbc_driver_class("org.h2.Driver");
|
||||
params.setDriver_profile("transwarp-inceptor");
|
||||
TranswarpAgent agent = new TranswarpAgent();
|
||||
agent.connect(params);
|
||||
return agent;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user