From e0da2d09d36958e17a9aaae291b339469d660223 Mon Sep 17 00:00:00 2001 From: t8y2 Date: Fri, 25 Sep 2026 15:51:58 +0800 Subject: [PATCH] refactor(workspace): split SQL and database driver crates --- .github/scripts/ci-config.mjs | 11 +- .github/scripts/ci-plan.mjs | 4 +- .github/scripts/ci-plan.test.mjs | 21 +- .github/scripts/ci-rust.mjs | 3 + .github/scripts/label-pull-request.mjs | 2 +- .github/scripts/label-pull-request.test.mjs | 6 +- .github/workflows/ci.yml | 6 +- Cargo.lock | 241 +++++++++++- Cargo.toml | 2 +- .../java/com/dbx/agent/EwkbWktDecoder.java | 2 +- .../java/com/dbx/agent/PostgresLikeAgent.java | 2 +- .../com/dbx/agent/EwkbWktDecoderTest.java | 2 +- agents/drivers/duckdb/Cargo.lock | 232 ++++++++++- .../RedisKeyBrowser.infiniteScroll.spec.ts | 2 +- ...idebarTableMutationRuntime.saphana.spec.ts | 2 +- .../useSqlExecutionMultiDbFocus.spec.ts | 2 +- .../src/i18n/__tests__/backendErrors.spec.ts | 8 +- apps/desktop/src/i18n/backend-errors.ts | 2 +- apps/desktop/src/lib/README.md | 2 +- .../databaseObjectCapabilities.spec.ts | 2 +- .../elasticsearch/elasticsearchRequestRisk.ts | 2 +- .../src/lib/redis/redisDatabaseIndex.ts | 2 +- apps/desktop/src/lib/solr/solrRequestRisk.ts | 2 +- apps/desktop/src/lib/sql/newQueryContext.ts | 2 +- crates/dbx-core/src/data/database_export.rs | 2 +- crates/dbx-core/src/data/transfer.rs | 2 +- crates/dbx-core/src/schema/mod.rs | 2 +- crates/dbx-driver-agent/Cargo.toml | 32 ++ .../assets/agent-protocol-v2.json | 0 .../src/agent_catalog.rs | 0 .../src/agent_connection.rs | 0 .../src}/agent_driver.rs | 2 +- .../src/agent_manager.rs | 0 .../src/agent_offline_export.rs | 0 .../src/agent_recovery.rs | 0 .../src/agent_runtime.rs | 0 .../src/agent_service.rs | 0 .../src/backend_error.rs | 0 .../src/database_capabilities.rs | 0 crates/dbx-driver-agent/src/lib.rs | 21 + crates/dbx-driver-elasticsearch/Cargo.toml | 19 + .../src/lib.rs} | 9 +- crates/dbx-driver-mongodb/Cargo.toml | 23 ++ crates/dbx-driver-mongodb/src/lib.rs | 9 + .../src}/mongo_driver.rs | 0 .../src/mongo_oidc.rs | 0 .../src/mongo_shell.rs | 0 crates/dbx-driver-mysql/Cargo.toml | 29 ++ .../src/db => dbx-driver-mysql/src}/dolt.rs | 0 .../src/db => dbx-driver-mysql/src}/doris.rs | 0 crates/dbx-driver-mysql/src/lib.rs | 17 + .../src}/manticoresearch.rs | 0 .../src/db => dbx-driver-mysql/src}/mysql.rs | 0 .../src}/mysql_compatible.rs | 0 .../db => dbx-driver-mysql/src}/ob_oracle.rs | 0 .../src}/oceanbase_mysql.rs | 0 .../db => dbx-driver-mysql/src}/starrocks.rs | 0 .../src/db => dbx-driver-mysql/src}/tidb.rs | 0 crates/dbx-driver-postgres/Cargo.toml | 34 ++ crates/dbx-driver-postgres/src/lib.rs | 10 + .../src}/postgres.rs | 0 crates/dbx-driver-redis/Cargo.toml | 19 + .../src/lib.rs} | 99 +++-- crates/dbx-driver-sqlserver/Cargo.toml | 24 ++ crates/dbx-driver-sqlserver/src/lib.rs | 11 + .../src}/sqlserver.rs | 0 crates/dbx-driver-support/Cargo.toml | 27 ++ crates/dbx-driver-support/src/db.rs | 318 ++++++++++++++++ .../db => dbx-driver-support/src}/ddl_scan.rs | 0 .../src}/document_result.rs | 0 .../src}/elasticsearch_sql.rs | 0 .../src/execution.rs | 0 .../src}/file_validator.rs | 0 .../src}/http_tunnel.rs | 0 crates/dbx-driver-support/src/lib.rs | 16 + .../src/runtime_config.rs | 0 .../src/ssh_config.rs | 0 .../src}/ssh_host_key.rs | 0 .../src/db => dbx-driver-support/src}/wkb.rs | 0 crates/dbx-drivers/Cargo.toml | 19 +- crates/dbx-drivers/src/db/mod.rs | 359 +----------------- crates/dbx-drivers/src/lib.rs | 26 +- .../tests/live_sqlserver_issue_9984_real.rs | 2 +- crates/dbx-sql-core/Cargo.toml | 18 + .../src/dml_preview_sql.rs | 0 crates/dbx-sql-core/src/lib.rs | 18 + .../src/mysql_ddl_normalize.rs | 0 .../src/mysql_event_sql.rs | 0 .../src/object_source_sql.rs | 0 .../src/query_execution_sql.rs | 0 crates/{dbx-sql => dbx-sql-core}/src/sql.rs | 0 .../src/sql_analysis.rs | 0 .../src/sql_diagnostics.rs | 0 .../src/sql_editability.rs | 0 .../src/sql_error_position.rs | 0 .../{dbx-sql => dbx-sql-core}/src/sql_risk.rs | 0 .../src/sqlserver_temporal.rs | 0 .../src/tdsql_mysql.rs | 0 .../src/value_literals.rs | 0 crates/dbx-sql-data/Cargo.toml | 21 + .../src/data_grid_extractors/contract.rs | 0 .../src/data_grid_extractors/delimited.rs | 0 .../src/data_grid_extractors/document.rs | 0 .../src/data_grid_extractors/json.rs | 0 .../src/data_grid_extractors/mod.rs | 0 .../src/data_grid_extractors/raw.rs | 0 .../src/data_grid_extractors/sql.rs | 0 .../src/data_grid_extractors/tests.rs | 0 .../src/data_grid_iotdb_sql.rs | 0 .../src/data_grid_neo4j_sql.rs | 0 .../src/data_grid_salesforce_sql.rs | 0 .../src/data_grid_sql.rs | 0 .../src/data_grid_tdengine_sql.rs | 0 .../src/database_search_sql.rs | 0 crates/dbx-sql-data/src/lib.rs | 10 + .../src/query_result_sql.rs | 0 crates/dbx-sql-dialect/Cargo.toml | 20 + crates/{dbx-sql => dbx-sql-dialect}/build.rs | 0 .../src/dml_binding.rs | 0 crates/dbx-sql-dialect/src/lib.rs | 6 + .../src/sql_dialect.rs | 0 .../src/sql_dialect/capabilities.rs | 0 .../src/sql_dialect/ddl_profile.rs | 0 .../src/sql_dialect/descriptor.rs | 0 .../src/sql_dialect/descriptor_snapshots.rs | 0 .../src/sql_dialect/dialect_loader.rs | 0 .../src/sql_dialect/dialect_types.rs | 0 .../src/sql_dialect/dialect_yaml.rs | 0 .../src/sql_dialect/hot_reload.rs | 0 .../src/sql_dialect/identifiers.rs | 0 .../src/sql_dialect/inference.rs | 0 ...criptor_snapshots__all_dialects_info.snap} | 2 +- ...criptor_snapshots__duckdb_descriptor.snap} | 2 +- ...scriptor_snapshots__mysql_descriptor.snap} | 2 +- ...ct__descriptor_snapshots__mysql_info.snap} | 2 +- ...snapshots__mysql_to_postgres_mapping.snap} | 2 +- ...criptor_snapshots__oracle_descriptor.snap} | 2 +- ...iptor_snapshots__postgres_descriptor.snap} | 2 +- ..._descriptor_snapshots__postgres_info.snap} | 2 +- ...snapshots__postgres_to_mysql_mapping.snap} | 2 +- ...criptor_snapshots__sqlite_descriptor.snap} | 2 +- ...ptor_snapshots__sqlserver_descriptor.snap} | 2 +- .../src/sql_dialect/table_select.rs | 0 .../src/sql_dialect/tests.rs | 0 .../src/sql_dialect/type_rewrite.rs | 0 .../src/sql_dialect/types.rs | 0 crates/dbx-sql-schema/Cargo.toml | 25 ++ .../src/db_admin_sql.rs | 0 crates/dbx-sql-schema/src/lib.rs | 10 + .../src/schema_diff.rs | 0 .../src/schema_diff/sqlserver_dependencies.rs | 0 .../src/sql_parser/ast_filter.rs | 0 .../src/sql_parser/git.rs | 0 .../src/sql_parser/input.rs | 0 .../src/sql_parser/meta.rs | 0 .../src/sql_parser/mod.rs | 0 .../src/table_structure_sql.rs | 0 .../src/table_structure_sql/column_alter.rs | 0 .../src/table_structure_sql/column_format.rs | 0 .../src/table_structure_sql/columns.rs | 0 .../src/table_structure_sql/comments.rs | 0 .../src/table_structure_sql/create_table.rs | 0 .../src/table_structure_sql/dialect.rs | 0 .../src/table_structure_sql/foreign_keys.rs | 0 .../src/table_structure_sql/indexes.rs | 0 .../src/table_structure_sql/mysql_engine.rs | 0 .../src/table_structure_sql/owner.rs | 0 .../src/table_structure_sql/partitions.rs | 0 .../src/table_structure_sql/tests.rs | 0 .../src/table_structure_sql/triggers.rs | 0 .../src/table_structure_sql/types.rs | 0 .../src/table_structure_sql/util.rs | 0 .../src/table_structure_sql/validation.rs | 0 crates/dbx-sql/Cargo.toml | 30 +- crates/dbx-sql/src/lib.rs | 32 +- deploy/Dockerfile | 19 +- docs/salesforce-soql-integration-spec.md | 4 +- .../sqlserverResultFocus6189.test.ts | 2 +- scripts/core-architecture.test.mjs | 28 +- scripts/release.mjs | 2 +- 180 files changed, 1422 insertions(+), 538 deletions(-) create mode 100644 crates/dbx-driver-agent/Cargo.toml rename crates/{dbx-drivers => dbx-driver-agent}/assets/agent-protocol-v2.json (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_catalog.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_connection.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-agent/src}/agent_driver.rs (99%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_manager.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_offline_export.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_recovery.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_runtime.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/agent_service.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/backend_error.rs (100%) rename crates/{dbx-drivers => dbx-driver-agent}/src/database_capabilities.rs (100%) create mode 100644 crates/dbx-driver-agent/src/lib.rs create mode 100644 crates/dbx-driver-elasticsearch/Cargo.toml rename crates/{dbx-drivers/src/db/elasticsearch_driver.rs => dbx-driver-elasticsearch/src/lib.rs} (99%) create mode 100644 crates/dbx-driver-mongodb/Cargo.toml create mode 100644 crates/dbx-driver-mongodb/src/lib.rs rename crates/{dbx-drivers/src/db => dbx-driver-mongodb/src}/mongo_driver.rs (100%) rename crates/{dbx-drivers => dbx-driver-mongodb}/src/mongo_oidc.rs (100%) rename crates/{dbx-drivers => dbx-driver-mongodb}/src/mongo_shell.rs (100%) create mode 100644 crates/dbx-driver-mysql/Cargo.toml rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/dolt.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/doris.rs (100%) create mode 100644 crates/dbx-driver-mysql/src/lib.rs rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/manticoresearch.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/mysql.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/mysql_compatible.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/ob_oracle.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/oceanbase_mysql.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/starrocks.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-mysql/src}/tidb.rs (100%) create mode 100644 crates/dbx-driver-postgres/Cargo.toml create mode 100644 crates/dbx-driver-postgres/src/lib.rs rename crates/{dbx-drivers/src/db => dbx-driver-postgres/src}/postgres.rs (100%) create mode 100644 crates/dbx-driver-redis/Cargo.toml rename crates/{dbx-drivers/src/db/redis_driver.rs => dbx-driver-redis/src/lib.rs} (98%) create mode 100644 crates/dbx-driver-sqlserver/Cargo.toml create mode 100644 crates/dbx-driver-sqlserver/src/lib.rs rename crates/{dbx-drivers/src/db => dbx-driver-sqlserver/src}/sqlserver.rs (100%) create mode 100644 crates/dbx-driver-support/Cargo.toml create mode 100644 crates/dbx-driver-support/src/db.rs rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/ddl_scan.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/document_result.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/elasticsearch_sql.rs (100%) rename crates/{dbx-drivers => dbx-driver-support}/src/execution.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/file_validator.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/http_tunnel.rs (100%) create mode 100644 crates/dbx-driver-support/src/lib.rs rename crates/{dbx-drivers => dbx-driver-support}/src/runtime_config.rs (100%) rename crates/{dbx-drivers => dbx-driver-support}/src/ssh_config.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/ssh_host_key.rs (100%) rename crates/{dbx-drivers/src/db => dbx-driver-support/src}/wkb.rs (100%) create mode 100644 crates/dbx-sql-core/Cargo.toml rename crates/{dbx-sql => dbx-sql-core}/src/dml_preview_sql.rs (100%) create mode 100644 crates/dbx-sql-core/src/lib.rs rename crates/{dbx-sql => dbx-sql-core}/src/mysql_ddl_normalize.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/mysql_event_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/object_source_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/query_execution_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sql.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sql_analysis.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sql_diagnostics.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sql_editability.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sql_error_position.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sql_risk.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/sqlserver_temporal.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/tdsql_mysql.rs (100%) rename crates/{dbx-sql => dbx-sql-core}/src/value_literals.rs (100%) create mode 100644 crates/dbx-sql-data/Cargo.toml rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/contract.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/delimited.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/document.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/json.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/mod.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/raw.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/sql.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_extractors/tests.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_iotdb_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_neo4j_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_salesforce_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/data_grid_tdengine_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-data}/src/database_search_sql.rs (100%) create mode 100644 crates/dbx-sql-data/src/lib.rs rename crates/{dbx-sql => dbx-sql-data}/src/query_result_sql.rs (100%) create mode 100644 crates/dbx-sql-dialect/Cargo.toml rename crates/{dbx-sql => dbx-sql-dialect}/build.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/dml_binding.rs (100%) create mode 100644 crates/dbx-sql-dialect/src/lib.rs rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/capabilities.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/ddl_profile.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/descriptor.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/descriptor_snapshots.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/dialect_loader.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/dialect_types.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/dialect_yaml.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/hot_reload.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/identifiers.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/inference.rs (100%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__all_dialects_info.snap} (99%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap} (91%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_descriptor.snap} (91%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_info.snap} (94%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap} (96%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__oracle_descriptor.snap} (91%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_descriptor.snap} (91%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_info.snap} (94%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap} (95%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap} (91%) rename crates/{dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap => dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap} (91%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/table_select.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/tests.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/type_rewrite.rs (100%) rename crates/{dbx-sql => dbx-sql-dialect}/src/sql_dialect/types.rs (100%) create mode 100644 crates/dbx-sql-schema/Cargo.toml rename crates/{dbx-sql => dbx-sql-schema}/src/db_admin_sql.rs (100%) create mode 100644 crates/dbx-sql-schema/src/lib.rs rename crates/{dbx-sql => dbx-sql-schema}/src/schema_diff.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/schema_diff/sqlserver_dependencies.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/sql_parser/ast_filter.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/sql_parser/git.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/sql_parser/input.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/sql_parser/meta.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/sql_parser/mod.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/column_alter.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/column_format.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/columns.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/comments.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/create_table.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/dialect.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/foreign_keys.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/indexes.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/mysql_engine.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/owner.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/partitions.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/tests.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/triggers.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/types.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/util.rs (100%) rename crates/{dbx-sql => dbx-sql-schema}/src/table_structure_sql/validation.rs (100%) diff --git a/.github/scripts/ci-config.mjs b/.github/scripts/ci-config.mjs index 0283dbb63b..91a256e41b 100644 --- a/.github/scripts/ci-config.mjs +++ b/.github/scripts/ci-config.mjs @@ -1,6 +1,13 @@ export const rustGroups = { - foundation: ["dbx-types", "dbx-platform", "dbx-sql", "dbx-formats", "dbx-ai-provider", "dbx-plugin-runtime"], - drivers: ["dbx-drivers", "dbx-sqlite-worker"], + foundation: [ + "dbx-types", "dbx-platform", "dbx-sql-core", "dbx-sql-dialect", "dbx-sql-data", "dbx-sql-schema", "dbx-sql", + "dbx-formats", "dbx-ai-provider", "dbx-plugin-runtime", + ], + drivers: [ + "dbx-driver-support", "dbx-driver-agent", "dbx-driver-elasticsearch", "dbx-driver-mongodb", + "dbx-driver-mysql", "dbx-driver-postgres", "dbx-driver-redis", "dbx-driver-sqlserver", "dbx-drivers", + "dbx-sqlite-worker", + ], application: ["dbx", "dbx-core", "dbx-web", "dbx-cli", "dbx-mcp"], }; diff --git a/.github/scripts/ci-plan.mjs b/.github/scripts/ci-plan.mjs index 14f5b3fa4b..b35f56b45d 100644 --- a/.github/scripts/ci-plan.mjs +++ b/.github/scripts/ci-plan.mjs @@ -42,7 +42,7 @@ export function planCi({ files, metadata, root, eventName = "pull_request", rust if (owner) affected.add(owner[0]); else if (/^(?:crates|src-tauri)\//.test(file)) unknownRust = true; if (file.startsWith("plugins/connection-types/")) affected.add("dbx-types"); - if (file.startsWith("plugins/dialects/")) affected.add("dbx-sql"); + if (file.startsWith("plugins/dialects/")) affected.add("dbx-sql-dialect"); } const knownGroups = new Set(Object.values(rustGroups).flat()); const unknownMember = packages.some((pkg) => !knownGroups.has(pkg.name)); @@ -72,7 +72,7 @@ export function planCi({ files, metadata, root, eventName = "pull_request", rust && !nativeDrivers.some((driver) => file.startsWith(`agents/drivers/${driver}/`)) && !jdbcDrivers.some((driver) => file.startsWith(`agents/drivers/${driver}/`))) || files.some((file) => file.startsWith(".github/scripts/bump-agent-versions.") || file === ".github/workflows/agents-release.yml" - || file === "crates/dbx-drivers/assets/agent-protocol-v2.json"); + || file === "crates/dbx-driver-agent/assets/agent-protocol-v2.json"); const allAgents = sharedAgents || (agentsChanged && nativeChanges.size === 0 && !javaDriverChanges); if (allAgents) { for (const driver of nativeDrivers) nativeChanges.add(driver); diff --git a/.github/scripts/ci-plan.test.mjs b/.github/scripts/ci-plan.test.mjs index 2feb2b15ee..be478f6b6b 100644 --- a/.github/scripts/ci-plan.test.mjs +++ b/.github/scripts/ci-plan.test.mjs @@ -12,9 +12,22 @@ import { assertCoverage, parseCoverage } from "./ci-rust-coverage.mjs"; const root = "/fixture"; const graph = { - "dbx-types": [], "dbx-platform": [], "dbx-formats": [], "dbx-sql": ["dbx-types"], + "dbx-types": [], "dbx-platform": [], "dbx-formats": [], "dbx-sql-core": ["dbx-types"], + "dbx-sql-dialect": ["dbx-types"], "dbx-sql-data": ["dbx-sql-core", "dbx-sql-dialect", "dbx-types"], + "dbx-sql-schema": ["dbx-sql-core", "dbx-sql-dialect", "dbx-types"], + "dbx-sql": ["dbx-sql-core", "dbx-sql-data", "dbx-sql-dialect", "dbx-sql-schema", "dbx-types"], "dbx-ai-provider": ["dbx-platform"], "dbx-plugin-runtime": ["dbx-platform", "dbx-types"], - "dbx-sqlite-worker": [], "dbx-drivers": ["dbx-platform", "dbx-sql", "dbx-types", "dbx-sqlite-worker"], + "dbx-driver-support": ["dbx-platform", "dbx-types"], + "dbx-driver-agent": ["dbx-platform", "dbx-sql-core", "dbx-types"], + "dbx-driver-elasticsearch": ["dbx-driver-support", "dbx-types"], + "dbx-driver-mongodb": ["dbx-driver-support", "dbx-types"], + "dbx-driver-mysql": ["dbx-driver-support", "dbx-sql-core", "dbx-sql-dialect", "dbx-types"], + "dbx-driver-postgres": ["dbx-driver-support", "dbx-sql-core", "dbx-types"], + "dbx-driver-redis": ["dbx-driver-support", "dbx-types"], + "dbx-driver-sqlserver": ["dbx-driver-support", "dbx-sql-core", "dbx-sql-data", "dbx-types"], + "dbx-sqlite-worker": [], "dbx-drivers": ["dbx-driver-agent", "dbx-driver-elasticsearch", "dbx-driver-mongodb", + "dbx-driver-mysql", "dbx-driver-postgres", "dbx-driver-redis", "dbx-driver-sqlserver", "dbx-driver-support", + "dbx-platform", "dbx-sql-core", "dbx-sql-data", "dbx-sql-dialect", "dbx-types", "dbx-sqlite-worker"], "dbx-core": ["dbx-drivers", "dbx-sql", "dbx-types", "dbx-platform", "dbx-ai-provider", "dbx-plugin-runtime", "dbx-formats"], "dbx-mcp": ["dbx-core"], "dbx-cli": ["dbx-core", "dbx-mcp"], "dbx-web": ["dbx-core", "dbx-mcp"], "dbx": ["dbx-core", "dbx-mcp"], }; @@ -51,7 +64,7 @@ test("core and consumer changes do not pull unrelated foundation test groups", ( const cli = plan(["crates/dbx-cli/src/main.rs"]); assert.deepEqual(groups(cli), ["application"]); assert.equal(cli.agents, false); - assert.deepEqual(groups(plan(["crates/dbx-drivers/src/db/postgres.rs"])), ["drivers", "application"]); + assert.deepEqual(groups(plan(["crates/dbx-driver-postgres/src/postgres.rs"])), ["drivers", "application"]); }); test("code generation and test-only dependencies participate in impact analysis", () => { @@ -123,7 +136,7 @@ test("known JDBC driver changes only select Java agent tests", () => { test("shared Agent inputs and unknown native modules never silently lose coverage", () => { for (const file of ["agents/common/src/main/java/Protocol.java", "agents/scripts/validate_agents.py", "agents/build.gradle", - "agents/drivers/new-driver/main.go", "crates/dbx-drivers/assets/agent-protocol-v2.json", ".github/workflows/agents-release.yml"]) { + "agents/drivers/new-driver/main.go", "crates/dbx-driver-agent/assets/agent-protocol-v2.json", ".github/workflows/agents-release.yml"]) { const result = plan([file]); assert.equal(result.agent_go.include.length, 10, file); assert.equal(result.agent_integration.include.length, 16, file); diff --git a/.github/scripts/ci-rust.mjs b/.github/scripts/ci-rust.mjs index abecfdd123..0649237568 100644 --- a/.github/scripts/ci-rust.mjs +++ b/.github/scripts/ci-rust.mjs @@ -12,9 +12,12 @@ export function rustCommand(action, group, mode) { if (mode === "full") capabilityFeatures.push("system-fonts"); const appFeatures = ["dbx", "dbx-core", "dbx-web"].flatMap((name) => capabilityFeatures.map((feature) => `${name}/${feature}`)); const foundationFeatures = ["dbx-types/mq-admin", "dbx-types/openapi", "dbx-sql/duckdb-sidecar", "dbx-sql/openapi", + "dbx-sql-data/openapi", "dbx-sql-schema/duckdb-sidecar", "dbx-platform/downloads", "dbx-platform/host-prompts", "dbx-platform/test-support", "dbx-plugin-runtime/default", "dbx-plugin-runtime/test-support"]; const features = group === "foundation" ? foundationFeatures : group === "drivers" ? [ ...["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-bundled", "sqlite-sqlcipher", "test-support"].map((feature) => `dbx-drivers/${feature}`), + "dbx-driver-agent/test-support", "dbx-driver-mysql/test-support", "dbx-driver-redis/test-support", + "dbx-driver-support/test-support", "dbx-sqlite-worker/runtime", "dbx-types/openapi", "dbx-sql/openapi", ] : appFeatures; const packages = group === "workspace" ? ["--workspace"] : rustGroups[group].flatMap((name) => ["--package", name]); diff --git a/.github/scripts/label-pull-request.mjs b/.github/scripts/label-pull-request.mjs index b0e4da48f3..e5bf852279 100644 --- a/.github/scripts/label-pull-request.mjs +++ b/.github/scripts/label-pull-request.mjs @@ -30,7 +30,7 @@ export const STATIC_LABEL_SPECS = { "tests-only": { color: "bfd4f2", description: "Changes only tests, fixtures, or snapshots" }, }; -const CORE_CRATE_PATH = /^crates\/dbx-(?:core|drivers|sql|formats|types|platform|ai-provider|plugin-runtime)\//; +const CORE_CRATE_PATH = /^crates\/dbx-(?:core|driver(?:s|-[^/]+)|sql(?:-[^/]+)?|formats|types|platform|ai-provider|plugin-runtime)\//; const TYPE_LABELS = new Set(["bug", "enhancement", "documentation", "maintenance"]); const LABEL_PALETTE = [ diff --git a/.github/scripts/label-pull-request.test.mjs b/.github/scripts/label-pull-request.test.mjs index ed340f1fd3..aee05725e4 100644 --- a/.github/scripts/label-pull-request.test.mjs +++ b/.github/scripts/label-pull-request.test.mjs @@ -108,8 +108,8 @@ test("recognizes extracted core crates and their database sources", () => { const result = evaluatePullRequestLabels({ title: "fix(sql): preserve PostgreSQL query contracts", changedFiles: [ - "crates/dbx-drivers/src/db/postgres.rs", - "crates/dbx-sql/src/sql.rs", + "crates/dbx-driver-postgres/src/postgres.rs", + "crates/dbx-sql-core/src/sql.rs", "crates/dbx-types/src/types.rs", "crates/dbx-platform/src/process.rs", "crates/dbx-ai-provider/src/ai.rs", @@ -129,7 +129,7 @@ test("collapses broad area and database changes", () => { changedFiles: [ "agents/drivers/mysql/build.gradle", "apps/desktop/src/components/grid/GeometryViewer.vue", - "crates/dbx-drivers/src/db/postgres.rs", + "crates/dbx-driver-postgres/src/postgres.rs", "crates/dbx-mcp/src/main.rs", "crates/dbx-web/src/main.rs", "docs/content/docs/geometry.mdx", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b156a660e0..67cad05e6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -960,7 +960,8 @@ jobs: - 'scripts/run-check.mjs' - 'crates/dbx-core/src/**' - 'crates/dbx-drivers/src/**' - - 'crates/dbx-sql/src/**' + - 'crates/dbx-driver-*/src/**' + - 'crates/dbx-sql*/src/**' - 'crates/dbx-formats/src/**' - 'crates/dbx-types/src/**' - 'crates/dbx-plugin-runtime/src/**' @@ -1002,7 +1003,7 @@ jobs: - '.github/workflows/ci.yml' agents: - 'agents/**' - - 'crates/dbx-drivers/assets/agent-protocol-v2.json' + - 'crates/dbx-driver-agent/assets/agent-protocol-v2.json' - 'crates/dbx-core/Cargo.toml' - 'Cargo.toml' - 'Cargo.lock' @@ -1045,6 +1046,7 @@ jobs: - 'src-tauri/src/commands/update.rs' - 'crates/dbx-core/Cargo.toml' - 'crates/dbx-drivers/**' + - 'crates/dbx-driver-*/**' - 'crates/dbx-platform/**' - 'crates/dbx-core/src/host/update.rs' - 'Cargo.toml' diff --git a/Cargo.lock b/Cargo.lock index ec3fd687a8..4a239f8cc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2212,6 +2212,172 @@ dependencies = [ "zstd", ] +[[package]] +name = "dbx-driver-agent" +version = "0.1.0" +dependencies = [ + "chrono", + "crc", + "dbx-platform", + "dbx-sql-core", + "dbx-types", + "flate2", + "futures", + "log", + "percent-encoding", + "reqwest 0.12.28", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "tokio", + "tokio-util", + "uuid", + "zip", + "zstd", +] + +[[package]] +name = "dbx-driver-elasticsearch" +version = "0.1.0" +dependencies = [ + "dbx-driver-support", + "dbx-types", + "percent-encoding", + "regex", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "dbx-driver-mongodb" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "chrono", + "dbx-driver-support", + "dbx-types", + "futures", + "json5", + "log", + "mongodb", + "percent-encoding", + "reqwest 0.12.28", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "uuid", +] + +[[package]] +name = "dbx-driver-mysql" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-driver-support", + "dbx-sql-core", + "dbx-sql-dialect", + "dbx-types", + "futures", + "log", + "mysql_async", + "mysql_common", + "percent-encoding", + "rust_decimal", + "serde_json", + "sqlparser", + "tokio", + "tokio-util", +] + +[[package]] +name = "dbx-driver-postgres" +version = "0.1.0" +dependencies = [ + "bytes", + "chrono", + "dbx-driver-support", + "dbx-sql-core", + "dbx-types", + "deadpool-postgres", + "futures", + "log", + "openssl", + "percent-encoding", + "portpicker", + "postgres-openssl", + "rustls 0.23.43", + "rustls-pemfile 2.2.0", + "serde_json", + "sqlparser", + "tempfile", + "tokio", + "tokio-openssl", + "tokio-postgres", + "tokio-postgres-rustls", + "tokio-util", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "dbx-driver-redis" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "dbx-driver-support", + "dbx-types", + "redis", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "dbx-driver-sqlserver" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-driver-support", + "dbx-sql-core", + "dbx-sql-data", + "dbx-types", + "futures", + "log", + "serde", + "serde_json", + "sqlparser", + "tiberius", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "dbx-driver-support" +version = "0.1.0" +dependencies = [ + "bytes", + "dbx-platform", + "dbx-types", + "portpicker", + "reqwest 0.12.28", + "russh", + "serde", + "serde_json", + "sqlparser", + "tokio", + "tokio-util", + "uuid", +] + [[package]] name = "dbx-drivers" version = "0.1.0" @@ -2222,8 +2388,18 @@ dependencies = [ "chrono", "crc", "csv", + "dbx-driver-agent", + "dbx-driver-elasticsearch", + "dbx-driver-mongodb", + "dbx-driver-mysql", + "dbx-driver-postgres", + "dbx-driver-redis", + "dbx-driver-sqlserver", + "dbx-driver-support", "dbx-platform", - "dbx-sql", + "dbx-sql-core", + "dbx-sql-data", + "dbx-sql-dialect", "dbx-sqlite-worker", "dbx-types", "deadpool-postgres", @@ -2232,7 +2408,6 @@ dependencies = [ "json5", "libsqlite3-hotbundle", "log", - "mongodb", "mysql_async", "mysql_common", "nix 0.31.3", @@ -2241,7 +2416,6 @@ dependencies = [ "percent-encoding", "portpicker", "postgres-openssl", - "redis", "regex", "reqwest 0.12.28", "rusqlite", @@ -2350,14 +2524,69 @@ dependencies = [ [[package]] name = "dbx-sql" version = "0.1.0" +dependencies = [ + "dbx-sql-core", + "dbx-sql-data", + "dbx-sql-dialect", + "dbx-sql-schema", + "dbx-types", +] + +[[package]] +name = "dbx-sql-core" +version = "0.1.0" dependencies = [ "chrono", "dbx-types", "encoding_rs", + "log", + "regex", + "serde", + "serde_json", + "sqlparser", + "unicode-ident", + "winnow 1.0.4", +] + +[[package]] +name = "dbx-sql-data" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-sql-core", + "dbx-sql-dialect", + "dbx-types", + "serde", + "serde_json", + "sqlparser", + "unicode-width", + "utoipa", +] + +[[package]] +name = "dbx-sql-dialect" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-types", "insta", "log", "notify", "percent-encoding", + "serde", + "serde_yaml", + "tempfile", + "tokio", +] + +[[package]] +name = "dbx-sql-schema" +version = "0.1.0" +dependencies = [ + "dbx-sql-core", + "dbx-sql-dialect", + "dbx-types", + "log", "rayon", "regex", "rusqlite", @@ -2365,12 +2594,6 @@ dependencies = [ "serde_json", "serde_yaml", "sqlparser", - "tempfile", - "tokio", - "unicode-ident", - "unicode-width", - "utoipa", - "winnow 1.0.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9a740f8bb9..fa0a54ef80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["src-tauri", "crates/dbx-core", "crates/dbx-drivers", "crates/dbx-sql", "crates/dbx-plugin-runtime", "crates/dbx-formats", "crates/dbx-types", "crates/dbx-platform", "crates/dbx-ai-provider", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"] +members = ["src-tauri", "crates/dbx-core", "crates/dbx-driver-agent", "crates/dbx-driver-support", "crates/dbx-driver-elasticsearch", "crates/dbx-driver-mongodb", "crates/dbx-driver-mysql", "crates/dbx-driver-postgres", "crates/dbx-driver-redis", "crates/dbx-driver-sqlserver", "crates/dbx-drivers", "crates/dbx-sql", "crates/dbx-sql-core", "crates/dbx-sql-data", "crates/dbx-sql-dialect", "crates/dbx-sql-schema", "crates/dbx-plugin-runtime", "crates/dbx-formats", "crates/dbx-types", "crates/dbx-platform", "crates/dbx-ai-provider", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"] exclude = ["vendor/tauri-plugin-updater", "vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/tiberius", "vendor/webview2-com-sys", "vendor/wry"] [patch.crates-io] diff --git a/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java b/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java index 8a9c658eed..d1e5837263 100644 --- a/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java +++ b/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java @@ -7,7 +7,7 @@ import java.util.List; /** * Decode PostGIS EWKB / WKB byte streams (and PGgeometry / hex strings) into * OGC Well-Known Text. Behaves identically to the Rust implementation in - * {@code crates/dbx-drivers/src/db/postgres.rs::ewkb_to_wkt}, including the + * {@code crates/dbx-driver-support/src/wkb.rs::wkb_to_wkt}, including the * {@code Z} / {@code M} / {@code ZM} dimension suffixes and {@code EMPTY} * geometries. Used by the JDBC agent layer so that PostgresLike databases * such as HighGo / KingBase / Vastbase / openGauss / GaussDB return WKT for diff --git a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java index a29fb9af44..ba8922e862 100644 --- a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java @@ -79,7 +79,7 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { /** * Wrap {@link AbstractJdbcAgent#resultValue} so PostGIS-style {@code geometry} * and {@code geography} columns are decoded into WKT (matching the native - * tokio_postgres path in {@code crates/dbx-drivers/src/db/postgres.rs}). + * tokio_postgres path in {@code crates/dbx-driver-postgres/src/postgres.rs}). */ private JdbcExecutor.ColumnAwareResultValueReader geometryAwareResolver() { return (rs, index, sqlType, columnTypeName) -> { diff --git a/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java b/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java index 7fdf74c3fd..b16f0703ae 100644 --- a/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java +++ b/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java @@ -22,7 +22,7 @@ class EwkbWktDecoderTest { // ────────────────────────────────────────────────────────────────────── // Cross-checked against the Rust ewkb_to_wkt tests in - // crates/dbx-drivers/src/db/postgres.rs (same hex inputs, same expected WKT). + // crates/dbx-driver-support/src/wkb.rs (same hex inputs, same expected WKT). // ────────────────────────────────────────────────────────────────────── @Test diff --git a/agents/drivers/duckdb/Cargo.lock b/agents/drivers/duckdb/Cargo.lock index c03b0b2b7b..7d4df119aa 100644 --- a/agents/drivers/duckdb/Cargo.lock +++ b/agents/drivers/duckdb/Cargo.lock @@ -1316,6 +1316,167 @@ dependencies = [ "zip 4.6.1", ] +[[package]] +name = "dbx-driver-agent" +version = "0.1.0" +dependencies = [ + "chrono", + "crc", + "dbx-platform", + "dbx-sql-core", + "dbx-types", + "flate2", + "futures", + "log", + "percent-encoding", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "tokio", + "tokio-util", + "uuid", + "zip 4.6.1", + "zstd", +] + +[[package]] +name = "dbx-driver-elasticsearch" +version = "0.1.0" +dependencies = [ + "dbx-driver-support", + "dbx-types", + "percent-encoding", + "regex", + "reqwest", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "dbx-driver-mongodb" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "chrono", + "dbx-driver-support", + "dbx-types", + "futures", + "json5", + "log", + "mongodb", + "percent-encoding", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "uuid", +] + +[[package]] +name = "dbx-driver-mysql" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-driver-support", + "dbx-sql-core", + "dbx-sql-dialect", + "dbx-types", + "futures", + "log", + "mysql_async", + "percent-encoding", + "rust_decimal", + "serde_json", + "sqlparser", + "tokio", + "tokio-util", +] + +[[package]] +name = "dbx-driver-postgres" +version = "0.1.0" +dependencies = [ + "bytes", + "chrono", + "dbx-driver-support", + "dbx-sql-core", + "dbx-types", + "deadpool-postgres", + "futures", + "log", + "openssl", + "percent-encoding", + "postgres-openssl", + "rustls 0.23.42", + "rustls-pemfile 2.2.0", + "serde_json", + "sqlparser", + "tokio", + "tokio-postgres", + "tokio-postgres-rustls", + "tokio-util", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "dbx-driver-redis" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "dbx-driver-support", + "dbx-types", + "redis", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "dbx-driver-sqlserver" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-driver-support", + "dbx-sql-core", + "dbx-sql-data", + "dbx-types", + "futures", + "log", + "serde", + "serde_json", + "sqlparser", + "tiberius", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "dbx-driver-support" +version = "0.1.0" +dependencies = [ + "bytes", + "dbx-platform", + "dbx-types", + "portpicker", + "reqwest", + "russh", + "serde", + "serde_json", + "sqlparser", + "tokio", + "tokio-util", + "uuid", +] + [[package]] name = "dbx-drivers" version = "0.1.0" @@ -1325,8 +1486,18 @@ dependencies = [ "chrono", "crc", "csv", + "dbx-driver-agent", + "dbx-driver-elasticsearch", + "dbx-driver-mongodb", + "dbx-driver-mysql", + "dbx-driver-postgres", + "dbx-driver-redis", + "dbx-driver-sqlserver", + "dbx-driver-support", "dbx-platform", - "dbx-sql", + "dbx-sql-core", + "dbx-sql-data", + "dbx-sql-dialect", "dbx-sqlite-worker", "dbx-types", "deadpool-postgres", @@ -1334,14 +1505,12 @@ dependencies = [ "futures", "json5", "log", - "mongodb", "mysql_async", "openssl", "pageant", "percent-encoding", "portpicker", "postgres-openssl", - "redis", "regex", "reqwest", "rusqlite", @@ -1434,23 +1603,72 @@ dependencies = [ [[package]] name = "dbx-sql" version = "0.1.0" +dependencies = [ + "dbx-sql-core", + "dbx-sql-data", + "dbx-sql-dialect", + "dbx-sql-schema", + "dbx-types", +] + +[[package]] +name = "dbx-sql-core" +version = "0.1.0" dependencies = [ "chrono", "dbx-types", "encoding_rs", "log", + "regex", + "serde", + "serde_json", + "sqlparser", + "unicode-ident", + "winnow", +] + +[[package]] +name = "dbx-sql-data" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-sql-core", + "dbx-sql-dialect", + "dbx-types", + "serde", + "serde_json", + "sqlparser", + "unicode-width", +] + +[[package]] +name = "dbx-sql-dialect" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-types", + "log", "notify", "percent-encoding", + "serde", + "serde_yaml", + "tokio", +] + +[[package]] +name = "dbx-sql-schema" +version = "0.1.0" +dependencies = [ + "dbx-sql-core", + "dbx-sql-dialect", + "dbx-types", + "log", "rayon", "regex", "serde", "serde_json", "serde_yaml", "sqlparser", - "tokio", - "unicode-ident", - "unicode-width", - "winnow", ] [[package]] diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts b/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts index 928402bee2..96b7740d22 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts @@ -1003,7 +1003,7 @@ describe("RedisKeyBrowser continuation ownership (issue #7779)", () => { // `iterations` (aka `max_iterations`) is the 6th positional arg the frontend // sends to `redisScanKeysBatch` — the same unit the backend spends as real -// Redis SCAN calls (see `crates/dbx-drivers/src/db/redis_driver.rs`). +// Redis SCAN calls (see `crates/dbx-driver-redis/src/lib.rs`). function totalIterationsRequested(): number { return mocks.redisScanKeysBatch.mock.calls.reduce((sum: number, call: unknown[]) => sum + (call[5] as number), 0); } diff --git a/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts b/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts index 23f999fe4d..be6c5153eb 100644 --- a/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts +++ b/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts @@ -220,7 +220,7 @@ describe("useSidebarTableMutationRuntime SAP HANA schema-scoped actions", () => it("classifies a JDBC-agent-routed timeout (SAP HANA/Oracle/DB2/SQL Server) as timed-out, not failed, even though its message has no timeout wording", async () => { const { feature } = runtime(""); - // AgentCallError::Timeout carries no `detail` (crates/dbx-drivers/src/backend_error.rs), + // AgentCallError::Timeout carries no `detail` (crates/dbx-driver-agent/src/backend_error.rs), // so BackendErrorException.message degrades to a generic fallback with no // "timed out" text — only the structured backendError distinguishes it. const agentTimeoutError = { diff --git a/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts b/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts index 71966c6e06..fcaaf11874 100644 --- a/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts +++ b/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts @@ -2,7 +2,7 @@ * Issue #6189 — SQL Server "focus jumps to the message result". * * Background: - * - `crates/dbx-drivers/src/db/sqlserver.rs:523-542` synthesizes a pseudo result with a + * - `crates/dbx-driver-sqlserver/src/sqlserver.rs` synthesizes a pseudo result with a * single "Message" column and `server_message: true` for any batch segment that * produced only server messages (PRINT / "DBCC execution completed" / ...). * - `stores/queryStore.ts:4727` then picks the first result that HAS COLUMNS as the diff --git a/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts b/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts index c8850529a0..1fced962e1 100644 --- a/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts +++ b/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts @@ -48,7 +48,7 @@ const STRUCTURED_BACKEND_ERROR_KEYS = [ "backendErrors.unknown", ] as const; -// Reproduces the exact string crates/dbx-drivers/src/agent_service.rs builds on +// Reproduces the exact string crates/dbx-driver-agent/src/agent_service.rs builds on // Windows: `\` line continuations strip the newline plus the following indent. const WINDOWS_JRE_REMOVE_ERROR = [ "Failed to remove the old JRE directory: C:\\dbx\\jre21", @@ -644,9 +644,9 @@ describe("backend error wording is pinned to the Rust sources", () => { ["crates/dbx-core/src/data/mongodb_import_export.rs", "MongoDB Legacy Agent does not support insertMany; upgrade or reinstall the MongoDB Legacy driver"], ["crates/dbx-core/src/data/mongodb_import_export.rs", "MongoDB Legacy Agent returned an invalid find cursor"], ["crates/dbx-core/src/query/mongo_ops.rs", "MongoDB Legacy Agent rejected "], - ["crates/dbx-drivers/src/agent_service.rs", "Failed to remove the old JRE directory: "], - ["crates/dbx-drivers/src/agent_service.rs", "is in use by drivers: "], - ["crates/dbx-drivers/src/agent_service.rs", "agent-registry.json not found in the ZIP; not a valid offline driver package."], + ["crates/dbx-driver-agent/src/agent_service.rs", "Failed to remove the old JRE directory: "], + ["crates/dbx-driver-agent/src/agent_service.rs", "is in use by drivers: "], + ["crates/dbx-driver-agent/src/agent_service.rs", "agent-registry.json not found in the ZIP; not a valid offline driver package."], ["crates/dbx-core/src/admin/mq/adapters/kafka.rs", "Kafka does not support unloading topics"], ["crates/dbx-plugin-runtime/src/plugins/installer.rs", "Plugin update source change requires confirmation:"], ["crates/dbx-plugin-runtime/src/plugins/installer.rs", "Plugin downgrade to version "], diff --git a/apps/desktop/src/i18n/backend-errors.ts b/apps/desktop/src/i18n/backend-errors.ts index 12186996d4..8ba485d27f 100644 --- a/apps/desktop/src/i18n/backend-errors.ts +++ b/apps/desktop/src/i18n/backend-errors.ts @@ -149,7 +149,7 @@ const patterns: [RegExp, string][] = [ // Legacy bundled DuckDB error kept for compatibility with older backends. [/^The previous DuckDB query is still stopping\. Please try again shortly\.$/, "editor.duckdbDraining"], - // Driver / JRE management (crates/dbx-drivers/src/agent_service.rs, routes/agents.rs) + // Driver / JRE management (crates/dbx-driver-agent/src/agent_service.rs, routes/agents.rs) // The Windows variant is multi-line, so it must be tried before the single-line one. [/^Failed to remove the old JRE directory: (.+)\nPossible causes:[\s\S]*\(original error: ([\s\S]+)\)$/, "driverStore.jreDirRemoveFailedWindows"], [/^Failed to remove the old JRE directory: (.+) \(original error: ([\s\S]+)\)$/, "driverStore.jreDirRemoveFailed"], diff --git a/apps/desktop/src/lib/README.md b/apps/desktop/src/lib/README.md index 7342e68b6b..e4a70e8caf 100644 --- a/apps/desktop/src/lib/README.md +++ b/apps/desktop/src/lib/README.md @@ -7,7 +7,7 @@ - `app`, `tabs`, `sidebar`, `connection`: shell, navigation, and connection UI state helpers. - `database`, `metadata`, `schema`, `table`: relational database metadata, capabilities, DDL, and table-object helpers. - `sql`, `sql/semantic`, `editor`, `query`, `history`, `savedSql`: SQL editing, execution, diagnostics, history, and saved SQL behavior. - Keep fixed SQL completion statements and their database-specific variants in `sql/sqlSnippetTemplates.ts`; cross-check row-limiting families against `crates/dbx-sql/src/sql_dialect/capabilities.rs` and authoritative dialect documentation, and never replace a user-customized built-in body with a generated default. + Keep fixed SQL completion statements and their database-specific variants in `sql/sqlSnippetTemplates.ts`; cross-check row-limiting families against `crates/dbx-sql-dialect/src/sql_dialect/capabilities.rs` and authoritative dialect documentation, and never replace a user-customized built-in body with a generated default. - `dataGrid`: result/grid rendering, editing, previews, pagination, and export helpers tied to the grid. - `ai`, `mcp`: AI assistant and MCP configuration helpers. - `redis`, `mongo`, `elasticsearch`, `etcd`, `kv`, `mq`, `nacos`, `zookeeper`, `webdav`: non-relational or service-specific helpers. diff --git a/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts b/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts index 2746342df0..d2d2187ca4 100644 --- a/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts +++ b/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts @@ -69,7 +69,7 @@ describe("databaseObjectCapabilities", () => { it("exposes materialized views for StarRocks only", () => { // StarRocks has a dedicated MV listing/classification path in - // crates/dbx-drivers/src/db/mysql.rs (`list_starrocks_tables` + + // crates/dbx-driver-mysql/src/mysql.rs (`list_starrocks_tables` + // `classify_starrocks_materialized_views`). expect(sidebarObjectKindsForDatabase("starrocks")).toContain("MATERIALIZED_VIEW"); diff --git a/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts b/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts index 566567b4c7..dd5b59bc64 100644 --- a/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts +++ b/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts @@ -2,7 +2,7 @@ * Read/write classification for Elasticsearch-compatible REST requests. * * Mirrors `classify_search_engine_query_risk` in - * `crates/dbx-sql/src/query_execution_sql.rs` so the desktop guards (read-only + * `crates/dbx-sql-core/src/query_execution_sql.rs` so the desktop guards (read-only * unlock, production safety) agree with the backend read-only gate instead of * treating every `GET`/`POST` request as an unrecognized — and therefore * unsafe — statement. diff --git a/apps/desktop/src/lib/redis/redisDatabaseIndex.ts b/apps/desktop/src/lib/redis/redisDatabaseIndex.ts index 01870136cd..8a1c1b6d6f 100644 --- a/apps/desktop/src/lib/redis/redisDatabaseIndex.ts +++ b/apps/desktop/src/lib/redis/redisDatabaseIndex.ts @@ -1,4 +1,4 @@ -// Mirrors dbx-core `redis_database_index()` (crates/dbx-drivers/src/db/redis_driver.rs): +// Mirrors dbx-core `redis_database_index()` (crates/dbx-driver-redis/src/lib.rs): // a Redis database is a numeric index. Anything else (e.g. redis-cli flags like // "0 --tls --insecure" pasted into the field or imported from a dirty URL) fails // to parse on the backend and silently falls back to db 0, so every frontend diff --git a/apps/desktop/src/lib/solr/solrRequestRisk.ts b/apps/desktop/src/lib/solr/solrRequestRisk.ts index 9f57da19f9..8bd203b106 100644 --- a/apps/desktop/src/lib/solr/solrRequestRisk.ts +++ b/apps/desktop/src/lib/solr/solrRequestRisk.ts @@ -1,7 +1,7 @@ /** * Read/write classification for Apache Solr REST requests. * - * Mirrors `classify_solr_query_risk` in `crates/dbx-sql/src/query_execution_sql.rs` + * Mirrors `classify_solr_query_risk` in `crates/dbx-sql-core/src/query_execution_sql.rs` * so the desktop guards (read-only unlock, production safety) agree with the * backend read-only gate instead of treating every request as unrecognized. * diff --git a/apps/desktop/src/lib/sql/newQueryContext.ts b/apps/desktop/src/lib/sql/newQueryContext.ts index 0121b02228..e3c5aa237e 100644 --- a/apps/desktop/src/lib/sql/newQueryContext.ts +++ b/apps/desktop/src/lib/sql/newQueryContext.ts @@ -151,7 +151,7 @@ export function resolveNewQueryTable(input: ResolveNewQueryTableInput): NewQuery * * Time-series engines get the same rolling-window scoping the sidebar * quick-open uses (see `default_time_series_predicate` in - * `crates/dbx-sql/src/sql_dialect/table_select.rs`). Without a `time` + * `crates/dbx-sql-dialect/src/sql_dialect/table_select.rs`). Without a `time` * predicate InfluxDB scans every shard (v1/v2) or every Parquet file * (v3), which turns "let me draft a query against this table" into a * full-history scan the moment the user hits Run. VictoriaMetrics diff --git a/crates/dbx-core/src/data/database_export.rs b/crates/dbx-core/src/data/database_export.rs index be98afb649..fd6ababe73 100644 --- a/crates/dbx-core/src/data/database_export.rs +++ b/crates/dbx-core/src/data/database_export.rs @@ -212,7 +212,7 @@ fn exports_schema_wide_objects(request: &DatabaseExportRequest) -> bool { } /// MySQL lists triggers and events as schema-wide objects (see -/// `crates/dbx-drivers/src/db/mysql.rs`), which is what the export writes out. Other +/// `crates/dbx-driver-mysql/src/mysql.rs`), which is what the export writes out. Other /// engines either report triggers per table (PostgreSQL) or not at all, so their export /// stays unchanged. fn exports_mysql_trigger_objects(db_type: DatabaseType) -> bool { diff --git a/crates/dbx-core/src/data/transfer.rs b/crates/dbx-core/src/data/transfer.rs index 0310f0e5d2..0dd2eff69b 100644 --- a/crates/dbx-core/src/data/transfer.rs +++ b/crates/dbx-core/src/data/transfer.rs @@ -2249,7 +2249,7 @@ fn postgres_index_column_sql( ) -> String { // The base key text: a real column is quoted as an identifier; an expression/functional // key part arrives as raw expression text (the per-column `pg_get_indexdef` omits the - // opclass — see `crates/dbx-drivers/src/db/postgres.rs`), so quoting the whole thing as + // opclass — see `crates/dbx-driver-postgres/src/postgres.rs`), so quoting the whole thing as // an identifier would turn it into a nonexistent column reference (#6295). let base = if is_expression { column.to_string() } else { quote_identifier(column, &DatabaseType::Postgres) }; // The opclass is read separately from `pg_index.indclass` for every key position diff --git a/crates/dbx-core/src/schema/mod.rs b/crates/dbx-core/src/schema/mod.rs index 6812543ccc..8d13044840 100644 --- a/crates/dbx-core/src/schema/mod.rs +++ b/crates/dbx-core/src/schema/mod.rs @@ -14524,7 +14524,7 @@ fn render_postgres_table_ddl_with_constraints_and_partition_info( .map(|(i, c)| { // A real column is quoted via `pg_ident`; an expression/functional key part // arrives as raw expression text (the per-column `pg_get_indexdef` omits the - // opclass — see `crates/dbx-drivers/src/db/postgres.rs`), so quoting the whole + // opclass — see `crates/dbx-driver-postgres/src/postgres.rs`), so quoting the whole // thing as an identifier would turn it into a nonexistent column reference // (#6295). let is_expr = idx.key_is_expression.get(i).copied().unwrap_or(false); diff --git a/crates/dbx-driver-agent/Cargo.toml b/crates/dbx-driver-agent/Cargo.toml new file mode 100644 index 0000000000..88e413b9d4 --- /dev/null +++ b/crates/dbx-driver-agent/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "dbx-driver-agent" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +test-support = [] + +[dependencies] +dbx-platform = { path = "../dbx-platform", features = ["host-prompts", "downloads"] } +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-types = { path = "../dbx-types", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +crc = "3" +flate2 = "1" +futures = "0.3" +log = "0.4" +percent-encoding = "2" +reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "stream", "rustls-tls", "socks"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sha2 = "0.10" +tar = "0.4" +tempfile = "3" +tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7", features = ["compat"] } +uuid = { version = "1", features = ["v4", "serde"] } +zip = { version = "4", default-features = false, features = ["deflate"] } +zstd = "0.13" diff --git a/crates/dbx-drivers/assets/agent-protocol-v2.json b/crates/dbx-driver-agent/assets/agent-protocol-v2.json similarity index 100% rename from crates/dbx-drivers/assets/agent-protocol-v2.json rename to crates/dbx-driver-agent/assets/agent-protocol-v2.json diff --git a/crates/dbx-drivers/src/agent_catalog.rs b/crates/dbx-driver-agent/src/agent_catalog.rs similarity index 100% rename from crates/dbx-drivers/src/agent_catalog.rs rename to crates/dbx-driver-agent/src/agent_catalog.rs diff --git a/crates/dbx-drivers/src/agent_connection.rs b/crates/dbx-driver-agent/src/agent_connection.rs similarity index 100% rename from crates/dbx-drivers/src/agent_connection.rs rename to crates/dbx-driver-agent/src/agent_connection.rs diff --git a/crates/dbx-drivers/src/db/agent_driver.rs b/crates/dbx-driver-agent/src/agent_driver.rs similarity index 99% rename from crates/dbx-drivers/src/db/agent_driver.rs rename to crates/dbx-driver-agent/src/agent_driver.rs index 6eda8ea046..57fee4cf7f 100644 --- a/crates/dbx-drivers/src/db/agent_driver.rs +++ b/crates/dbx-driver-agent/src/agent_driver.rs @@ -5448,7 +5448,7 @@ for line in sys.stdin: #[test] fn agent_protocol_matches_contract_file() { let contract: serde_json::Value = - serde_json::from_str(include_str!("../../assets/agent-protocol-v2.json")).unwrap(); + serde_json::from_str(include_str!("../assets/agent-protocol-v2.json")).unwrap(); assert_eq!(contract["protocolVersion"], AGENT_PROTOCOL_VERSION); assert_eq!(contract["handshakeMethod"], AgentMethod::Handshake.as_str()); diff --git a/crates/dbx-drivers/src/agent_manager.rs b/crates/dbx-driver-agent/src/agent_manager.rs similarity index 100% rename from crates/dbx-drivers/src/agent_manager.rs rename to crates/dbx-driver-agent/src/agent_manager.rs diff --git a/crates/dbx-drivers/src/agent_offline_export.rs b/crates/dbx-driver-agent/src/agent_offline_export.rs similarity index 100% rename from crates/dbx-drivers/src/agent_offline_export.rs rename to crates/dbx-driver-agent/src/agent_offline_export.rs diff --git a/crates/dbx-drivers/src/agent_recovery.rs b/crates/dbx-driver-agent/src/agent_recovery.rs similarity index 100% rename from crates/dbx-drivers/src/agent_recovery.rs rename to crates/dbx-driver-agent/src/agent_recovery.rs diff --git a/crates/dbx-drivers/src/agent_runtime.rs b/crates/dbx-driver-agent/src/agent_runtime.rs similarity index 100% rename from crates/dbx-drivers/src/agent_runtime.rs rename to crates/dbx-driver-agent/src/agent_runtime.rs diff --git a/crates/dbx-drivers/src/agent_service.rs b/crates/dbx-driver-agent/src/agent_service.rs similarity index 100% rename from crates/dbx-drivers/src/agent_service.rs rename to crates/dbx-driver-agent/src/agent_service.rs diff --git a/crates/dbx-drivers/src/backend_error.rs b/crates/dbx-driver-agent/src/backend_error.rs similarity index 100% rename from crates/dbx-drivers/src/backend_error.rs rename to crates/dbx-driver-agent/src/backend_error.rs diff --git a/crates/dbx-drivers/src/database_capabilities.rs b/crates/dbx-driver-agent/src/database_capabilities.rs similarity index 100% rename from crates/dbx-drivers/src/database_capabilities.rs rename to crates/dbx-driver-agent/src/database_capabilities.rs diff --git a/crates/dbx-driver-agent/src/lib.rs b/crates/dbx-driver-agent/src/lib.rs new file mode 100644 index 0000000000..0c233c855b --- /dev/null +++ b/crates/dbx-driver-agent/src/lib.rs @@ -0,0 +1,21 @@ +#![recursion_limit = "256"] + +pub use dbx_platform::download::DownloadSource; +pub use dbx_platform::{path_utils, process}; +pub use dbx_sql_core::sql_error_position; +pub use dbx_types::{database_manifest, models, types}; + +pub mod agent_catalog; +pub mod agent_connection; +pub mod agent_driver; +pub mod agent_manager; +pub mod agent_offline_export; +pub mod agent_recovery; +pub mod agent_runtime; +pub mod agent_service; +pub mod backend_error; +pub mod database_capabilities; + +pub mod db { + pub use crate::agent_driver; +} diff --git a/crates/dbx-driver-elasticsearch/Cargo.toml b/crates/dbx-driver-elasticsearch/Cargo.toml new file mode 100644 index 0000000000..251e853611 --- /dev/null +++ b/crates/dbx-driver-elasticsearch/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "dbx-driver-elasticsearch" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-types = { path = "../dbx-types", default-features = false } +percent-encoding = "2" +regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +tokio = { version = "1", features = ["full"] } + +[dev-dependencies] +uuid = { version = "1", features = ["v4"] } diff --git a/crates/dbx-drivers/src/db/elasticsearch_driver.rs b/crates/dbx-driver-elasticsearch/src/lib.rs similarity index 99% rename from crates/dbx-drivers/src/db/elasticsearch_driver.rs rename to crates/dbx-driver-elasticsearch/src/lib.rs index daa2dd2397..160e20c0b2 100644 --- a/crates/dbx-drivers/src/db/elasticsearch_driver.rs +++ b/crates/dbx-driver-elasticsearch/src/lib.rs @@ -10,9 +10,16 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use super::{apply_tls_certificates, http_client_builder, with_connection_timeout}; use crate::db::document_result::DocumentQueryResult; use crate::types::QueryResult; +use dbx_driver_support::db::{apply_tls_certificates, http_client_builder, with_connection_timeout}; + +pub mod db { + pub use dbx_driver_support::db::*; + pub use dbx_driver_support::{document_result, elasticsearch_sql}; +} + +pub use dbx_types::types; const ELASTICSEARCH_PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS .add(b' ') diff --git a/crates/dbx-driver-mongodb/Cargo.toml b/crates/dbx-driver-mongodb/Cargo.toml new file mode 100644 index 0000000000..0a79bdecea --- /dev/null +++ b/crates/dbx-driver-mongodb/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "dbx-driver-mongodb" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-types = { path = "../dbx-types", default-features = false } +base64 = "0.22" +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +json5 = "0.4" +log = "0.4" +mongodb = { version = "3.2.5", features = ["socks5-proxy"] } +percent-encoding = "2" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sha2 = "0.10" +tokio = { version = "1", features = ["full"] } +uuid = { version = "1", features = ["v4"] } diff --git a/crates/dbx-driver-mongodb/src/lib.rs b/crates/dbx-driver-mongodb/src/lib.rs new file mode 100644 index 0000000000..c808b148a0 --- /dev/null +++ b/crates/dbx-driver-mongodb/src/lib.rs @@ -0,0 +1,9 @@ +#![recursion_limit = "256"] + +pub use dbx_driver_support::db::{safe_i64_to_json, with_connection_timeout, JS_MAX_SAFE_INTEGER}; +pub use dbx_driver_support::document_result; +pub use dbx_types::{models, types}; + +pub mod mongo_driver; +pub mod mongo_oidc; +pub mod mongo_shell; diff --git a/crates/dbx-drivers/src/db/mongo_driver.rs b/crates/dbx-driver-mongodb/src/mongo_driver.rs similarity index 100% rename from crates/dbx-drivers/src/db/mongo_driver.rs rename to crates/dbx-driver-mongodb/src/mongo_driver.rs diff --git a/crates/dbx-drivers/src/mongo_oidc.rs b/crates/dbx-driver-mongodb/src/mongo_oidc.rs similarity index 100% rename from crates/dbx-drivers/src/mongo_oidc.rs rename to crates/dbx-driver-mongodb/src/mongo_oidc.rs diff --git a/crates/dbx-drivers/src/mongo_shell.rs b/crates/dbx-driver-mongodb/src/mongo_shell.rs similarity index 100% rename from crates/dbx-drivers/src/mongo_shell.rs rename to crates/dbx-driver-mongodb/src/mongo_shell.rs diff --git a/crates/dbx-driver-mysql/Cargo.toml b/crates/dbx-driver-mysql/Cargo.toml new file mode 100644 index 0000000000..f76110a894 --- /dev/null +++ b/crates/dbx-driver-mysql/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "dbx-driver-mysql" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +test-support = [] + +[dependencies] +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-sql-dialect = { path = "../dbx-sql-dialect" } +dbx-types = { path = "../dbx-types", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +log = "0.4" +mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "client_ed25519", "chrono", "rust_decimal"] } +percent-encoding = "2" +rust_decimal = { version = "1", features = ["serde"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7", features = ["compat"] } + +[dev-dependencies] +mysql_common = { git = "https://github.com/t8y2/rust_mysql_common.git", rev = "47740d85a8277167a5717afac785d37b46a36296", default-features = false } diff --git a/crates/dbx-drivers/src/db/dolt.rs b/crates/dbx-driver-mysql/src/dolt.rs similarity index 100% rename from crates/dbx-drivers/src/db/dolt.rs rename to crates/dbx-driver-mysql/src/dolt.rs diff --git a/crates/dbx-drivers/src/db/doris.rs b/crates/dbx-driver-mysql/src/doris.rs similarity index 100% rename from crates/dbx-drivers/src/db/doris.rs rename to crates/dbx-driver-mysql/src/doris.rs diff --git a/crates/dbx-driver-mysql/src/lib.rs b/crates/dbx-driver-mysql/src/lib.rs new file mode 100644 index 0000000000..09d7f19648 --- /dev/null +++ b/crates/dbx-driver-mysql/src/lib.rs @@ -0,0 +1,17 @@ +#![recursion_limit = "256"] + +pub use dbx_driver_support::db::*; +pub use dbx_driver_support::{db, execution, file_validator, wkb}; +pub use dbx_sql_core::{mysql_event_sql, sql}; +pub use dbx_sql_dialect::sql_dialect; +pub use dbx_types::{models, types}; + +pub mod dolt; +pub mod doris; +pub mod manticoresearch; +pub mod mysql; +pub mod mysql_compatible; +pub mod ob_oracle; +pub mod oceanbase_mysql; +pub mod starrocks; +pub mod tidb; diff --git a/crates/dbx-drivers/src/db/manticoresearch.rs b/crates/dbx-driver-mysql/src/manticoresearch.rs similarity index 100% rename from crates/dbx-drivers/src/db/manticoresearch.rs rename to crates/dbx-driver-mysql/src/manticoresearch.rs diff --git a/crates/dbx-drivers/src/db/mysql.rs b/crates/dbx-driver-mysql/src/mysql.rs similarity index 100% rename from crates/dbx-drivers/src/db/mysql.rs rename to crates/dbx-driver-mysql/src/mysql.rs diff --git a/crates/dbx-drivers/src/db/mysql_compatible.rs b/crates/dbx-driver-mysql/src/mysql_compatible.rs similarity index 100% rename from crates/dbx-drivers/src/db/mysql_compatible.rs rename to crates/dbx-driver-mysql/src/mysql_compatible.rs diff --git a/crates/dbx-drivers/src/db/ob_oracle.rs b/crates/dbx-driver-mysql/src/ob_oracle.rs similarity index 100% rename from crates/dbx-drivers/src/db/ob_oracle.rs rename to crates/dbx-driver-mysql/src/ob_oracle.rs diff --git a/crates/dbx-drivers/src/db/oceanbase_mysql.rs b/crates/dbx-driver-mysql/src/oceanbase_mysql.rs similarity index 100% rename from crates/dbx-drivers/src/db/oceanbase_mysql.rs rename to crates/dbx-driver-mysql/src/oceanbase_mysql.rs diff --git a/crates/dbx-drivers/src/db/starrocks.rs b/crates/dbx-driver-mysql/src/starrocks.rs similarity index 100% rename from crates/dbx-drivers/src/db/starrocks.rs rename to crates/dbx-driver-mysql/src/starrocks.rs diff --git a/crates/dbx-drivers/src/db/tidb.rs b/crates/dbx-driver-mysql/src/tidb.rs similarity index 100% rename from crates/dbx-drivers/src/db/tidb.rs rename to crates/dbx-driver-mysql/src/tidb.rs diff --git a/crates/dbx-driver-postgres/Cargo.toml b/crates/dbx-driver-postgres/Cargo.toml new file mode 100644 index 0000000000..5bf63c16b9 --- /dev/null +++ b/crates/dbx-driver-postgres/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "dbx-driver-postgres" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-types = { path = "../dbx-types", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +bytes = "1" +deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } +futures = "0.3" +log = "0.4" +openssl = { version = "0.10", features = ["vendored"] } +percent-encoding = "2" +postgres-openssl = "0.5" +rustls = { version = "0.23", features = ["aws-lc-rs"] } +rustls-pemfile = "2.2" +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +tokio = { version = "1", features = ["full"] } +tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-uuid-1", "with-serde_json-1"] } +tokio-postgres-rustls = "0.13" +tokio-util = { version = "0.7", features = ["compat"] } +uuid = { version = "1", features = ["v4", "serde"] } +webpki-roots = "0.26" + +[dev-dependencies] +portpicker = "0.1.1" +tempfile = "3" +tokio-openssl = "0.6" diff --git a/crates/dbx-driver-postgres/src/lib.rs b/crates/dbx-driver-postgres/src/lib.rs new file mode 100644 index 0000000000..c8082e6e10 --- /dev/null +++ b/crates/dbx-driver-postgres/src/lib.rs @@ -0,0 +1,10 @@ +#![recursion_limit = "256"] + +pub use dbx_driver_support::db::*; +pub use dbx_driver_support::{db, execution, file_validator, wkb}; +pub use dbx_sql_core::{sql, sql_error_position}; +pub use dbx_types::{models, types}; + +mod postgres; + +pub use postgres::*; diff --git a/crates/dbx-drivers/src/db/postgres.rs b/crates/dbx-driver-postgres/src/postgres.rs similarity index 100% rename from crates/dbx-drivers/src/db/postgres.rs rename to crates/dbx-driver-postgres/src/postgres.rs diff --git a/crates/dbx-driver-redis/Cargo.toml b/crates/dbx-driver-redis/Cargo.toml new file mode 100644 index 0000000000..6be021a92d --- /dev/null +++ b/crates/dbx-driver-redis/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "dbx-driver-redis" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +test-support = [] + +[dependencies] +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-types = { path = "../dbx-types", default-features = false } +base64 = "0.22" +redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tls-rustls-insecure", "tokio-rustls-comp", "sentinel", "cluster-async"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +tokio = { version = "1", features = ["full"] } diff --git a/crates/dbx-drivers/src/db/redis_driver.rs b/crates/dbx-driver-redis/src/lib.rs similarity index 98% rename from crates/dbx-drivers/src/db/redis_driver.rs rename to crates/dbx-driver-redis/src/lib.rs index 1eea06696b..46ab0384fd 100644 --- a/crates/dbx-drivers/src/db/redis_driver.rs +++ b/crates/dbx-driver-redis/src/lib.rs @@ -1,5 +1,6 @@ -use crate::models::connection::{ConnectionConfig, DatabaseConnectionInfo}; use base64::Engine; +use dbx_driver_support::db::{connection_timeout, json_value_for_js, safe_i64_to_json, CONNECTION_TIMEOUT_SECS}; +use dbx_types::models::connection::{ConnectionConfig, DatabaseConnectionInfo}; use redis::{ aio::ConnectionLike, cluster::ClusterClient, @@ -12,8 +13,6 @@ use serde::{Deserialize, Serialize}; use std::{collections::HashMap, future::Future, sync::OnceLock, time::Duration, time::Instant}; use tokio::sync::{Mutex, MutexGuard}; -use super::json_value_for_js; - const STREAM_ENTRY_PAGE_SIZE: usize = 50; const STREAM_PENDING_PAGE_SIZE: usize = 100; const COLLECTION_PAGE_SIZE: usize = 200; @@ -711,11 +710,11 @@ pub async fn connect_sentinel(config: &ConnectionConfig) -> Result(&mut con), ) .await - .map_err(|_| format!("Redis Sentinel master lookup timed out ({}s)", super::CONNECTION_TIMEOUT_SECS))? + .map_err(|_| format!("Redis Sentinel master lookup timed out ({}s)", CONNECTION_TIMEOUT_SECS))? .map_err(|e| format!("Redis Sentinel master lookup failed: {e}"))?; redis_sentinel_master_endpoint(raw) @@ -772,9 +771,9 @@ pub async fn connect_cluster(config: &ConnectionConfig) -> Result con, @@ -789,16 +788,18 @@ pub async fn connect_cluster(config: &ConnectionConfig) -> Result return Err(err), }; - let cluster_ping = - match tokio::time::timeout(super::connection_timeout(), redis::cmd("PING").query_async::(&mut con)) - .await - { - Ok(Ok(_)) => Ok(()), - // Reachable and authenticated, only the PING privilege is missing (issue #9394). - Ok(Err(error)) if is_redis_command_denied_error(&error.to_string()) => Ok(()), - Ok(Err(error)) => Err(format!("Redis cluster ping failed: {error}")), - Err(_) => Err(format!("Redis cluster ping timed out ({}s)", super::CONNECTION_TIMEOUT_SECS)), - }; + let cluster_ping = match tokio::time::timeout( + connection_timeout(), + redis::cmd("PING").query_async::(&mut con), + ) + .await + { + Ok(Ok(_)) => Ok(()), + // Reachable and authenticated, only the PING privilege is missing (issue #9394). + Ok(Err(error)) if is_redis_command_denied_error(&error.to_string()) => Ok(()), + Ok(Err(error)) => Err(format!("Redis cluster ping failed: {error}")), + Err(_) => Err(format!("Redis cluster ping timed out ({}s)", CONNECTION_TIMEOUT_SECS)), + }; match cluster_ping { Ok(_) => { let seed_routes = identity_routes(&seed_nodes); @@ -897,7 +898,7 @@ pub async fn connect_routed_cluster( } { let mut con = cluster_any_connection(&pool).await?; - redis_ping(&mut con, "Redis cluster", super::connection_timeout()).await?; + redis_ping(&mut con, "Redis cluster", connection_timeout()).await?; } Ok(pool) } @@ -906,11 +907,11 @@ pub async fn test_connection(connection: &RedisConnection) -> Result<(), String> match connection { RedisConnection::Direct(con) => { let mut con = con.lock().await; - redis_ping(&mut *con, "Redis", super::connection_timeout()).await + redis_ping(&mut *con, "Redis", connection_timeout()).await } RedisConnection::Cluster(cluster) => { let mut con = cluster_any_connection(cluster).await?; - redis_ping(&mut con, "Redis cluster", super::connection_timeout()).await + redis_ping(&mut con, "Redis cluster", connection_timeout()).await } } } @@ -934,22 +935,20 @@ async fn redis_database_connection_info(con: &mut C) -> Result(con), - ) - .await - { - Ok(Ok(info)) => info, - // Managed Redis accounts and proxies frequently allow connecting but not `INFO`. The - // connection is still usable, so report the identity we do know instead of failing all - // server metadata (issue #9394). - Ok(Err(error)) if is_redis_command_denied_error(&error.to_string()) => String::new(), - Ok(Err(error)) => return Err(format!("Redis INFO command failed: {error}")), - Err(_) => { - return Err(format!("Redis INFO command timed out ({}s)", super::CONNECTION_TIMEOUT_SECS)); - } - }; + let info = + match tokio::time::timeout(connection_timeout(), redis::cmd("INFO").arg("server").query_async::(con)) + .await + { + Ok(Ok(info)) => info, + // Managed Redis accounts and proxies frequently allow connecting but not `INFO`. The + // connection is still usable, so report the identity we do know instead of failing all + // server metadata (issue #9394). + Ok(Err(error)) if is_redis_command_denied_error(&error.to_string()) => String::new(), + Ok(Err(error)) => return Err(format!("Redis INFO command failed: {error}")), + Err(_) => { + return Err(format!("Redis INFO command timed out ({}s)", CONNECTION_TIMEOUT_SECS)); + } + }; let product_version = info.lines().find_map(|line| { let (key, value) = line.trim().split_once(':')?; (key.eq_ignore_ascii_case("redis_version") && !value.trim().is_empty()).then(|| value.trim().to_string()) @@ -1169,12 +1168,12 @@ fn parse_redis_port(port: &str) -> Result { } async fn connect_client(client: redis::Client) -> Result { - let mut con = tokio::time::timeout(super::connection_timeout(), client.get_multiplexed_async_connection()) + let mut con = tokio::time::timeout(connection_timeout(), client.get_multiplexed_async_connection()) .await - .map_err(|_| format!("Redis connection timed out ({}s)", super::CONNECTION_TIMEOUT_SECS))? + .map_err(|_| format!("Redis connection timed out ({}s)", CONNECTION_TIMEOUT_SECS))? .map_err(|e| format!("Redis connection failed: {e}"))?; - redis_ping(&mut con, "Redis", super::connection_timeout()).await?; + redis_ping(&mut con, "Redis", connection_timeout()).await?; Ok(con) } @@ -2139,7 +2138,7 @@ pub fn redis_command_raw_to_json(value: RedisRawValue) -> serde_json::Value { } RedisRawValue::SimpleString(value) => serde_json::Value::String(value), RedisRawValue::Okay => serde_json::Value::String("OK".to_string()), - RedisRawValue::Int(value) => super::safe_i64_to_json(value), + RedisRawValue::Int(value) => safe_i64_to_json(value), RedisRawValue::Double(value) => { serde_json::Number::from_f64(value).map_or(serde_json::Value::Null, serde_json::Value::Number) } @@ -4509,7 +4508,7 @@ mod tests { RedisRawValue, RedisSetItem, RedisStreamConsumer, RedisStreamEntry, RedisStreamField, RedisStreamGroup, RedisStreamPendingEntry, RedisValue, RedisValueData, RedisZsetItem, }; - use crate::models::connection::ConnectionConfig; + use dbx_types::models::connection::ConnectionConfig; use redis::{aio::ConnectionLike, Cmd, ConnectionAddr, Pipeline, RedisFuture}; struct FakeRedisConnection { @@ -7372,7 +7371,7 @@ mod tests { id: "redis".to_string(), name: "Redis".to_string(), note: String::new(), - db_type: crate::models::connection::DatabaseType::Redis, + db_type: dbx_types::models::connection::DatabaseType::Redis, driver_profile: None, driver_label: None, url_params: None, @@ -7391,10 +7390,10 @@ mod tests { init_script: None, color: None, transport_layers: Vec::new(), - connect_timeout_secs: crate::models::connection::default_connect_timeout_secs(), - query_timeout_secs: crate::models::connection::default_query_timeout_secs(), - idle_timeout_secs: crate::models::connection::default_idle_timeout_secs(), - keepalive_interval_secs: crate::models::connection::default_keepalive_interval_secs(), + connect_timeout_secs: dbx_types::models::connection::default_connect_timeout_secs(), + query_timeout_secs: dbx_types::models::connection::default_query_timeout_secs(), + idle_timeout_secs: dbx_types::models::connection::default_idle_timeout_secs(), + keepalive_interval_secs: dbx_types::models::connection::default_keepalive_interval_secs(), ssl: false, ca_cert_path: String::new(), client_cert_path: String::new(), @@ -7409,7 +7408,7 @@ mod tests { redis_sentinel_password: String::new(), redis_sentinel_tls: false, redis_cluster_nodes: String::new(), - redis_key_separator: crate::models::connection::default_redis_key_separator(), + redis_key_separator: dbx_types::models::connection::default_redis_key_separator(), redis_scan_page_size: None, redis_database_aliases: Default::default(), redis_key_templates: Vec::new(), diff --git a/crates/dbx-driver-sqlserver/Cargo.toml b/crates/dbx-driver-sqlserver/Cargo.toml new file mode 100644 index 0000000000..a1d6542c6f --- /dev/null +++ b/crates/dbx-driver-sqlserver/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "dbx-driver-sqlserver" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-sql-data = { path = "../dbx-sql-data" } +dbx-types = { path = "../dbx-types", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +log = "0.4" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] } +tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7", features = ["compat"] } +tracing = "0.1" +tracing-subscriber = "0.3" +uuid = { version = "1", features = ["v4", "serde"] } diff --git a/crates/dbx-driver-sqlserver/src/lib.rs b/crates/dbx-driver-sqlserver/src/lib.rs new file mode 100644 index 0000000000..2319d02311 --- /dev/null +++ b/crates/dbx-driver-sqlserver/src/lib.rs @@ -0,0 +1,11 @@ +#![recursion_limit = "256"] + +pub use dbx_driver_support::db::*; +pub use dbx_driver_support::{db, execution}; +pub use dbx_sql_core::{sql, sqlserver_temporal}; +pub use dbx_sql_data::query_result_sql; +pub use dbx_types::types; + +mod sqlserver; + +pub use sqlserver::*; diff --git a/crates/dbx-drivers/src/db/sqlserver.rs b/crates/dbx-driver-sqlserver/src/sqlserver.rs similarity index 100% rename from crates/dbx-drivers/src/db/sqlserver.rs rename to crates/dbx-driver-sqlserver/src/sqlserver.rs diff --git a/crates/dbx-driver-support/Cargo.toml b/crates/dbx-driver-support/Cargo.toml new file mode 100644 index 0000000000..b3a9bc4c6b --- /dev/null +++ b/crates/dbx-driver-support/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "dbx-driver-support" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +test-support = ["dbx-platform/test-support"] + +[dependencies] +dbx-platform = { path = "../dbx-platform" } +dbx-types = { path = "../dbx-types", default-features = false } +bytes = "1" +portpicker = "0.1.1" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +russh = "0.60" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +tokio = { version = "1", features = ["io-util", "net", "rt", "sync", "time"] } +tokio-util = { version = "0.7", features = ["rt"] } +uuid = { version = "1", features = ["v4"] } + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt-multi-thread", "test-util"] } diff --git a/crates/dbx-driver-support/src/db.rs b/crates/dbx-driver-support/src/db.rs new file mode 100644 index 0000000000..0b65c5b548 --- /dev/null +++ b/crates/dbx-driver-support/src/db.rs @@ -0,0 +1,318 @@ +use crate::path_utils::expand_tilde; +use reqwest::ClientBuilder; +use reqwest::{Certificate, Identity}; +use std::fmt; +use std::future::Future; +use std::time::Duration; + +// Re-export types so that `db::QueryResult` etc. work within dbx-core +pub use crate::file_validator::validate_file_path; +pub use crate::types::*; + +pub const CONNECTION_TIMEOUT_SECS: u64 = 5; +pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolCheckoutStage { + Wait, + Create, + Recycle, + Unknown, +} + +impl PoolCheckoutStage { + pub fn as_str(self) -> &'static str { + match self { + Self::Wait => "wait", + Self::Create => "create", + Self::Recycle => "recycle", + Self::Unknown => "unknown", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PoolCheckoutError { + Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration }, + Failed { database: &'static str, stage: PoolCheckoutStage, detail: String }, + Canceled, +} + +impl PoolCheckoutError { + pub fn stage(&self) -> Option { + match self { + Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage), + Self::Canceled => None, + } + } + + pub fn is_pool_saturation(&self) -> bool { + matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. }) + } +} + +impl fmt::Display for PoolCheckoutError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Timeout { database, stage, timeout } => write!( + formatter, + "{database} connection pool checkout timed out [stage={}, timeout_ms={}]", + stage.as_str(), + timeout.as_millis() + ), + Self::Failed { database, stage, detail } => { + write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str()) + } + Self::Canceled => formatter.write_str(crate::execution::QUERY_CANCELED), + } + } +} + +pub fn connection_timeout() -> Duration { + Duration::from_secs(CONNECTION_TIMEOUT_SECS) +} + +pub fn http_client_builder(timeout: Duration) -> ClientBuilder { + reqwest::Client::builder().connect_timeout(timeout).no_proxy() +} + +/// Augments an HTTP client builder with custom TLS material for drivers that +/// need to trust a non-system CA or present a client certificate (mTLS). +/// +/// * `ca_cert_path` adds an extra root certificate (PEM bundle) alongside the +/// system roots, so self-signed server certificates are trusted. +/// * `client_cert_path` + `client_key_path` (required together) are concatenated +/// into a single PEM identity, as expected by reqwest's rustls backend. +/// +/// Returns the augmented builder, or a descriptive error if a configured +/// certificate/key file cannot be read or parsed. Callers surface that error +/// instead of silently falling back to the system trust store, so a typo in a +/// cert path fails the connection loudly rather than being ignored. +pub fn apply_tls_certificates( + mut builder: ClientBuilder, + ca_cert_path: Option<&str>, + client_cert_path: Option<&str>, + client_key_path: Option<&str>, +) -> Result { + // The desktop build unifies reqwest with both `default-tls` and `rustls-tls`, + // and `TlsBackend` defaults to native-tls there, which rejects the PEM identity + // below with `incompatible TLS identity type`. Pin rustls like the Consul and + // etcd-metrics clients so custom CA roots and PEM client identities work in + // every target. + builder = builder.use_rustls_tls(); + if let Some(ca) = ca_cert_path.filter(|path| !path.is_empty()) { + let ca = expand_tilde(ca); + let contents = std::fs::read(&ca) + .map_err(|error| format!("Failed to read Elasticsearch CA certificate at {ca}: {error}"))?; + let certificates = Certificate::from_pem_bundle(&contents) + .map_err(|error| format!("Failed to parse Elasticsearch CA certificate at {ca}: {error}"))?; + for certificate in certificates { + builder = builder.add_root_certificate(certificate); + } + } + if let (Some(cert_path), Some(key_path)) = + (client_cert_path.filter(|path| !path.is_empty()), client_key_path.filter(|path| !path.is_empty())) + { + let cert_path = expand_tilde(cert_path); + let key_path = expand_tilde(key_path); + let mut pem = std::fs::read(&cert_path) + .map_err(|error| format!("Failed to read Elasticsearch client certificate at {cert_path}: {error}"))?; + // reqwest's rustls backend concatenates the cert and key as two PEM + // blocks; ensure they are newline-separated so parsing succeeds even + // when the cert file does not end with a trailing newline. + if !pem.ends_with(b"\n") { + pem.push(b'\n'); + } + pem.extend( + std::fs::read(&key_path) + .map_err(|error| format!("Failed to read Elasticsearch client key at {key_path}: {error}"))?, + ); + let identity = Identity::from_pem(&pem) + .map_err(|error| format!("Failed to parse Elasticsearch client identity: {error}"))?; + builder = builder.identity(identity); + } + Ok(builder) +} + +pub use dbx_types::json::JS_MAX_SAFE_INTEGER; +pub use dbx_types::json::{json_value_for_js, safe_i64_to_json, safe_u64_to_json}; + +pub fn hex_encode(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} + +pub fn binary_value_to_json(bytes: &[u8]) -> serde_json::Value { + serde_json::Value::String(format!("0x{}", hex_encode(bytes))) +} + +pub fn tcp_probe_timeout() -> Duration { + Duration::from_secs(TCP_PROBE_TIMEOUT_SECS) +} + +pub fn parse_connect_timeout(url: &str) -> Duration { + parse_connect_timeout_with_fallback(url, connection_timeout()) +} + +pub fn parse_connect_timeout_with_fallback(url: &str, fallback: Duration) -> Duration { + let Some(query) = url.split('?').nth(1) else { + return fallback; + }; + for param in query.split('&') { + let trimmed = param.trim(); + if trimmed.is_empty() { + continue; + } + let (key, value) = match trimmed.split_once('=') { + Some(pair) => pair, + None => continue, + }; + if key.eq_ignore_ascii_case("connect_timeout") + || key.eq_ignore_ascii_case("connectTimeout") + || key.eq_ignore_ascii_case("connection_timeout") + || key.eq_ignore_ascii_case("connectionTimeout") + { + if let Ok(v) = value.parse::() { + if (1..=300).contains(&v) { + return Duration::from_secs(v); + } + } + } + } + fallback +} + +pub async fn with_connection_timeout(label: &str, timeout: Duration, future: F) -> Result +where + F: Future>, +{ + tokio::time::timeout(timeout, future) + .await + .map_err(|_| format!("{label} connection timed out ({}s)", timeout.as_secs()))? +} + +pub async fn probe_tcp_endpoint(label: &str, host: &str, port: u16, timeout: Duration) -> Result<(), String> { + tokio::time::timeout(timeout, tokio::net::TcpStream::connect((host, port))) + .await + .map_err(|_| format!("{label} TCP connection timed out ({}s)", timeout.as_secs()))? + .map(|_| ()) + .map_err(|e| format!("{label} TCP connection failed: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn binary_values_are_displayed_as_prefixed_hex() { + assert_eq!(binary_value_to_json(&[0x00, 0x01, 0xab, 0xff]), serde_json::json!("0x0001abff")); + } +} + +#[cfg(test)] +mod tls_tests { + use super::{apply_tls_certificates, http_client_builder}; + use std::path::Path; + use std::process::Command; + use std::time::Duration; + + /// Generates a throwaway CA + client certificate/key (PEM) via openssl and + /// returns their paths. If openssl is unavailable the files stay empty, which + /// makes `apply_tls_certificates` surface a read error instead of degrading. + fn gen_certs(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) { + let ca_key = dir.join("ca.key"); + let ca_pem = dir.join("ca.pem"); + let csr = dir.join("client.csr"); + let client_key = dir.join("client.key"); + let client_crt = dir.join("client.crt"); + // Extensions force an X.509 v3 client cert; some toolchains (e.g. + // OpenSSL 3.0 on CI) emit v1 without one, which rustls rejects. + let client_ext = dir.join("client.ext"); + let _ = std::fs::write(&client_ext, "basicConstraints=critical,CA:FALSE\n"); + let _ = Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ca_key.to_str().unwrap(), + "-out", + ca_pem.to_str().unwrap(), + "-days", + "1", + "-subj", + "/CN=dbx-test-ca", + ]) + .output(); + let _ = Command::new("openssl") + .args([ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + client_key.to_str().unwrap(), + "-out", + csr.to_str().unwrap(), + "-subj", + "/CN=dbx-test-client", + ]) + .output(); + let _ = Command::new("openssl") + .args([ + "x509", + "-req", + "-in", + csr.to_str().unwrap(), + "-CA", + ca_pem.to_str().unwrap(), + "-CAkey", + ca_key.to_str().unwrap(), + "-CAcreateserial", + "-out", + client_crt.to_str().unwrap(), + "-days", + "1", + "-extfile", + client_ext.to_str().unwrap(), + ]) + .output(); + (ca_pem, client_crt, client_key) + } + + #[test] + fn apply_tls_certificates_builds_with_ca_and_client_cert() { + let dir = std::env::temp_dir().join(format!("dbx-tls-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (ca, client_crt, client_key) = gen_certs(&dir); + + let built = apply_tls_certificates( + http_client_builder(Duration::from_secs(5)), + ca.to_str(), + client_crt.to_str(), + client_key.to_str(), + ) + .expect("client should build with CA + client cert") + .build(); + assert!(built.is_ok(), "built client should be usable with CA + client cert: {:?}", built.err()); + + // A misconfigured CA path must now surface a descriptive error instead of + // silently degrading to the system trust store. + let missing = apply_tls_certificates( + http_client_builder(Duration::from_secs(5)), + Some("/nonexistent/ca.pem"), + None, + None, + ); + assert!(missing.is_err(), "missing CA path must surface a read error"); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/dbx-drivers/src/db/ddl_scan.rs b/crates/dbx-driver-support/src/ddl_scan.rs similarity index 100% rename from crates/dbx-drivers/src/db/ddl_scan.rs rename to crates/dbx-driver-support/src/ddl_scan.rs diff --git a/crates/dbx-drivers/src/db/document_result.rs b/crates/dbx-driver-support/src/document_result.rs similarity index 100% rename from crates/dbx-drivers/src/db/document_result.rs rename to crates/dbx-driver-support/src/document_result.rs diff --git a/crates/dbx-drivers/src/db/elasticsearch_sql.rs b/crates/dbx-driver-support/src/elasticsearch_sql.rs similarity index 100% rename from crates/dbx-drivers/src/db/elasticsearch_sql.rs rename to crates/dbx-driver-support/src/elasticsearch_sql.rs diff --git a/crates/dbx-drivers/src/execution.rs b/crates/dbx-driver-support/src/execution.rs similarity index 100% rename from crates/dbx-drivers/src/execution.rs rename to crates/dbx-driver-support/src/execution.rs diff --git a/crates/dbx-drivers/src/db/file_validator.rs b/crates/dbx-driver-support/src/file_validator.rs similarity index 100% rename from crates/dbx-drivers/src/db/file_validator.rs rename to crates/dbx-driver-support/src/file_validator.rs diff --git a/crates/dbx-drivers/src/db/http_tunnel.rs b/crates/dbx-driver-support/src/http_tunnel.rs similarity index 100% rename from crates/dbx-drivers/src/db/http_tunnel.rs rename to crates/dbx-driver-support/src/http_tunnel.rs diff --git a/crates/dbx-driver-support/src/lib.rs b/crates/dbx-driver-support/src/lib.rs new file mode 100644 index 0000000000..c0b30d6e9b --- /dev/null +++ b/crates/dbx-driver-support/src/lib.rs @@ -0,0 +1,16 @@ +#![recursion_limit = "256"] + +pub use dbx_platform::path_utils; +pub use dbx_types::{models, types}; + +pub mod db; +pub mod ddl_scan; +pub mod document_result; +pub mod elasticsearch_sql; +pub mod execution; +pub mod file_validator; +pub mod http_tunnel; +pub mod runtime_config; +pub mod ssh_config; +pub mod ssh_host_key; +pub mod wkb; diff --git a/crates/dbx-drivers/src/runtime_config.rs b/crates/dbx-driver-support/src/runtime_config.rs similarity index 100% rename from crates/dbx-drivers/src/runtime_config.rs rename to crates/dbx-driver-support/src/runtime_config.rs diff --git a/crates/dbx-drivers/src/ssh_config.rs b/crates/dbx-driver-support/src/ssh_config.rs similarity index 100% rename from crates/dbx-drivers/src/ssh_config.rs rename to crates/dbx-driver-support/src/ssh_config.rs diff --git a/crates/dbx-drivers/src/db/ssh_host_key.rs b/crates/dbx-driver-support/src/ssh_host_key.rs similarity index 100% rename from crates/dbx-drivers/src/db/ssh_host_key.rs rename to crates/dbx-driver-support/src/ssh_host_key.rs diff --git a/crates/dbx-drivers/src/db/wkb.rs b/crates/dbx-driver-support/src/wkb.rs similarity index 100% rename from crates/dbx-drivers/src/db/wkb.rs rename to crates/dbx-driver-support/src/wkb.rs diff --git a/crates/dbx-drivers/Cargo.toml b/crates/dbx-drivers/Cargo.toml index ad7f807af1..f6d04b922a 100644 --- a/crates/dbx-drivers/Cargo.toml +++ b/crates/dbx-drivers/Cargo.toml @@ -7,17 +7,27 @@ publish = false [features] default = [] -duckdb-sidecar = ["dbx-sql/duckdb-sidecar"] +duckdb-sidecar = [] dynamodb = ["dep:aws-sdk-dynamodb"] mq-admin = ["dbx-types/mq-admin"] sqlite-bundled = ["rusqlite/bundled"] sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl"] sqlite-multiple-ciphers = ["dep:libsqlite3-hotbundle"] -test-support = ["dbx-platform/test-support"] +test-support = ["dbx-driver-agent/test-support", "dbx-driver-mysql/test-support", "dbx-driver-redis/test-support", "dbx-driver-support/test-support", "dbx-platform/test-support"] [dependencies] +dbx-driver-agent = { path = "../dbx-driver-agent" } +dbx-driver-support = { path = "../dbx-driver-support" } +dbx-driver-elasticsearch = { path = "../dbx-driver-elasticsearch" } +dbx-driver-mongodb = { path = "../dbx-driver-mongodb" } +dbx-driver-mysql = { path = "../dbx-driver-mysql" } +dbx-driver-postgres = { path = "../dbx-driver-postgres" } +dbx-driver-redis = { path = "../dbx-driver-redis" } +dbx-driver-sqlserver = { path = "../dbx-driver-sqlserver" } dbx-types = { path = "../dbx-types", default-features = false } -dbx-sql = { path = "../dbx-sql", default-features = false } +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-sql-data = { path = "../dbx-sql-data" } +dbx-sql-dialect = { path = "../dbx-sql-dialect" } dbx-platform = { path = "../dbx-platform", features = ["host-prompts", "downloads"] } dbx-sqlite-worker = { path = "../dbx-sqlite-worker", default-features = false } serde = { version = "1.0", features = ["derive"] } @@ -43,13 +53,11 @@ rusqlite = { version = "0.32", default-features = false, features = ["load_exten libsqlite3-hotbundle = { git = "https://github.com/zipg/libsqlite3-hotbundle.git", rev = "7f12f203b4f4694a273410bd9b02d77f82b6114e", features = ["cipher-sqlcipher"], optional = true } mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "client_ed25519", "chrono", "rust_decimal"] } sqlparser = { version = "0.62.0", features = ["visitor"] } -redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tls-rustls-insecure", "tokio-rustls-comp", "sentinel", "cluster-async"] } rustls = { version = "0.23", features = ["aws-lc-rs"] } rustls-pemfile = "2.2" tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] } reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "stream", "rustls-tls", "socks"] } futures = "0.3" -mongodb = { version = "3.2.5", features = ["socks5-proxy"] } aws-sdk-dynamodb = { version = "=1.104.0", optional = true } russh = "0.60" pageant = "0.2" @@ -69,6 +77,7 @@ tempfile = "3" windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem", "Win32_Foundation", "Win32_Security"] } [dev-dependencies] +dbx-driver-support = { path = "../dbx-driver-support", features = ["test-support"] } dbx-platform = { path = "../dbx-platform", features = ["test-support"] } mysql_common = { git = "https://github.com/t8y2/rust_mysql_common.git", rev = "47740d85a8277167a5717afac785d37b46a36296", default-features = false } tokio = { version = "1", features = ["test-util"] } diff --git a/crates/dbx-drivers/src/db/mod.rs b/crates/dbx-drivers/src/db/mod.rs index 4dc7a9e510..a89ffa6f73 100644 --- a/crates/dbx-drivers/src/db/mod.rs +++ b/crates/dbx-drivers/src/db/mod.rs @@ -1,12 +1,10 @@ -pub mod agent_driver; +pub use dbx_driver_agent::agent_driver; pub mod clickhouse_driver; pub mod cloudberry; pub mod cloudflare_d1; pub use cloudflare_d1 as cloudflare_d1_driver; -pub mod ddl_scan; -pub mod document_result; -pub mod dolt; -pub mod doris; +pub use dbx_driver_mysql::{dolt, doris}; +pub use dbx_driver_support::{ddl_scan, document_result}; pub mod duckdb_sql; #[cfg(feature = "duckdb-sidecar")] pub mod duckdb_worker_process; @@ -19,359 +17,38 @@ pub mod dynamodb_driver; #[path = "dynamodb_driver_disabled.rs"] pub mod dynamodb_driver; pub mod easysearch_driver; -pub mod elasticsearch_driver; -pub mod elasticsearch_sql; -pub mod file_validator; +pub use dbx_driver_elasticsearch as elasticsearch_driver; +pub use dbx_driver_support::{elasticsearch_sql, file_validator}; pub mod hbase_driver; -pub mod http_tunnel; +pub use dbx_driver_support::http_tunnel; pub mod influxdb3_driver; pub mod influxdb_driver; -pub mod manticoresearch; +pub use dbx_driver_mysql::manticoresearch; pub mod meilisearch_driver; -pub mod mongo_driver; -pub mod mysql; -pub mod mysql_compatible; -pub mod ob_oracle; -pub mod oceanbase_mysql; +pub use dbx_driver_mongodb::mongo_driver; +pub use dbx_driver_mysql::{mysql, mysql_compatible, ob_oracle, oceanbase_mysql}; pub mod opentenbase; -pub mod postgres; +pub use dbx_driver_postgres as postgres; pub mod proxy_tunnel; pub mod questdb; -pub mod redis_driver; +pub use dbx_driver_redis as redis_driver; pub mod rqlite_driver; pub mod salesforce_driver; pub mod solr_driver; pub mod sqlite; pub mod sqlite_worker; -pub mod sqlserver; -pub mod ssh_host_key; +pub use dbx_driver_sqlserver as sqlserver; +pub use dbx_driver_support::ssh_host_key; pub use dbx_platform::ssh_prompt; pub mod ssh_tunnel; -pub mod starrocks; -pub use dbx_sql::tdsql_mysql; -pub mod tidb; +pub use dbx_driver_mysql::starrocks; +pub use dbx_driver_mysql::tidb; +pub use dbx_sql_core::tdsql_mysql; pub mod transport_layer_tunnel; pub mod turso_driver; pub mod vector_driver; pub mod victoriametrics_driver; -pub mod wkb; +pub use dbx_driver_support::wkb; -use crate::path_utils::expand_tilde; -use reqwest::ClientBuilder; -use reqwest::{Certificate, Identity}; -use std::fmt; -use std::future::Future; -use std::time::Duration; - -// Re-export types so that `db::QueryResult` etc. work within dbx-core pub use crate::mysql_event_sql::MysqlEventInfo; -pub use crate::types::*; -pub use file_validator::validate_file_path; - -pub const CONNECTION_TIMEOUT_SECS: u64 = 5; -pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PoolCheckoutStage { - Wait, - Create, - Recycle, - Unknown, -} - -impl PoolCheckoutStage { - pub fn as_str(self) -> &'static str { - match self { - Self::Wait => "wait", - Self::Create => "create", - Self::Recycle => "recycle", - Self::Unknown => "unknown", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PoolCheckoutError { - Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration }, - Failed { database: &'static str, stage: PoolCheckoutStage, detail: String }, - Canceled, -} - -impl PoolCheckoutError { - pub fn stage(&self) -> Option { - match self { - Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage), - Self::Canceled => None, - } - } - - pub fn is_pool_saturation(&self) -> bool { - matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. }) - } -} - -impl fmt::Display for PoolCheckoutError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Timeout { database, stage, timeout } => write!( - formatter, - "{database} connection pool checkout timed out [stage={}, timeout_ms={}]", - stage.as_str(), - timeout.as_millis() - ), - Self::Failed { database, stage, detail } => { - write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str()) - } - Self::Canceled => formatter.write_str(crate::execution::QUERY_CANCELED), - } - } -} - -pub fn connection_timeout() -> Duration { - Duration::from_secs(CONNECTION_TIMEOUT_SECS) -} - -pub fn http_client_builder(timeout: Duration) -> ClientBuilder { - reqwest::Client::builder().connect_timeout(timeout).no_proxy() -} - -/// Augments an HTTP client builder with custom TLS material for drivers that -/// need to trust a non-system CA or present a client certificate (mTLS). -/// -/// * `ca_cert_path` adds an extra root certificate (PEM bundle) alongside the -/// system roots, so self-signed server certificates are trusted. -/// * `client_cert_path` + `client_key_path` (required together) are concatenated -/// into a single PEM identity, as expected by reqwest's rustls backend. -/// -/// Returns the augmented builder, or a descriptive error if a configured -/// certificate/key file cannot be read or parsed. Callers surface that error -/// instead of silently falling back to the system trust store, so a typo in a -/// cert path fails the connection loudly rather than being ignored. -pub fn apply_tls_certificates( - mut builder: ClientBuilder, - ca_cert_path: Option<&str>, - client_cert_path: Option<&str>, - client_key_path: Option<&str>, -) -> Result { - // The desktop build unifies reqwest with both `default-tls` and `rustls-tls`, - // and `TlsBackend` defaults to native-tls there, which rejects the PEM identity - // below with `incompatible TLS identity type`. Pin rustls like the Consul and - // etcd-metrics clients so custom CA roots and PEM client identities work in - // every target. - builder = builder.use_rustls_tls(); - if let Some(ca) = ca_cert_path.filter(|path| !path.is_empty()) { - let ca = expand_tilde(ca); - let contents = std::fs::read(&ca) - .map_err(|error| format!("Failed to read Elasticsearch CA certificate at {ca}: {error}"))?; - let certificates = Certificate::from_pem_bundle(&contents) - .map_err(|error| format!("Failed to parse Elasticsearch CA certificate at {ca}: {error}"))?; - for certificate in certificates { - builder = builder.add_root_certificate(certificate); - } - } - if let (Some(cert_path), Some(key_path)) = - (client_cert_path.filter(|path| !path.is_empty()), client_key_path.filter(|path| !path.is_empty())) - { - let cert_path = expand_tilde(cert_path); - let key_path = expand_tilde(key_path); - let mut pem = std::fs::read(&cert_path) - .map_err(|error| format!("Failed to read Elasticsearch client certificate at {cert_path}: {error}"))?; - // reqwest's rustls backend concatenates the cert and key as two PEM - // blocks; ensure they are newline-separated so parsing succeeds even - // when the cert file does not end with a trailing newline. - if !pem.ends_with(b"\n") { - pem.push(b'\n'); - } - pem.extend( - std::fs::read(&key_path) - .map_err(|error| format!("Failed to read Elasticsearch client key at {key_path}: {error}"))?, - ); - let identity = Identity::from_pem(&pem) - .map_err(|error| format!("Failed to parse Elasticsearch client identity: {error}"))?; - builder = builder.identity(identity); - } - Ok(builder) -} - -pub use dbx_types::json::JS_MAX_SAFE_INTEGER; -pub use dbx_types::json::{json_value_for_js, safe_i64_to_json, safe_u64_to_json}; - -pub fn hex_encode(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut out = String::with_capacity(bytes.len() * 2); - for &byte in bytes { - out.push(HEX[(byte >> 4) as usize] as char); - out.push(HEX[(byte & 0x0f) as usize] as char); - } - out -} - -pub fn binary_value_to_json(bytes: &[u8]) -> serde_json::Value { - serde_json::Value::String(format!("0x{}", hex_encode(bytes))) -} - -pub fn tcp_probe_timeout() -> Duration { - Duration::from_secs(TCP_PROBE_TIMEOUT_SECS) -} - -pub fn parse_connect_timeout(url: &str) -> Duration { - parse_connect_timeout_with_fallback(url, connection_timeout()) -} - -pub fn parse_connect_timeout_with_fallback(url: &str, fallback: Duration) -> Duration { - let Some(query) = url.split('?').nth(1) else { - return fallback; - }; - for param in query.split('&') { - let trimmed = param.trim(); - if trimmed.is_empty() { - continue; - } - let (key, value) = match trimmed.split_once('=') { - Some(pair) => pair, - None => continue, - }; - if key.eq_ignore_ascii_case("connect_timeout") - || key.eq_ignore_ascii_case("connectTimeout") - || key.eq_ignore_ascii_case("connection_timeout") - || key.eq_ignore_ascii_case("connectionTimeout") - { - if let Ok(v) = value.parse::() { - if (1..=300).contains(&v) { - return Duration::from_secs(v); - } - } - } - } - fallback -} - -pub async fn with_connection_timeout(label: &str, timeout: Duration, future: F) -> Result -where - F: Future>, -{ - tokio::time::timeout(timeout, future) - .await - .map_err(|_| format!("{label} connection timed out ({}s)", timeout.as_secs()))? -} - -pub async fn probe_tcp_endpoint(label: &str, host: &str, port: u16, timeout: Duration) -> Result<(), String> { - tokio::time::timeout(timeout, tokio::net::TcpStream::connect((host, port))) - .await - .map_err(|_| format!("{label} TCP connection timed out ({}s)", timeout.as_secs()))? - .map(|_| ()) - .map_err(|e| format!("{label} TCP connection failed: {e}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn binary_values_are_displayed_as_prefixed_hex() { - assert_eq!(binary_value_to_json(&[0x00, 0x01, 0xab, 0xff]), serde_json::json!("0x0001abff")); - } -} - -#[cfg(test)] -mod tls_tests { - use super::{apply_tls_certificates, http_client_builder}; - use std::path::Path; - use std::process::Command; - use std::time::Duration; - - /// Generates a throwaway CA + client certificate/key (PEM) via openssl and - /// returns their paths. If openssl is unavailable the files stay empty, which - /// makes `apply_tls_certificates` surface a read error instead of degrading. - fn gen_certs(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) { - let ca_key = dir.join("ca.key"); - let ca_pem = dir.join("ca.pem"); - let csr = dir.join("client.csr"); - let client_key = dir.join("client.key"); - let client_crt = dir.join("client.crt"); - // Extensions force an X.509 v3 client cert; some toolchains (e.g. - // OpenSSL 3.0 on CI) emit v1 without one, which rustls rejects. - let client_ext = dir.join("client.ext"); - let _ = std::fs::write(&client_ext, "basicConstraints=critical,CA:FALSE\n"); - let _ = Command::new("openssl") - .args([ - "req", - "-x509", - "-newkey", - "rsa:2048", - "-nodes", - "-keyout", - ca_key.to_str().unwrap(), - "-out", - ca_pem.to_str().unwrap(), - "-days", - "1", - "-subj", - "/CN=dbx-test-ca", - ]) - .output(); - let _ = Command::new("openssl") - .args([ - "req", - "-newkey", - "rsa:2048", - "-nodes", - "-keyout", - client_key.to_str().unwrap(), - "-out", - csr.to_str().unwrap(), - "-subj", - "/CN=dbx-test-client", - ]) - .output(); - let _ = Command::new("openssl") - .args([ - "x509", - "-req", - "-in", - csr.to_str().unwrap(), - "-CA", - ca_pem.to_str().unwrap(), - "-CAkey", - ca_key.to_str().unwrap(), - "-CAcreateserial", - "-out", - client_crt.to_str().unwrap(), - "-days", - "1", - "-extfile", - client_ext.to_str().unwrap(), - ]) - .output(); - (ca_pem, client_crt, client_key) - } - - #[test] - fn apply_tls_certificates_builds_with_ca_and_client_cert() { - let dir = std::env::temp_dir().join(format!("dbx-tls-test-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let (ca, client_crt, client_key) = gen_certs(&dir); - - let built = apply_tls_certificates( - http_client_builder(Duration::from_secs(5)), - ca.to_str(), - client_crt.to_str(), - client_key.to_str(), - ) - .expect("client should build with CA + client cert") - .build(); - assert!(built.is_ok(), "built client should be usable with CA + client cert: {:?}", built.err()); - - // A misconfigured CA path must now surface a descriptive error instead of - // silently degrading to the system trust store. - let missing = apply_tls_certificates( - http_client_builder(Duration::from_secs(5)), - Some("/nonexistent/ca.pem"), - None, - None, - ); - assert!(missing.is_err(), "missing CA path must surface a read error"); - - let _ = std::fs::remove_dir_all(&dir); - } -} +pub use dbx_driver_support::db::*; diff --git a/crates/dbx-drivers/src/lib.rs b/crates/dbx-drivers/src/lib.rs index 12671f95b6..f66ccb4445 100644 --- a/crates/dbx-drivers/src/lib.rs +++ b/crates/dbx-drivers/src/lib.rs @@ -1,26 +1,20 @@ #![recursion_limit = "256"] -pub use dbx_sql::query_result_sql; +pub use dbx_sql_data::query_result_sql; +pub use dbx_driver_support::{runtime_config, ssh_config}; pub use dbx_platform::download::DownloadSource; pub use dbx_platform::{path_utils, process}; -pub use dbx_sql::{mysql_ddl_normalize, mysql_event_sql, sql, sql_dialect, sql_error_position, sqlserver_temporal}; +pub use dbx_sql_core::{mysql_ddl_normalize, mysql_event_sql, sql, sql_error_position, sqlserver_temporal}; +pub use dbx_sql_dialect::sql_dialect; pub use dbx_types::{database_manifest, models, types}; -pub mod agent_catalog; -pub mod agent_connection; -pub mod agent_manager; -pub mod agent_offline_export; -pub mod agent_recovery; -pub mod agent_runtime; -pub mod agent_service; -pub mod backend_error; -pub mod database_capabilities; +pub use dbx_driver_agent::{ + agent_catalog, agent_connection, agent_manager, agent_offline_export, agent_recovery, agent_runtime, agent_service, + backend_error, database_capabilities, +}; pub mod db; -pub mod execution; +pub use dbx_driver_support::execution; pub mod metadata; -pub mod mongo_oidc; -pub mod mongo_shell; -pub mod runtime_config; +pub use dbx_driver_mongodb::{mongo_oidc, mongo_shell}; pub mod salesforce_oauth; -pub mod ssh_config; diff --git a/crates/dbx-drivers/tests/live_sqlserver_issue_9984_real.rs b/crates/dbx-drivers/tests/live_sqlserver_issue_9984_real.rs index e581d948b1..ce96e67f04 100644 --- a/crates/dbx-drivers/tests/live_sqlserver_issue_9984_real.rs +++ b/crates/dbx-drivers/tests/live_sqlserver_issue_9984_real.rs @@ -1,6 +1,6 @@ use dbx_drivers::db::sqlserver::{self, SqlServerStreamItem}; use dbx_drivers::models::connection::DatabaseType; -use dbx_sql::data_grid_sql::format_grid_sql_literal; +use dbx_sql_data::data_grid_sql::format_grid_sql_literal; use serde_json::Value; use std::time::Duration; diff --git a/crates/dbx-sql-core/Cargo.toml b/crates/dbx-sql-core/Cargo.toml new file mode 100644 index 0000000000..3de2674f85 --- /dev/null +++ b/crates/dbx-sql-core/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "dbx-sql-core" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-types = { path = "../dbx-types", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +encoding_rs = "0.8" +log = "0.4" +regex = "1" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +unicode-ident = "1" +winnow = "1.0" diff --git a/crates/dbx-sql/src/dml_preview_sql.rs b/crates/dbx-sql-core/src/dml_preview_sql.rs similarity index 100% rename from crates/dbx-sql/src/dml_preview_sql.rs rename to crates/dbx-sql-core/src/dml_preview_sql.rs diff --git a/crates/dbx-sql-core/src/lib.rs b/crates/dbx-sql-core/src/lib.rs new file mode 100644 index 0000000000..0dab086d72 --- /dev/null +++ b/crates/dbx-sql-core/src/lib.rs @@ -0,0 +1,18 @@ +#![recursion_limit = "256"] + +pub use dbx_types::{database_manifest, models, types}; + +pub mod dml_preview_sql; +pub mod mysql_ddl_normalize; +pub mod mysql_event_sql; +pub mod object_source_sql; +pub mod query_execution_sql; +pub mod sql; +pub mod sql_analysis; +pub mod sql_diagnostics; +pub mod sql_editability; +pub mod sql_error_position; +pub mod sql_risk; +pub mod sqlserver_temporal; +pub mod tdsql_mysql; +pub mod value_literals; diff --git a/crates/dbx-sql/src/mysql_ddl_normalize.rs b/crates/dbx-sql-core/src/mysql_ddl_normalize.rs similarity index 100% rename from crates/dbx-sql/src/mysql_ddl_normalize.rs rename to crates/dbx-sql-core/src/mysql_ddl_normalize.rs diff --git a/crates/dbx-sql/src/mysql_event_sql.rs b/crates/dbx-sql-core/src/mysql_event_sql.rs similarity index 100% rename from crates/dbx-sql/src/mysql_event_sql.rs rename to crates/dbx-sql-core/src/mysql_event_sql.rs diff --git a/crates/dbx-sql/src/object_source_sql.rs b/crates/dbx-sql-core/src/object_source_sql.rs similarity index 100% rename from crates/dbx-sql/src/object_source_sql.rs rename to crates/dbx-sql-core/src/object_source_sql.rs diff --git a/crates/dbx-sql/src/query_execution_sql.rs b/crates/dbx-sql-core/src/query_execution_sql.rs similarity index 100% rename from crates/dbx-sql/src/query_execution_sql.rs rename to crates/dbx-sql-core/src/query_execution_sql.rs diff --git a/crates/dbx-sql/src/sql.rs b/crates/dbx-sql-core/src/sql.rs similarity index 100% rename from crates/dbx-sql/src/sql.rs rename to crates/dbx-sql-core/src/sql.rs diff --git a/crates/dbx-sql/src/sql_analysis.rs b/crates/dbx-sql-core/src/sql_analysis.rs similarity index 100% rename from crates/dbx-sql/src/sql_analysis.rs rename to crates/dbx-sql-core/src/sql_analysis.rs diff --git a/crates/dbx-sql/src/sql_diagnostics.rs b/crates/dbx-sql-core/src/sql_diagnostics.rs similarity index 100% rename from crates/dbx-sql/src/sql_diagnostics.rs rename to crates/dbx-sql-core/src/sql_diagnostics.rs diff --git a/crates/dbx-sql/src/sql_editability.rs b/crates/dbx-sql-core/src/sql_editability.rs similarity index 100% rename from crates/dbx-sql/src/sql_editability.rs rename to crates/dbx-sql-core/src/sql_editability.rs diff --git a/crates/dbx-sql/src/sql_error_position.rs b/crates/dbx-sql-core/src/sql_error_position.rs similarity index 100% rename from crates/dbx-sql/src/sql_error_position.rs rename to crates/dbx-sql-core/src/sql_error_position.rs diff --git a/crates/dbx-sql/src/sql_risk.rs b/crates/dbx-sql-core/src/sql_risk.rs similarity index 100% rename from crates/dbx-sql/src/sql_risk.rs rename to crates/dbx-sql-core/src/sql_risk.rs diff --git a/crates/dbx-sql/src/sqlserver_temporal.rs b/crates/dbx-sql-core/src/sqlserver_temporal.rs similarity index 100% rename from crates/dbx-sql/src/sqlserver_temporal.rs rename to crates/dbx-sql-core/src/sqlserver_temporal.rs diff --git a/crates/dbx-sql/src/tdsql_mysql.rs b/crates/dbx-sql-core/src/tdsql_mysql.rs similarity index 100% rename from crates/dbx-sql/src/tdsql_mysql.rs rename to crates/dbx-sql-core/src/tdsql_mysql.rs diff --git a/crates/dbx-sql/src/value_literals.rs b/crates/dbx-sql-core/src/value_literals.rs similarity index 100% rename from crates/dbx-sql/src/value_literals.rs rename to crates/dbx-sql-core/src/value_literals.rs diff --git a/crates/dbx-sql-data/Cargo.toml b/crates/dbx-sql-data/Cargo.toml new file mode 100644 index 0000000000..4daa57c61f --- /dev/null +++ b/crates/dbx-sql-data/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "dbx-sql-data" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +openapi = ["dep:utoipa", "dbx-types/openapi"] + +[dependencies] +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-sql-dialect = { path = "../dbx-sql-dialect" } +dbx-types = { path = "../dbx-types", default-features = false } +chrono = "0.4" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +unicode-width = "0.2" +utoipa = { version = "5.5", optional = true } diff --git a/crates/dbx-sql/src/data_grid_extractors/contract.rs b/crates/dbx-sql-data/src/data_grid_extractors/contract.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/contract.rs rename to crates/dbx-sql-data/src/data_grid_extractors/contract.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/delimited.rs b/crates/dbx-sql-data/src/data_grid_extractors/delimited.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/delimited.rs rename to crates/dbx-sql-data/src/data_grid_extractors/delimited.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/document.rs b/crates/dbx-sql-data/src/data_grid_extractors/document.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/document.rs rename to crates/dbx-sql-data/src/data_grid_extractors/document.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/json.rs b/crates/dbx-sql-data/src/data_grid_extractors/json.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/json.rs rename to crates/dbx-sql-data/src/data_grid_extractors/json.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/mod.rs b/crates/dbx-sql-data/src/data_grid_extractors/mod.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/mod.rs rename to crates/dbx-sql-data/src/data_grid_extractors/mod.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/raw.rs b/crates/dbx-sql-data/src/data_grid_extractors/raw.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/raw.rs rename to crates/dbx-sql-data/src/data_grid_extractors/raw.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/sql.rs b/crates/dbx-sql-data/src/data_grid_extractors/sql.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/sql.rs rename to crates/dbx-sql-data/src/data_grid_extractors/sql.rs diff --git a/crates/dbx-sql/src/data_grid_extractors/tests.rs b/crates/dbx-sql-data/src/data_grid_extractors/tests.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_extractors/tests.rs rename to crates/dbx-sql-data/src/data_grid_extractors/tests.rs diff --git a/crates/dbx-sql/src/data_grid_iotdb_sql.rs b/crates/dbx-sql-data/src/data_grid_iotdb_sql.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_iotdb_sql.rs rename to crates/dbx-sql-data/src/data_grid_iotdb_sql.rs diff --git a/crates/dbx-sql/src/data_grid_neo4j_sql.rs b/crates/dbx-sql-data/src/data_grid_neo4j_sql.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_neo4j_sql.rs rename to crates/dbx-sql-data/src/data_grid_neo4j_sql.rs diff --git a/crates/dbx-sql/src/data_grid_salesforce_sql.rs b/crates/dbx-sql-data/src/data_grid_salesforce_sql.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_salesforce_sql.rs rename to crates/dbx-sql-data/src/data_grid_salesforce_sql.rs diff --git a/crates/dbx-sql/src/data_grid_sql.rs b/crates/dbx-sql-data/src/data_grid_sql.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_sql.rs rename to crates/dbx-sql-data/src/data_grid_sql.rs diff --git a/crates/dbx-sql/src/data_grid_tdengine_sql.rs b/crates/dbx-sql-data/src/data_grid_tdengine_sql.rs similarity index 100% rename from crates/dbx-sql/src/data_grid_tdengine_sql.rs rename to crates/dbx-sql-data/src/data_grid_tdengine_sql.rs diff --git a/crates/dbx-sql/src/database_search_sql.rs b/crates/dbx-sql-data/src/database_search_sql.rs similarity index 100% rename from crates/dbx-sql/src/database_search_sql.rs rename to crates/dbx-sql-data/src/database_search_sql.rs diff --git a/crates/dbx-sql-data/src/lib.rs b/crates/dbx-sql-data/src/lib.rs new file mode 100644 index 0000000000..37e578f0e1 --- /dev/null +++ b/crates/dbx-sql-data/src/lib.rs @@ -0,0 +1,10 @@ +#![recursion_limit = "256"] + +pub use dbx_sql_core::{sql, sqlserver_temporal, tdsql_mysql, value_literals}; +pub use dbx_sql_dialect::sql_dialect; +pub use dbx_types::{database_manifest, models, types}; + +pub mod data_grid_extractors; +pub mod data_grid_sql; +pub mod database_search_sql; +pub mod query_result_sql; diff --git a/crates/dbx-sql/src/query_result_sql.rs b/crates/dbx-sql-data/src/query_result_sql.rs similarity index 100% rename from crates/dbx-sql/src/query_result_sql.rs rename to crates/dbx-sql-data/src/query_result_sql.rs diff --git a/crates/dbx-sql-dialect/Cargo.toml b/crates/dbx-sql-dialect/Cargo.toml new file mode 100644 index 0000000000..92d1ea66c2 --- /dev/null +++ b/crates/dbx-sql-dialect/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "dbx-sql-dialect" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-types = { path = "../dbx-types", default-features = false } +chrono = { version = "0.4", features = ["serde"] } +log = "0.4" +notify = { version = "7", default-features = false, features = ["macos_kqueue"] } +percent-encoding = "2" +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.9" +tokio = { version = "1", features = ["sync"] } + +[dev-dependencies] +insta = { version = "1", features = ["json", "glob"] } +tempfile = "3" diff --git a/crates/dbx-sql/build.rs b/crates/dbx-sql-dialect/build.rs similarity index 100% rename from crates/dbx-sql/build.rs rename to crates/dbx-sql-dialect/build.rs diff --git a/crates/dbx-sql/src/dml_binding.rs b/crates/dbx-sql-dialect/src/dml_binding.rs similarity index 100% rename from crates/dbx-sql/src/dml_binding.rs rename to crates/dbx-sql-dialect/src/dml_binding.rs diff --git a/crates/dbx-sql-dialect/src/lib.rs b/crates/dbx-sql-dialect/src/lib.rs new file mode 100644 index 0000000000..ea89bf44e6 --- /dev/null +++ b/crates/dbx-sql-dialect/src/lib.rs @@ -0,0 +1,6 @@ +#![recursion_limit = "256"] + +pub use dbx_types::{database_manifest, models, types}; + +pub mod dml_binding; +pub mod sql_dialect; diff --git a/crates/dbx-sql/src/sql_dialect.rs b/crates/dbx-sql-dialect/src/sql_dialect.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect.rs rename to crates/dbx-sql-dialect/src/sql_dialect.rs diff --git a/crates/dbx-sql/src/sql_dialect/capabilities.rs b/crates/dbx-sql-dialect/src/sql_dialect/capabilities.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/capabilities.rs rename to crates/dbx-sql-dialect/src/sql_dialect/capabilities.rs diff --git a/crates/dbx-sql/src/sql_dialect/ddl_profile.rs b/crates/dbx-sql-dialect/src/sql_dialect/ddl_profile.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/ddl_profile.rs rename to crates/dbx-sql-dialect/src/sql_dialect/ddl_profile.rs diff --git a/crates/dbx-sql/src/sql_dialect/descriptor.rs b/crates/dbx-sql-dialect/src/sql_dialect/descriptor.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/descriptor.rs rename to crates/dbx-sql-dialect/src/sql_dialect/descriptor.rs diff --git a/crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs b/crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs rename to crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs diff --git a/crates/dbx-sql/src/sql_dialect/dialect_loader.rs b/crates/dbx-sql-dialect/src/sql_dialect/dialect_loader.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/dialect_loader.rs rename to crates/dbx-sql-dialect/src/sql_dialect/dialect_loader.rs diff --git a/crates/dbx-sql/src/sql_dialect/dialect_types.rs b/crates/dbx-sql-dialect/src/sql_dialect/dialect_types.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/dialect_types.rs rename to crates/dbx-sql-dialect/src/sql_dialect/dialect_types.rs diff --git a/crates/dbx-sql/src/sql_dialect/dialect_yaml.rs b/crates/dbx-sql-dialect/src/sql_dialect/dialect_yaml.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/dialect_yaml.rs rename to crates/dbx-sql-dialect/src/sql_dialect/dialect_yaml.rs diff --git a/crates/dbx-sql/src/sql_dialect/hot_reload.rs b/crates/dbx-sql-dialect/src/sql_dialect/hot_reload.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/hot_reload.rs rename to crates/dbx-sql-dialect/src/sql_dialect/hot_reload.rs diff --git a/crates/dbx-sql/src/sql_dialect/identifiers.rs b/crates/dbx-sql-dialect/src/sql_dialect/identifiers.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/identifiers.rs rename to crates/dbx-sql-dialect/src/sql_dialect/identifiers.rs diff --git a/crates/dbx-sql/src/sql_dialect/inference.rs b/crates/dbx-sql-dialect/src/sql_dialect/inference.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/inference.rs rename to crates/dbx-sql-dialect/src/sql_dialect/inference.rs diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__all_dialects_info.snap similarity index 99% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__all_dialects_info.snap index e09b4dad2c..0987d4be50 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__all_dialects_info.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 67 expression: all --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap similarity index 91% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap index cf1eb10fb6..a65a2b3a4a 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 37 expression: desc --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_descriptor.snap similarity index 91% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_descriptor.snap index 8ed1d1bab9..ccc4e9086d 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 7 expression: desc --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_info.snap similarity index 94% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_info.snap index 5650324e85..43e90603ac 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_info.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 43 expression: info --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap similarity index 96% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap index 6cf6254ca3..e47afabef7 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 55 expression: "&matrix.rules" --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__oracle_descriptor.snap similarity index 91% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__oracle_descriptor.snap index 80c549bbad..86e43625a6 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__oracle_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 31 expression: desc --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_descriptor.snap similarity index 91% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_descriptor.snap index 64762409ca..9ed7487cd4 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 13 expression: desc --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_info.snap similarity index 94% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_info.snap index f97ed8c735..9cf526d373 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_info.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 49 expression: info --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap similarity index 95% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap index d92ebe94dc..6a39298a70 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 61 expression: "&matrix.rules" --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap similarity index 91% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap index ce3dda94b4..745bf3dffb 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 19 expression: desc --- diff --git a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap similarity index 91% rename from crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap rename to crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap index 131899676a..0629b549a6 100644 --- a/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap +++ b/crates/dbx-sql-dialect/src/sql_dialect/snapshots/dbx_sql_dialect__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql-dialect/src/sql_dialect/descriptor_snapshots.rs assertion_line: 25 expression: desc --- diff --git a/crates/dbx-sql/src/sql_dialect/table_select.rs b/crates/dbx-sql-dialect/src/sql_dialect/table_select.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/table_select.rs rename to crates/dbx-sql-dialect/src/sql_dialect/table_select.rs diff --git a/crates/dbx-sql/src/sql_dialect/tests.rs b/crates/dbx-sql-dialect/src/sql_dialect/tests.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/tests.rs rename to crates/dbx-sql-dialect/src/sql_dialect/tests.rs diff --git a/crates/dbx-sql/src/sql_dialect/type_rewrite.rs b/crates/dbx-sql-dialect/src/sql_dialect/type_rewrite.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/type_rewrite.rs rename to crates/dbx-sql-dialect/src/sql_dialect/type_rewrite.rs diff --git a/crates/dbx-sql/src/sql_dialect/types.rs b/crates/dbx-sql-dialect/src/sql_dialect/types.rs similarity index 100% rename from crates/dbx-sql/src/sql_dialect/types.rs rename to crates/dbx-sql-dialect/src/sql_dialect/types.rs diff --git a/crates/dbx-sql-schema/Cargo.toml b/crates/dbx-sql-schema/Cargo.toml new file mode 100644 index 0000000000..647fd2fb3c --- /dev/null +++ b/crates/dbx-sql-schema/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "dbx-sql-schema" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +duckdb-sidecar = [] + +[dependencies] +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-sql-dialect = { path = "../dbx-sql-dialect" } +dbx-types = { path = "../dbx-types", default-features = false } +log = "0.4" +rayon = "1" +regex = "1" +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +serde_yaml = "0.9" +sqlparser = { version = "0.62.0", features = ["visitor"] } + +[dev-dependencies] +rusqlite = { version = "0.32", default-features = false } diff --git a/crates/dbx-sql/src/db_admin_sql.rs b/crates/dbx-sql-schema/src/db_admin_sql.rs similarity index 100% rename from crates/dbx-sql/src/db_admin_sql.rs rename to crates/dbx-sql-schema/src/db_admin_sql.rs diff --git a/crates/dbx-sql-schema/src/lib.rs b/crates/dbx-sql-schema/src/lib.rs new file mode 100644 index 0000000000..c364c8138f --- /dev/null +++ b/crates/dbx-sql-schema/src/lib.rs @@ -0,0 +1,10 @@ +#![recursion_limit = "256"] + +pub use dbx_sql_core::sql; +pub use dbx_sql_dialect::sql_dialect; +pub use dbx_types::{database_manifest, models, types}; + +pub mod db_admin_sql; +pub mod schema_diff; +pub mod sql_parser; +pub mod table_structure_sql; diff --git a/crates/dbx-sql/src/schema_diff.rs b/crates/dbx-sql-schema/src/schema_diff.rs similarity index 100% rename from crates/dbx-sql/src/schema_diff.rs rename to crates/dbx-sql-schema/src/schema_diff.rs diff --git a/crates/dbx-sql/src/schema_diff/sqlserver_dependencies.rs b/crates/dbx-sql-schema/src/schema_diff/sqlserver_dependencies.rs similarity index 100% rename from crates/dbx-sql/src/schema_diff/sqlserver_dependencies.rs rename to crates/dbx-sql-schema/src/schema_diff/sqlserver_dependencies.rs diff --git a/crates/dbx-sql/src/sql_parser/ast_filter.rs b/crates/dbx-sql-schema/src/sql_parser/ast_filter.rs similarity index 100% rename from crates/dbx-sql/src/sql_parser/ast_filter.rs rename to crates/dbx-sql-schema/src/sql_parser/ast_filter.rs diff --git a/crates/dbx-sql/src/sql_parser/git.rs b/crates/dbx-sql-schema/src/sql_parser/git.rs similarity index 100% rename from crates/dbx-sql/src/sql_parser/git.rs rename to crates/dbx-sql-schema/src/sql_parser/git.rs diff --git a/crates/dbx-sql/src/sql_parser/input.rs b/crates/dbx-sql-schema/src/sql_parser/input.rs similarity index 100% rename from crates/dbx-sql/src/sql_parser/input.rs rename to crates/dbx-sql-schema/src/sql_parser/input.rs diff --git a/crates/dbx-sql/src/sql_parser/meta.rs b/crates/dbx-sql-schema/src/sql_parser/meta.rs similarity index 100% rename from crates/dbx-sql/src/sql_parser/meta.rs rename to crates/dbx-sql-schema/src/sql_parser/meta.rs diff --git a/crates/dbx-sql/src/sql_parser/mod.rs b/crates/dbx-sql-schema/src/sql_parser/mod.rs similarity index 100% rename from crates/dbx-sql/src/sql_parser/mod.rs rename to crates/dbx-sql-schema/src/sql_parser/mod.rs diff --git a/crates/dbx-sql/src/table_structure_sql.rs b/crates/dbx-sql-schema/src/table_structure_sql.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql.rs rename to crates/dbx-sql-schema/src/table_structure_sql.rs diff --git a/crates/dbx-sql/src/table_structure_sql/column_alter.rs b/crates/dbx-sql-schema/src/table_structure_sql/column_alter.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/column_alter.rs rename to crates/dbx-sql-schema/src/table_structure_sql/column_alter.rs diff --git a/crates/dbx-sql/src/table_structure_sql/column_format.rs b/crates/dbx-sql-schema/src/table_structure_sql/column_format.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/column_format.rs rename to crates/dbx-sql-schema/src/table_structure_sql/column_format.rs diff --git a/crates/dbx-sql/src/table_structure_sql/columns.rs b/crates/dbx-sql-schema/src/table_structure_sql/columns.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/columns.rs rename to crates/dbx-sql-schema/src/table_structure_sql/columns.rs diff --git a/crates/dbx-sql/src/table_structure_sql/comments.rs b/crates/dbx-sql-schema/src/table_structure_sql/comments.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/comments.rs rename to crates/dbx-sql-schema/src/table_structure_sql/comments.rs diff --git a/crates/dbx-sql/src/table_structure_sql/create_table.rs b/crates/dbx-sql-schema/src/table_structure_sql/create_table.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/create_table.rs rename to crates/dbx-sql-schema/src/table_structure_sql/create_table.rs diff --git a/crates/dbx-sql/src/table_structure_sql/dialect.rs b/crates/dbx-sql-schema/src/table_structure_sql/dialect.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/dialect.rs rename to crates/dbx-sql-schema/src/table_structure_sql/dialect.rs diff --git a/crates/dbx-sql/src/table_structure_sql/foreign_keys.rs b/crates/dbx-sql-schema/src/table_structure_sql/foreign_keys.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/foreign_keys.rs rename to crates/dbx-sql-schema/src/table_structure_sql/foreign_keys.rs diff --git a/crates/dbx-sql/src/table_structure_sql/indexes.rs b/crates/dbx-sql-schema/src/table_structure_sql/indexes.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/indexes.rs rename to crates/dbx-sql-schema/src/table_structure_sql/indexes.rs diff --git a/crates/dbx-sql/src/table_structure_sql/mysql_engine.rs b/crates/dbx-sql-schema/src/table_structure_sql/mysql_engine.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/mysql_engine.rs rename to crates/dbx-sql-schema/src/table_structure_sql/mysql_engine.rs diff --git a/crates/dbx-sql/src/table_structure_sql/owner.rs b/crates/dbx-sql-schema/src/table_structure_sql/owner.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/owner.rs rename to crates/dbx-sql-schema/src/table_structure_sql/owner.rs diff --git a/crates/dbx-sql/src/table_structure_sql/partitions.rs b/crates/dbx-sql-schema/src/table_structure_sql/partitions.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/partitions.rs rename to crates/dbx-sql-schema/src/table_structure_sql/partitions.rs diff --git a/crates/dbx-sql/src/table_structure_sql/tests.rs b/crates/dbx-sql-schema/src/table_structure_sql/tests.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/tests.rs rename to crates/dbx-sql-schema/src/table_structure_sql/tests.rs diff --git a/crates/dbx-sql/src/table_structure_sql/triggers.rs b/crates/dbx-sql-schema/src/table_structure_sql/triggers.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/triggers.rs rename to crates/dbx-sql-schema/src/table_structure_sql/triggers.rs diff --git a/crates/dbx-sql/src/table_structure_sql/types.rs b/crates/dbx-sql-schema/src/table_structure_sql/types.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/types.rs rename to crates/dbx-sql-schema/src/table_structure_sql/types.rs diff --git a/crates/dbx-sql/src/table_structure_sql/util.rs b/crates/dbx-sql-schema/src/table_structure_sql/util.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/util.rs rename to crates/dbx-sql-schema/src/table_structure_sql/util.rs diff --git a/crates/dbx-sql/src/table_structure_sql/validation.rs b/crates/dbx-sql-schema/src/table_structure_sql/validation.rs similarity index 100% rename from crates/dbx-sql/src/table_structure_sql/validation.rs rename to crates/dbx-sql-schema/src/table_structure_sql/validation.rs diff --git a/crates/dbx-sql/Cargo.toml b/crates/dbx-sql/Cargo.toml index f51cc86e8b..197787a919 100644 --- a/crates/dbx-sql/Cargo.toml +++ b/crates/dbx-sql/Cargo.toml @@ -7,30 +7,12 @@ publish = false [features] default = [] -duckdb-sidecar = [] -openapi = ["dep:utoipa", "dbx-types/openapi"] +duckdb-sidecar = ["dbx-sql-schema/duckdb-sidecar"] +openapi = ["dbx-sql-data/openapi", "dbx-types/openapi"] [dependencies] +dbx-sql-core = { path = "../dbx-sql-core" } +dbx-sql-data = { path = "../dbx-sql-data" } +dbx-sql-dialect = { path = "../dbx-sql-dialect" } +dbx-sql-schema = { path = "../dbx-sql-schema" } dbx-types = { path = "../dbx-types", default-features = false } -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } -utoipa = { version = "5.5", optional = true } -regex = "1" -sqlparser = { version = "0.62.0", features = ["visitor"] } -winnow = "1.0" -rayon = "1" -log = "0.4" -tokio = { version = "1", features = ["sync", "time", "rt", "macros"] } -chrono = { version = "0.4", features = ["serde"] } -percent-encoding = "2" -encoding_rs = "0.8" -unicode-ident = "1" -unicode-width = "0.2" -serde_yaml = "0.9" -notify = { version = "7", default-features = false, features = ["macos_kqueue"] } - -[dev-dependencies] -insta = { version = "1", features = ["json", "glob"] } -tempfile = "3" -rusqlite = { version = "0.32", default-features = false } -tokio = { version = "1", features = ["test-util", "rt-multi-thread"] } diff --git a/crates/dbx-sql/src/lib.rs b/crates/dbx-sql/src/lib.rs index 5c1d1267a5..38f24a6fc2 100644 --- a/crates/dbx-sql/src/lib.rs +++ b/crates/dbx-sql/src/lib.rs @@ -1,28 +1,10 @@ #![recursion_limit = "256"] +pub use dbx_sql_core::{ + dml_preview_sql, mysql_ddl_normalize, mysql_event_sql, object_source_sql, query_execution_sql, sql, sql_analysis, + sql_diagnostics, sql_editability, sql_error_position, sql_risk, sqlserver_temporal, tdsql_mysql, value_literals, +}; +pub use dbx_sql_data::{data_grid_extractors, data_grid_sql, database_search_sql, query_result_sql}; +pub use dbx_sql_dialect::{dml_binding, sql_dialect}; +pub use dbx_sql_schema::{db_admin_sql, schema_diff, sql_parser, table_structure_sql}; pub use dbx_types::{database_manifest, models, types}; - -pub mod data_grid_extractors; -pub mod data_grid_sql; -pub mod database_search_sql; -pub mod db_admin_sql; -pub mod dml_binding; -pub mod dml_preview_sql; -pub mod mysql_ddl_normalize; -pub mod mysql_event_sql; -pub mod object_source_sql; -pub mod query_execution_sql; -pub mod query_result_sql; -pub mod schema_diff; -pub mod sql; -pub mod sql_analysis; -pub mod sql_diagnostics; -pub mod sql_dialect; -pub mod sql_editability; -pub mod sql_error_position; -pub mod sql_parser; -pub mod sql_risk; -pub mod sqlserver_temporal; -pub mod table_structure_sql; -pub mod tdsql_mysql; -pub mod value_literals; diff --git a/deploy/Dockerfile b/deploy/Dockerfile index b3295bc36e..2f72f448ac 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -38,9 +38,21 @@ COPY crates/dbx-core/Cargo.toml crates/dbx-core/ COPY crates/dbx-platform/Cargo.toml crates/dbx-platform/ COPY crates/dbx-types/Cargo.toml crates/dbx-types/ COPY crates/dbx-sql/Cargo.toml crates/dbx-sql/ +COPY crates/dbx-sql-core/Cargo.toml crates/dbx-sql-core/ +COPY crates/dbx-sql-data/Cargo.toml crates/dbx-sql-data/ +COPY crates/dbx-sql-dialect/Cargo.toml crates/dbx-sql-dialect/ +COPY crates/dbx-sql-schema/Cargo.toml crates/dbx-sql-schema/ COPY crates/dbx-formats/Cargo.toml crates/dbx-formats/ COPY crates/dbx-ai-provider/Cargo.toml crates/dbx-ai-provider/ COPY crates/dbx-plugin-runtime/Cargo.toml crates/dbx-plugin-runtime/ +COPY crates/dbx-driver-agent/Cargo.toml crates/dbx-driver-agent/ +COPY crates/dbx-driver-support/Cargo.toml crates/dbx-driver-support/ +COPY crates/dbx-driver-elasticsearch/Cargo.toml crates/dbx-driver-elasticsearch/ +COPY crates/dbx-driver-mongodb/Cargo.toml crates/dbx-driver-mongodb/ +COPY crates/dbx-driver-mysql/Cargo.toml crates/dbx-driver-mysql/ +COPY crates/dbx-driver-postgres/Cargo.toml crates/dbx-driver-postgres/ +COPY crates/dbx-driver-redis/Cargo.toml crates/dbx-driver-redis/ +COPY crates/dbx-driver-sqlserver/Cargo.toml crates/dbx-driver-sqlserver/ COPY crates/dbx-drivers/Cargo.toml crates/dbx-drivers/ COPY crates/dbx-sqlite-worker/Cargo.toml crates/dbx-sqlite-worker/ COPY crates/dbx-web/Cargo.toml crates/dbx-web/ @@ -50,7 +62,10 @@ COPY src-tauri/Cargo.toml src-tauri/ # Copy every vendored crate: root Cargo.toml [patch.crates-io] points at # vendor/{ctor,rumqttc,dirs-sys,pageant,wry}; omitting any breaks the offline build. COPY vendor/ vendor/ -RUN for crate in dbx-platform dbx-types dbx-sql dbx-formats dbx-ai-provider dbx-plugin-runtime dbx-drivers dbx-sqlite-worker; do \ +RUN for crate in dbx-platform dbx-types dbx-sql dbx-sql-core dbx-sql-data dbx-sql-dialect dbx-sql-schema \ + dbx-formats dbx-ai-provider dbx-plugin-runtime dbx-driver-agent dbx-driver-support \ + dbx-driver-elasticsearch dbx-driver-mongodb dbx-driver-mysql dbx-driver-postgres \ + dbx-driver-redis dbx-driver-sqlserver dbx-drivers dbx-sqlite-worker; do \ mkdir -p "crates/$crate/src" && printf "" > "crates/$crate/src/lib.rs"; \ done \ && echo 'fn main() {}' > crates/dbx-sqlite-worker/src/main.rs @@ -76,7 +91,7 @@ RUN case "$TARGETARCH" in \ # Copy real sources and rebuild (only application code recompiles) COPY crates/ crates/ -# dbx-sql/build.rs embeds plugins/dialects; dbx-types/build.rs embeds +# dbx-sql-dialect/build.rs embeds plugins/dialects; dbx-types/build.rs embeds # plugins/connection-types (database manifest + DatabaseType enum); # either directory missing makes the build script panic. COPY plugins/dialects/ plugins/dialects/ diff --git a/docs/salesforce-soql-integration-spec.md b/docs/salesforce-soql-integration-spec.md index bb6b137ca3..9c026e78db 100644 --- a/docs/salesforce-soql-integration-spec.md +++ b/docs/salesforce-soql-integration-spec.md @@ -30,7 +30,7 @@ | # | 方式 | 模式 | 说明 | |---|------|------|------| -| 1 | OAuth Authorization Code + PKCE | 桌面 | 主路径。复用 `crates/dbx-drivers/src/mongo_oidc.rs` 的模式：本地 `TcpListener` 监听回调 → 系统浏览器授权 → code 换 token。Connected App 需配置 `refresh_token` scope 以获取长期令牌 | +| 1 | OAuth Authorization Code + PKCE | 桌面 | 主路径。复用 `crates/dbx-driver-mongodb/src/mongo_oidc.rs` 的模式：本地 `TcpListener` 监听回调 → 系统浏览器授权 → code 换 token。Connected App 需配置 `refresh_token` scope 以获取长期令牌 | | 2 | OAuth Device Flow | 桌面 + **web** | `grant_type=urn:ietf:params:oauth:grant-type:device_code`。UI 显示 user code，用户在任意设备浏览器输入。**这是 web/Docker 模式下唯一的浏览器授权方式** | | 3 | 粘贴 Access Token + instance_url | 桌面 + web | 调试/CI/兜底。Session ID 可直接作 Bearer token。无 refresh 能力，过期后需重新粘贴 | | 4 | Username-Password OAuth | 桌面 + web | ⚠️ Salesforce 已弃用（新 Connected App 无法启用）。仅作为 BYO Connected App 的兼容项，UI 上明确标注 deprecated；密码需拼接 security token | @@ -249,7 +249,7 @@ capabilities 覆盖（MVP）： - prepare 时以驱动的 `parse_salesforce_statement` 为权威校验，保证不会为 org 必然拒绝的语句发令牌；`upsert`/bulk/composite 一律 `SALESFORCE_DML_INVALID`（一次只碰一条记录）。 - 身份查询是 best-effort：失败只在摘要里写「身份未知」，仍然完成 prepare，把是否 apply 的决定权留给人。 -风险分级（`crates/dbx-sql/src/query_execution_sql.rs`，fail-closed）：SOQL → `Read`；带 Id 的单记录 DML → `ScopedWrite`（映射 `SqlRisk::Write`，只需 safe_write）；无 Id / 无法判定 → `OpaqueWrite`（映射 `SqlRisk::Ddl`，需 allow_dangerous_sql）。 +风险分级（`crates/dbx-sql-core/src/query_execution_sql.rs`，fail-closed）：SOQL → `Read`；带 Id 的单记录 DML → `ScopedWrite`（映射 `SqlRisk::Write`，只需 safe_write）；无 Id / 无法判定 → `OpaqueWrite`（映射 `SqlRisk::Ddl`，需 allow_dangerous_sql）。 前端开关位置：**设置 → MCP → 连接范围**，仅对 `db_type === "salesforce"` 的连接行渲染（`McpResourceScopePicker.vue`），文案 `settings.mcpConnectionPolicyAllowSalesforceDml`；有效执行模式为只读时点击只提示不保存。工具白名单里三个新工具可独立勾选，因此可以只给 agent 读能力（`current_user` + SOQL）而不给写通道。 diff --git a/packages/app-tests/sqlserverResultFocus6189.test.ts b/packages/app-tests/sqlserverResultFocus6189.test.ts index e66dd256e3..6231a89beb 100644 --- a/packages/app-tests/sqlserverResultFocus6189.test.ts +++ b/packages/app-tests/sqlserverResultFocus6189.test.ts @@ -51,7 +51,7 @@ function sqlServerConn(id: string): ConnectionConfig { const THREE_RESULTS = "SELECT 1 AS a; SELECT 2 AS b; SELECT 3 AS c;"; const TRAILING_MESSAGE = "SELECT 9 AS z; PRINT N'DBCC execution completed.';"; -/** Mirrors the shapes crates/dbx-drivers/src/db/sqlserver.rs:523-542 actually produces. */ +/** Mirrors the shapes crates/dbx-driver-sqlserver/src/sqlserver.rs actually produces. */ function resultsFor(sql: string) { if (sql.includes("SELECT 3 AS c")) { return [ diff --git a/scripts/core-architecture.test.mjs b/scripts/core-architecture.test.mjs index 42d2856316..4369d95136 100644 --- a/scripts/core-architecture.test.mjs +++ b/scripts/core-architecture.test.mjs @@ -17,10 +17,26 @@ const allowedDependencies = { "dbx-types": [], "dbx-platform": [], "dbx-formats": [], - "dbx-sql": ["dbx-types"], + "dbx-sql-core": ["dbx-types"], + "dbx-sql-dialect": ["dbx-types"], + "dbx-sql-data": ["dbx-sql-core", "dbx-sql-dialect", "dbx-types"], + "dbx-sql-schema": ["dbx-sql-core", "dbx-sql-dialect", "dbx-types"], + "dbx-sql": ["dbx-sql-core", "dbx-sql-data", "dbx-sql-dialect", "dbx-sql-schema", "dbx-types"], "dbx-ai-provider": ["dbx-platform"], "dbx-plugin-runtime": ["dbx-types", "dbx-platform"], - "dbx-drivers": ["dbx-types", "dbx-sql", "dbx-platform", "dbx-sqlite-worker"], + "dbx-driver-support": ["dbx-types", "dbx-platform"], + "dbx-driver-agent": ["dbx-types", "dbx-sql-core", "dbx-platform"], + "dbx-driver-elasticsearch": ["dbx-types", "dbx-driver-support"], + "dbx-driver-mongodb": ["dbx-types", "dbx-driver-support"], + "dbx-driver-mysql": ["dbx-types", "dbx-sql-core", "dbx-sql-dialect", "dbx-driver-support"], + "dbx-driver-postgres": ["dbx-types", "dbx-sql-core", "dbx-driver-support"], + "dbx-driver-redis": ["dbx-types", "dbx-driver-support"], + "dbx-driver-sqlserver": ["dbx-types", "dbx-sql-core", "dbx-sql-data", "dbx-driver-support"], + "dbx-drivers": [ + "dbx-types", "dbx-sql-core", "dbx-sql-data", "dbx-sql-dialect", "dbx-platform", "dbx-sqlite-worker", + "dbx-driver-support", "dbx-driver-agent", "dbx-driver-elasticsearch", "dbx-driver-mongodb", + "dbx-driver-mysql", "dbx-driver-postgres", "dbx-driver-redis", "dbx-driver-sqlserver", + ], }; const core = members.get("dbx-core"); @@ -78,11 +94,11 @@ test("core forwards capability flags without enabling driver defaults or test ho test("code generators and runtime assets belong to their implementation crates", () => { assert.match(read("crates/dbx-types/build.rs"), /plugins\/connection-types/); - assert.match(read("crates/dbx-sql/build.rs"), /plugins\/dialects/); + assert.match(read("crates/dbx-sql-dialect/build.rs"), /plugins\/dialects/); assert.match(read("crates/dbx-ai-provider/src/ai_pi_agent_cli.rs"), /assets\/pi-mcp-bridge\.mjs/); assert.ok(existsSync(path.join(root, "crates/dbx-ai-provider/assets/pi-mcp-bridge.mjs"))); assert.deepEqual( - JSON.parse(read("crates/dbx-drivers/assets/agent-protocol-v2.json")), + JSON.parse(read("crates/dbx-driver-agent/assets/agent-protocol-v2.json")), JSON.parse(read("agents/common/src/main/resources/agent-protocol-v2.json")), ); assert.match(read("crates/dbx-platform/src/lib.rs"), /#\[cfg\(all\(target_os = "windows", target_env = "gnu"\)\)\]\s*mod nanosleep_stub/); @@ -106,10 +122,10 @@ test("CI and release tracking follow the new source owners", () => { const ci = read(".github/workflows/ci.yml"); assert.ok(ci.includes("node --test scripts/core-architecture.test.mjs")); assert.ok(ci.includes("'crates/**'")); - for (const input of ["plugins/dialects/**", "plugins/connection-types/**", "crates/dbx-drivers/assets/agent-protocol-v2.json"]) { + for (const input of ["plugins/dialects/**", "plugins/connection-types/**", "crates/dbx-driver-agent/assets/agent-protocol-v2.json"]) { assert.ok(ci.includes(`'${input}'`), `CI misses ${input}`); } - assert.ok(read("scripts/release.mjs").includes('"crates/dbx-drivers/src/mongo_shell.rs"')); + assert.ok(read("scripts/release.mjs").includes('"crates/dbx-driver-mongodb/src/mongo_shell.rs"')); }); test("the standalone DuckDB lockfile includes core's internal dependency closure", () => { diff --git a/scripts/release.mjs b/scripts/release.mjs index 21ec5591a9..57ad6e2902 100644 --- a/scripts/release.mjs +++ b/scripts/release.mjs @@ -42,7 +42,7 @@ const PACKAGE_RELEASE_PATHS = [ "packages/mcp-win32-x64/", "crates/dbx-mcp/", "crates/dbx-cli/", - "crates/dbx-drivers/src/mongo_shell.rs", + "crates/dbx-driver-mongodb/src/mongo_shell.rs", "Cargo.toml", "Cargo.lock", ".github/workflows/mcp-release.yml",