diff --git a/.github/scripts/label-pull-request.mjs b/.github/scripts/label-pull-request.mjs index 6e09acec0e..b0e4da48f3 100644 --- a/.github/scripts/label-pull-request.mjs +++ b/.github/scripts/label-pull-request.mjs @@ -30,6 +30,8 @@ export const STATIC_LABEL_SPECS = { "tests-only": { color: "bfd4f2", description: "Changes only tests, fixtures, or snapshots" }, }; +const CORE_CRATE_PATH = /^crates\/dbx-(?:core|drivers|sql|formats|types|platform|ai-provider|plugin-runtime)\//; + const TYPE_LABELS = new Set(["bug", "enhancement", "documentation", "maintenance"]); const LABEL_PALETTE = [ "0e8a16", @@ -133,7 +135,7 @@ export function inferAreaLabels(changedFiles) { || file.startsWith("src-tauri/"))) { labels.add("area/desktop"); } - if (has((file) => file.startsWith("crates/dbx-core/") || file.startsWith("packages/node-core/"))) { + if (has((file) => CORE_CRATE_PATH.test(file) || file.startsWith("packages/node-core/"))) { labels.add("area/core"); } if (has((file) => file.startsWith("crates/dbx-web/") || file.startsWith("examples/web-api/"))) { @@ -220,7 +222,7 @@ export function inferDatabaseTypes(changedFiles, knownDatabaseTypes) { if (file.startsWith("plugins/jdbc/")) add("jdbc"); - if (file.startsWith("apps/desktop/src/") || file.startsWith("crates/dbx-core/src/")) { + if (file.startsWith("apps/desktop/src/") || CORE_CRATE_PATH.test(file)) { for (const [pattern, databaseType] of DATABASE_PATH_TOKENS) { if (pattern.test(file)) add(databaseType); } diff --git a/.github/scripts/label-pull-request.test.mjs b/.github/scripts/label-pull-request.test.mjs index ba7f372ab6..ed340f1fd3 100644 --- a/.github/scripts/label-pull-request.test.mjs +++ b/.github/scripts/label-pull-request.test.mjs @@ -104,13 +104,32 @@ test("recognizes conventional titles with a full-width colon", () => { assert.deepEqual(result.labels, ["area/core", "bug"]); }); +test("recognizes extracted core crates and their database sources", () => { + const result = evaluatePullRequestLabels({ + title: "fix(sql): preserve PostgreSQL query contracts", + changedFiles: [ + "crates/dbx-drivers/src/db/postgres.rs", + "crates/dbx-sql/src/sql.rs", + "crates/dbx-types/src/types.rs", + "crates/dbx-platform/src/process.rs", + "crates/dbx-ai-provider/src/ai.rs", + "crates/dbx-formats/src/csv_export.rs", + "crates/dbx-plugin-runtime/src/plugins.rs", + ], + knownDatabaseTypes, + }); + + assert.deepEqual(result.labels, ["area/core", "bug", "db/postgres"]); + assert.deepEqual(result.databaseTypes, ["postgres"]); +}); + test("collapses broad area and database changes", () => { const result = evaluatePullRequestLabels({ title: "feat: add cross-runtime geometry support", changedFiles: [ "agents/drivers/mysql/build.gradle", "apps/desktop/src/components/grid/GeometryViewer.vue", - "crates/dbx-core/src/db/postgres.rs", + "crates/dbx-drivers/src/db/postgres.rs", "crates/dbx-mcp/src/main.rs", "crates/dbx-web/src/main.rs", "docs/content/docs/geometry.mdx", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 384c613cea..8bfaad5b65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,6 +324,9 @@ jobs: # PR caches are large and branch-scoped; restore them from main without saving per-PR copies. save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Core architecture contracts + run: node --test scripts/core-architecture.test.mjs + - name: Cargo fmt check run: cargo fmt --check @@ -555,7 +558,13 @@ jobs: - 'package.json' - '.oxfmtrc.json' - 'scripts/run-check.mjs' - - 'crates/dbx-core/src/docs/**' + - 'crates/dbx-core/src/**' + - 'crates/dbx-drivers/src/**' + - 'crates/dbx-sql/src/**' + - 'crates/dbx-formats/src/**' + - 'crates/dbx-types/src/**' + - 'crates/dbx-plugin-runtime/src/**' + - 'crates/dbx-ai-provider/**' - 'crates/dbx-core/assets/docs-export.*' - '.github/workflows/ci.yml' packages: @@ -571,6 +580,9 @@ jobs: - '.github/workflows/ci.yml' rust: - 'crates/**' + - 'plugins/connection-types/**' + - 'plugins/dialects/**' + - 'scripts/core-architecture.test.mjs' - 'src-tauri/**' - 'vendor/**' - 'Cargo.toml' @@ -590,6 +602,7 @@ jobs: - '.github/workflows/ci.yml' agents: - 'agents/**' + - 'crates/dbx-drivers/assets/agent-protocol-v2.json' - 'crates/dbx-core/Cargo.toml' - 'Cargo.toml' - 'Cargo.lock' @@ -631,8 +644,9 @@ jobs: - 'src-tauri/windows/nsis/**' - 'src-tauri/src/commands/update.rs' - 'crates/dbx-core/Cargo.toml' - - 'crates/dbx-core/src/db/postgres.rs' - - 'crates/dbx-core/src/update.rs' + - 'crates/dbx-drivers/**' + - 'crates/dbx-platform/**' + - 'crates/dbx-core/src/host/update.rs' - 'Cargo.toml' - 'Cargo.lock' - 'vendor/ctor/**' diff --git a/Cargo.lock b/Cargo.lock index f2f6d50399..4ceca174ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2076,6 +2076,24 @@ dependencies = [ "zip", ] +[[package]] +name = "dbx-ai-provider" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "bytes", + "dbx-platform", + "futures", + "httpdate", + "log", + "reqwest 0.12.28", + "serde", + "serde_json", + "tempfile", + "tokio", + "uuid", +] + [[package]] name = "dbx-cli" version = "0.4.90" @@ -2096,11 +2114,9 @@ version = "0.1.0" dependencies = [ "aes 0.8.4", "aes-gcm 0.10.3", - "anyhow", "arc-swap", "argon2", "async-trait", - "aws-sdk-dynamodb", "base64 0.22.1", "bytes", "calamine", @@ -2108,31 +2124,27 @@ dependencies = [ "chrono", "crc", "csv", - "dbx-sqlite-worker", + "dbx-ai-provider", + "dbx-drivers", + "dbx-formats", + "dbx-platform", + "dbx-plugin-runtime", + "dbx-sql", + "dbx-types", "deadpool-postgres", - "ed25519-dalek 2.2.0", "encoding_rs", "flate2", "font-kit", - "fs2", "futures", - "httpdate", - "insta", - "json5", "jsonwebtoken", - "libsqlite3-hotbundle", "log", "minijinja", "mongodb", "mysql_async", - "mysql_common", "nix", "notify", - "openssl", - "pageant", "percent-encoding", "portpicker", - "postgres-openssl", "prometheus-parse", "quick-xml 0.37.5", "rayon", @@ -2141,14 +2153,10 @@ dependencies = [ "reqwest 0.12.28", "rumqttc", "rusqlite", - "russh", - "rust_decimal", "rustls 0.23.43", "rustls-pemfile 2.2.0", - "semver", "serde", "serde_json", - "serde_yaml", "serde_yaml_ng", "sha1 0.10.7", "sha2 0.10.9", @@ -2157,6 +2165,59 @@ dependencies = [ "sysinfo", "tar", "tempfile", + "tokio", + "tokio-postgres", + "tokio-util", + "uuid", + "webpki-roots 0.26.11", + "windows-sys 0.61.2", + "zip", + "zstd", +] + +[[package]] +name = "dbx-drivers" +version = "0.1.0" +dependencies = [ + "aws-sdk-dynamodb", + "base64 0.22.1", + "bytes", + "chrono", + "crc", + "csv", + "dbx-platform", + "dbx-sql", + "dbx-sqlite-worker", + "dbx-types", + "deadpool-postgres", + "flate2", + "futures", + "json5", + "libsqlite3-hotbundle", + "log", + "mongodb", + "mysql_async", + "mysql_common", + "nix", + "openssl", + "pageant", + "percent-encoding", + "portpicker", + "postgres-openssl", + "redis", + "regex", + "reqwest 0.12.28", + "rusqlite", + "russh", + "rust_decimal", + "rustls 0.23.43", + "rustls-pemfile 2.2.0", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlparser", + "tar", + "tempfile", "tiberius", "tokio", "tokio-openssl", @@ -2165,17 +2226,26 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", - "unicode-ident", - "unicode-width", - "utoipa", "uuid", "webpki-roots 0.26.11", "windows-sys 0.61.2", - "winnow 1.0.4", "zip", "zstd", ] +[[package]] +name = "dbx-formats" +version = "0.1.0" +dependencies = [ + "calamine", + "chrono", + "serde", + "serde_json", + "tempfile", + "uuid", + "zip", +] + [[package]] name = "dbx-mcp" version = "0.4.90" @@ -2201,6 +2271,68 @@ dependencies = [ "uuid", ] +[[package]] +name = "dbx-platform" +version = "0.1.0" +dependencies = [ + "futures", + "nix", + "reqwest 0.12.28", + "serde", + "tokio", + "uuid", +] + +[[package]] +name = "dbx-plugin-runtime" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "dbx-platform", + "dbx-types", + "ed25519-dalek 2.2.0", + "fs2", + "futures", + "log", + "reqwest 0.12.28", + "semver", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "uuid", + "zip", +] + +[[package]] +name = "dbx-sql" +version = "0.1.0" +dependencies = [ + "chrono", + "dbx-types", + "encoding_rs", + "insta", + "log", + "notify", + "percent-encoding", + "rayon", + "regex", + "rusqlite", + "serde", + "serde_json", + "serde_yaml", + "sqlparser", + "tempfile", + "tokio", + "unicode-ident", + "unicode-width", + "utoipa", + "winnow 1.0.4", +] + [[package]] name = "dbx-sqlite-worker" version = "0.1.1" @@ -2211,6 +2343,18 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "dbx-types" +version = "0.1.0" +dependencies = [ + "percent-encoding", + "serde", + "serde_json", + "serde_yaml_ng", + "utoipa", + "uuid", +] + [[package]] name = "dbx-web" version = "0.6.16" diff --git a/Cargo.toml b/Cargo.toml index b53cf69841..d48456cda9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["src-tauri", "crates/dbx-core", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"] +members = ["src-tauri", "crates/dbx-core", "crates/dbx-drivers", "crates/dbx-sql", "crates/dbx-plugin-runtime", "crates/dbx-formats", "crates/dbx-types", "crates/dbx-platform", "crates/dbx-ai-provider", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"] exclude = ["vendor/tauri-plugin-updater", "vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/tiberius", "vendor/wry"] [patch.crates-io] diff --git a/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java b/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java index 8c171d4780..8a9c658eed 100644 --- a/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java +++ b/agents/common/src/main/java/com/dbx/agent/EwkbWktDecoder.java @@ -7,7 +7,7 @@ import java.util.List; /** * Decode PostGIS EWKB / WKB byte streams (and PGgeometry / hex strings) into * OGC Well-Known Text. Behaves identically to the Rust implementation in - * {@code crates/dbx-core/src/db/postgres.rs::ewkb_to_wkt}, including the + * {@code crates/dbx-drivers/src/db/postgres.rs::ewkb_to_wkt}, including the * {@code Z} / {@code M} / {@code ZM} dimension suffixes and {@code EMPTY} * geometries. Used by the JDBC agent layer so that PostgresLike databases * such as HighGo / KingBase / Vastbase / openGauss / GaussDB return WKT for diff --git a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java index e0dbb313f5..8e1e54b155 100644 --- a/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java +++ b/agents/common/src/main/java/com/dbx/agent/PostgresLikeAgent.java @@ -79,7 +79,7 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent { /** * Wrap {@link AbstractJdbcAgent#resultValue} so PostGIS-style {@code geometry} * and {@code geography} columns are decoded into WKT (matching the native - * tokio_postgres path in {@code crates/dbx-core/src/db/postgres.rs}). + * tokio_postgres path in {@code crates/dbx-drivers/src/db/postgres.rs}). */ private JdbcExecutor.ColumnAwareResultValueReader geometryAwareResolver() { return (rs, index, sqlType, columnTypeName) -> { diff --git a/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java b/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java index 1d460d5616..7fdf74c3fd 100644 --- a/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java +++ b/agents/common/src/test/java/com/dbx/agent/EwkbWktDecoderTest.java @@ -22,7 +22,7 @@ class EwkbWktDecoderTest { // ────────────────────────────────────────────────────────────────────── // Cross-checked against the Rust ewkb_to_wkt tests in - // crates/dbx-core/src/db/postgres.rs (same hex inputs, same expected WKT). + // crates/dbx-drivers/src/db/postgres.rs (same hex inputs, same expected WKT). // ────────────────────────────────────────────────────────────────────── @Test diff --git a/apps/desktop/src/components/editor/EditorSettingsDialog.vue b/apps/desktop/src/components/editor/EditorSettingsDialog.vue index c0a2800178..ecd0bb12fa 100644 --- a/apps/desktop/src/components/editor/EditorSettingsDialog.vue +++ b/apps/desktop/src/components/editor/EditorSettingsDialog.vue @@ -4413,7 +4413,7 @@ function globalInstructionsTooLong(): boolean { } // Agent turn limit for DBX's API-backed agent loop. CLI providers enforce their own limits. -// Mirrors DEFAULT/MIN/MAX_MAX_AGENT_TURNS in crates/dbx-core/src/agent_loop.rs — +// Mirrors DEFAULT/MIN/MAX_MAX_AGENT_TURNS in crates/dbx-core/src/ai/agent_loop.rs — // keep in sync; the backend clamp on save/load is the actual source of truth. const editMaxAgentTurns = ref(undefined); const maxAgentTurnsSaving = ref(false); diff --git a/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts b/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts index a847bcbb38..91bdc7a775 100644 --- a/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts +++ b/apps/desktop/src/components/redis/RedisKeyBrowser.infiniteScroll.spec.ts @@ -1001,7 +1001,7 @@ describe("RedisKeyBrowser continuation ownership (issue #7779)", () => { // `iterations` (aka `max_iterations`) is the 6th positional arg the frontend // sends to `redisScanKeysBatch` — the same unit the backend spends as real -// Redis SCAN calls (see `crates/dbx-core/src/db/redis_driver.rs`). +// Redis SCAN calls (see `crates/dbx-drivers/src/db/redis_driver.rs`). function totalIterationsRequested(): number { return mocks.redisScanKeysBatch.mock.calls.reduce((sum: number, call: unknown[]) => sum + (call[5] as number), 0); } diff --git a/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts b/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts index bbc7a67840..23f999fe4d 100644 --- a/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts +++ b/apps/desktop/src/composables/__tests__/useSidebarTableMutationRuntime.saphana.spec.ts @@ -220,7 +220,7 @@ describe("useSidebarTableMutationRuntime SAP HANA schema-scoped actions", () => it("classifies a JDBC-agent-routed timeout (SAP HANA/Oracle/DB2/SQL Server) as timed-out, not failed, even though its message has no timeout wording", async () => { const { feature } = runtime(""); - // AgentCallError::Timeout carries no `detail` (crates/dbx-core/src/backend_error.rs), + // AgentCallError::Timeout carries no `detail` (crates/dbx-drivers/src/backend_error.rs), // so BackendErrorException.message degrades to a generic fallback with no // "timed out" text — only the structured backendError distinguishes it. const agentTimeoutError = { diff --git a/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts b/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts index a054ef4d79..7023502aea 100644 --- a/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts +++ b/apps/desktop/src/composables/__tests__/useSqlExecutionMultiDbFocus.spec.ts @@ -2,7 +2,7 @@ * Issue #6189 — SQL Server "focus jumps to the message result". * * Background: - * - `crates/dbx-core/src/db/sqlserver.rs:523-542` synthesizes a pseudo result with a + * - `crates/dbx-drivers/src/db/sqlserver.rs:523-542` synthesizes a pseudo result with a * single "Message" column and `server_message: true` for any batch segment that * produced only server messages (PRINT / "DBCC execution completed" / ...). * - `stores/queryStore.ts:4727` then picks the first result that HAS COLUMNS as the diff --git a/apps/desktop/src/docs/__tests__/annotationTypes.spec.ts b/apps/desktop/src/docs/__tests__/annotationTypes.spec.ts index 4a920b6f9c..45beb622e9 100644 --- a/apps/desktop/src/docs/__tests__/annotationTypes.spec.ts +++ b/apps/desktop/src/docs/__tests__/annotationTypes.spec.ts @@ -7,7 +7,7 @@ import { ANNOTATION_FILE_KEYS, COLUMN_ANNOTATION_KEYS, GROUP_ANNOTATION_KEYS, PR // that Rust does not declare turns every save into a deserialization error at // runtime. vue-tsc cannot see across the language boundary, so this reads the // Rust source and compares the field sets directly. -const rustSource = readFileSync(path.resolve(__dirname, "../../../../../crates/dbx-core/src/docs/annotations.rs"), "utf8"); +const rustSource = readFileSync(path.resolve(__dirname, "../../../../../crates/dbx-core/src/data/docs/annotations.rs"), "utf8"); function rustFields(structName: string): string[] { const start = rustSource.indexOf(`pub struct ${structName} {`); diff --git a/apps/desktop/src/docs/types.ts b/apps/desktop/src/docs/types.ts index b1b7573564..89e5a12d76 100644 --- a/apps/desktop/src/docs/types.ts +++ b/apps/desktop/src/docs/types.ts @@ -1,4 +1,4 @@ -// Mirrors crates/dbx-core/src/docs/snapshot.rs. There is no Rust->TS codegen +// Mirrors crates/dbx-core/src/data/docs/snapshot.rs. There is no Rust->TS codegen // in this repo, so this file is maintained by hand — see // __tests__/fixtureConformance.spec.ts, which validates it against a snapshot // generated by the real Rust code and fails if the two drift apart. diff --git a/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts b/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts index e099d5dafb..9dadfb7278 100644 --- a/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts +++ b/apps/desktop/src/i18n/__tests__/backendErrors.spec.ts @@ -46,7 +46,7 @@ const STRUCTURED_BACKEND_ERROR_KEYS = [ "backendErrors.unknown", ] as const; -// Reproduces the exact string crates/dbx-core/src/agent_service.rs builds on +// Reproduces the exact string crates/dbx-drivers/src/agent_service.rs builds on // Windows: `\` line continuations strip the newline plus the following indent. const WINDOWS_JRE_REMOVE_ERROR = [ "Failed to remove the old JRE directory: C:\\dbx\\jre21", @@ -123,7 +123,7 @@ const CASES: { name: string; message: string; key: string; params?: Record { const rust = (path: string) => readFileSync(new URL(`../../../../../${path}`, import.meta.url), "utf8"); test.each([ - ["crates/dbx-core/src/query_result_export.rs", "Streaming export is unsupported for this query. Simplify it or use a supported driver."], - ["crates/dbx-core/src/query_result_export.rs", "Streaming export needs a result-set session, but this driver returned no session_id."], - ["crates/dbx-core/src/mongodb_import_export.rs", "MongoDB Legacy Agent does not support insertMany; upgrade or reinstall the MongoDB Legacy driver"], - ["crates/dbx-core/src/mongodb_import_export.rs", "MongoDB Legacy Agent returned an invalid find cursor"], - ["crates/dbx-core/src/mongo_ops.rs", "MongoDB Legacy Agent rejected "], - ["crates/dbx-core/src/agent_service.rs", "Failed to remove the old JRE directory: "], - ["crates/dbx-core/src/agent_service.rs", "is in use by drivers: "], - ["crates/dbx-core/src/agent_service.rs", "agent-registry.json not found in the ZIP; not a valid offline driver package."], - ["crates/dbx-core/src/mq/adapters/kafka.rs", "Kafka does not support unloading topics"], + ["crates/dbx-core/src/data/query_result_export.rs", "Streaming export is unsupported for this query. Simplify it or use a supported driver."], + ["crates/dbx-core/src/data/query_result_export.rs", "Streaming export needs a result-set session, but this driver returned no session_id."], + ["crates/dbx-core/src/data/mongodb_import_export.rs", "MongoDB Legacy Agent does not support insertMany; upgrade or reinstall the MongoDB Legacy driver"], + ["crates/dbx-core/src/data/mongodb_import_export.rs", "MongoDB Legacy Agent returned an invalid find cursor"], + ["crates/dbx-core/src/query/mongo_ops.rs", "MongoDB Legacy Agent rejected "], + ["crates/dbx-drivers/src/agent_service.rs", "Failed to remove the old JRE directory: "], + ["crates/dbx-drivers/src/agent_service.rs", "is in use by drivers: "], + ["crates/dbx-drivers/src/agent_service.rs", "agent-registry.json not found in the ZIP; not a valid offline driver package."], + ["crates/dbx-core/src/admin/mq/adapters/kafka.rs", "Kafka does not support unloading topics"], ["crates/dbx-web/src/auth.rs", "Please try again in {remaining}s"], ["crates/dbx-web/src/routes/agents.rs", "Close these database connections before updating drivers: "], ["src-tauri/src/commands/agents.rs", "Close these database connections before updating drivers: "], diff --git a/apps/desktop/src/i18n/backend-errors.ts b/apps/desktop/src/i18n/backend-errors.ts index 1be5b5739d..389fe4f771 100644 --- a/apps/desktop/src/i18n/backend-errors.ts +++ b/apps/desktop/src/i18n/backend-errors.ts @@ -128,7 +128,7 @@ const patterns: [RegExp, string][] = [ // SSH keyboard-interactive prompts (for example JumpServer TOTP). [/^(?:SSH layer \d+ failed:\s*)?SSH keyboard-interactive authentication was cancelled$/, "connection.sshTotpCancelled"], - // Query result export limits (crates/dbx-core/src/query_result_export.rs) + // Query result export limits (crates/dbx-core/src/data/query_result_export.rs) [/^Streaming export is unsupported for this query\. Simplify it or use a supported driver\.$/, "exportProgress.streamingUnsupported"], // MongoDB Legacy agent failures: messages raised by the agent arrive wrapped in "Agent RPC // error (): ", and the import path prefixes the row a batch-level failure belongs to. @@ -146,7 +146,7 @@ const patterns: [RegExp, string][] = [ // Legacy bundled DuckDB error kept for compatibility with older backends. [/^The previous DuckDB query is still stopping\. Please try again shortly\.$/, "editor.duckdbDraining"], - // Driver / JRE management (crates/dbx-core/src/agent_service.rs, routes/agents.rs) + // Driver / JRE management (crates/dbx-drivers/src/agent_service.rs, routes/agents.rs) // The Windows variant is multi-line, so it must be tried before the single-line one. [/^Failed to remove the old JRE directory: (.+)\nPossible causes:[\s\S]*\(original error: ([\s\S]+)\)$/, "driverStore.jreDirRemoveFailedWindows"], [/^Failed to remove the old JRE directory: (.+) \(original error: ([\s\S]+)\)$/, "driverStore.jreDirRemoveFailed"], @@ -154,7 +154,7 @@ const patterns: [RegExp, string][] = [ [/^agent-registry\.json not found in the ZIP; not a valid offline driver package\.$/, "driverStore.offlinePackageRegistryMissing"], [/^Close these database connections before updating drivers: (.+)$/, "driverStore.driverUpdateBlocked"], - // Message queues (crates/dbx-core/src/mq/adapters/kafka.rs) + // Message queues (crates/dbx-core/src/admin/mq/adapters/kafka.rs) [/^Kafka does not support unloading topics$/, "mqClients.unloadTopicUnsupportedKafka"], // Filesystem (src-tauri/src/commands/fs_open.rs) diff --git a/apps/desktop/src/lib/README.md b/apps/desktop/src/lib/README.md index ecf854364e..7342e68b6b 100644 --- a/apps/desktop/src/lib/README.md +++ b/apps/desktop/src/lib/README.md @@ -7,7 +7,7 @@ - `app`, `tabs`, `sidebar`, `connection`: shell, navigation, and connection UI state helpers. - `database`, `metadata`, `schema`, `table`: relational database metadata, capabilities, DDL, and table-object helpers. - `sql`, `sql/semantic`, `editor`, `query`, `history`, `savedSql`: SQL editing, execution, diagnostics, history, and saved SQL behavior. - Keep fixed SQL completion statements and their database-specific variants in `sql/sqlSnippetTemplates.ts`; cross-check row-limiting families against `crates/dbx-core/src/sql_dialect/capabilities.rs` and authoritative dialect documentation, and never replace a user-customized built-in body with a generated default. + Keep fixed SQL completion statements and their database-specific variants in `sql/sqlSnippetTemplates.ts`; cross-check row-limiting families against `crates/dbx-sql/src/sql_dialect/capabilities.rs` and authoritative dialect documentation, and never replace a user-customized built-in body with a generated default. - `dataGrid`: result/grid rendering, editing, previews, pagination, and export helpers tied to the grid. - `ai`, `mcp`: AI assistant and MCP configuration helpers. - `redis`, `mongo`, `elasticsearch`, `etcd`, `kv`, `mq`, `nacos`, `zookeeper`, `webdav`: non-relational or service-specific helpers. @@ -38,5 +38,5 @@ Schema Diff review and execution must preserve these invariants: Regression coverage belongs in: - `packages/app-tests/schemaDiff.test.ts` -- `crates/dbx-core/src/query.rs` +- `crates/dbx-core/src/query/mod.rs` - `src-tauri/src/commands/query.rs` diff --git a/apps/desktop/src/lib/__tests__/dataGrid/columnAlign.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/columnAlign.spec.ts index 74d78aa075..a2ce7d8485 100644 --- a/apps/desktop/src/lib/__tests__/dataGrid/columnAlign.spec.ts +++ b/apps/desktop/src/lib/__tests__/dataGrid/columnAlign.spec.ts @@ -101,7 +101,7 @@ describe("isNumericColumnType", () => { it("covers the full cross-database numeric whitelist used by the alignment classifier", () => { // Cross-database coverage matrix. Every entry here MUST stay in sync with - // the Rust classifier in crates/dbx-core/src/xlsx_export.rs so that the + // the Rust classifier in crates/dbx-formats/src/xlsx_export.rs so that the // grid, the front-end XLSX exporter and the Rust XLSX exporter agree. const numericTypesByDatabase: Record = { mysql: ["tinyint", "smallint", "mediumint", "int", "integer", "bigint", "float", "double", "decimal", "dec", "fixed"], diff --git a/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts b/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts index 393ecd1e14..3c0e0bcbef 100644 --- a/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts +++ b/apps/desktop/src/lib/__tests__/database/databaseObjectCapabilities.spec.ts @@ -70,7 +70,7 @@ describe("databaseObjectCapabilities", () => { it("exposes materialized views for StarRocks only", () => { // StarRocks has a dedicated MV listing/classification path in - // crates/dbx-core/src/db/mysql.rs (`list_starrocks_tables` + + // crates/dbx-drivers/src/db/mysql.rs (`list_starrocks_tables` + // `classify_starrocks_materialized_views`). expect(sidebarObjectKindsForDatabase("starrocks")).toContain("MATERIALIZED_VIEW"); diff --git a/apps/desktop/src/lib/ai/ai.ts b/apps/desktop/src/lib/ai/ai.ts index a31ed8b62a..534d322979 100644 --- a/apps/desktop/src/lib/ai/ai.ts +++ b/apps/desktop/src/lib/ai/ai.ts @@ -19,7 +19,7 @@ const VECTOR_DB_TYPES: ReadonlySet = new Set([ "milvus", "weaviate", "chromadb", - // If modifying this, also update is_vector_db() in crates/dbx-core/src/agent_tools.rs. + // If modifying this, also update is_vector_db() in crates/dbx-core/src/ai/agent_tools.rs. ]); export function isVectorDbType(dbType: DatabaseType): boolean { diff --git a/apps/desktop/src/lib/ai/maxAgentTurns.ts b/apps/desktop/src/lib/ai/maxAgentTurns.ts index f02275bb95..bdf6a9d6ac 100644 --- a/apps/desktop/src/lib/ai/maxAgentTurns.ts +++ b/apps/desktop/src/lib/ai/maxAgentTurns.ts @@ -1,4 +1,4 @@ -// Mirrors DEFAULT/MIN/MAX_MAX_AGENT_TURNS in crates/dbx-core/src/agent_loop.rs. +// Mirrors DEFAULT/MIN/MAX_MAX_AGENT_TURNS in crates/dbx-core/src/ai/agent_loop.rs. // The backend clamp remains the source of truth for persisted values. export const MAX_AGENT_TURNS_DEFAULT = 30; export const MAX_AGENT_TURNS_MIN = 5; diff --git a/apps/desktop/src/lib/backend/__tests__/consulCoreEndpointContract.spec.ts b/apps/desktop/src/lib/backend/__tests__/consulCoreEndpointContract.spec.ts index a906c84b4b..9c6e861f42 100644 --- a/apps/desktop/src/lib/backend/__tests__/consulCoreEndpointContract.spec.ts +++ b/apps/desktop/src/lib/backend/__tests__/consulCoreEndpointContract.spec.ts @@ -106,7 +106,7 @@ const contracts: Contract[] = [ const sourceCache = new Map(); function source(file: string): string { - if (!sourceCache.has(file)) sourceCache.set(file, readFileSync(join(process.cwd(), "crates/dbx-core/src/consul", file), "utf8")); + if (!sourceCache.has(file)) sourceCache.set(file, readFileSync(join(process.cwd(), "crates/dbx-core/src/admin/consul", file), "utf8")); return sourceCache.get(file)!; } diff --git a/apps/desktop/src/lib/backend/__tests__/elasticsearchIndexTransportContract.spec.ts b/apps/desktop/src/lib/backend/__tests__/elasticsearchIndexTransportContract.spec.ts index e6f47e3857..f72e8c1611 100644 --- a/apps/desktop/src/lib/backend/__tests__/elasticsearchIndexTransportContract.spec.ts +++ b/apps/desktop/src/lib/backend/__tests__/elasticsearchIndexTransportContract.spec.ts @@ -6,7 +6,7 @@ const http = readFileSync(new URL("../http.ts", import.meta.url), "utf8"); const api = readFileSync(new URL("../api.ts", import.meta.url), "utf8"); const tauriRegistry = readFileSync(new URL("../../../../../../src-tauri/src/lib.rs", import.meta.url), "utf8"); const webRegistry = readFileSync(new URL("../../../../../../crates/dbx-web/src/main.rs", import.meta.url), "utf8"); -const coreOps = readFileSync(new URL("../../../../../../crates/dbx-core/src/document_ops.rs", import.meta.url), "utf8"); +const coreOps = readFileSync(new URL("../../../../../../crates/dbx-core/src/query/document_ops.rs", import.meta.url), "utf8"); function functionBody(source: string, operation: string): string { const start = source.indexOf(`export async function ${operation}(`); diff --git a/apps/desktop/src/lib/connection/spannerResourcePath.ts b/apps/desktop/src/lib/connection/spannerResourcePath.ts index b37491bc04..2a440eb214 100644 --- a/apps/desktop/src/lib/connection/spannerResourcePath.ts +++ b/apps/desktop/src/lib/connection/spannerResourcePath.ts @@ -123,7 +123,7 @@ export function spannerSchemaDisplayName(schema: string): string { /** * Percent-encode a Spanner resource path for display, keeping the `/` separators * intact — they are structure, not data. Mirrors `encode_spanner_resource_path` - * in crates/dbx-core/src/models/connection.rs. + * in crates/dbx-types/src/models/connection.rs. * * Every other database type keeps whole-value encoding, where a slash inside a * database name really is data and should be escaped. diff --git a/apps/desktop/src/lib/database/databaseFeatureSupport.ts b/apps/desktop/src/lib/database/databaseFeatureSupport.ts index df17c97762..241c282f24 100644 --- a/apps/desktop/src/lib/database/databaseFeatureSupport.ts +++ b/apps/desktop/src/lib/database/databaseFeatureSupport.ts @@ -267,7 +267,7 @@ const ORACLE_STICKY_TRANSACTION_TYPES: ReadonlySet = new Set(["oracle", /** Databases whose manual-transaction toolbar hides Commit/Rollback until an * unproven statement dirties the session. Mirrors the Rust proof gate - * (crates/dbx-core/src/query.rs + sql_risk.rs `prove_read_only_for_database`). + * (crates/dbx-core/src/query/mod.rs + sql_risk.rs `prove_read_only_for_database`). * Every member must also be in TRANSACTION_SUPPORTED_TYPES above: a database * cannot reach manual mode (and this UX) without explicit transaction control * (#9018). Family members like doris/kingbase join only when their transaction diff --git a/apps/desktop/src/lib/database/databaseObjectCapabilities.ts b/apps/desktop/src/lib/database/databaseObjectCapabilities.ts index a934695923..776bfb3f53 100644 --- a/apps/desktop/src/lib/database/databaseObjectCapabilities.ts +++ b/apps/desktop/src/lib/database/databaseObjectCapabilities.ts @@ -68,7 +68,7 @@ const DATABASE_TYPE_OBJECTS = new Map([ ["duckdb", TABLE_VIEW_OBJECTS], ["clickhouse", TABLE_VIEW_OBJECTS], // Doris: backend listing path still uses the generic SHOW TABLES path (see - // `list_tables_once` for `PoolKind::Mysql` in crates/dbx-core/src/schema.rs) + // `list_tables_once` for `PoolKind::Mysql` in crates/dbx-core/src/schema/mod.rs) // and lacks a MV classifier. Keep Doris on TABLE_VIEW_OBJECTS until a // Doris-specific MV listing/classification lands, otherwise the UI advertises // MV support that the backend cannot route. diff --git a/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts b/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts index 0d0fb16b41..566567b4c7 100644 --- a/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts +++ b/apps/desktop/src/lib/elasticsearch/elasticsearchRequestRisk.ts @@ -2,7 +2,7 @@ * Read/write classification for Elasticsearch-compatible REST requests. * * Mirrors `classify_search_engine_query_risk` in - * `crates/dbx-core/src/query_execution_sql.rs` so the desktop guards (read-only + * `crates/dbx-sql/src/query_execution_sql.rs` so the desktop guards (read-only * unlock, production safety) agree with the backend read-only gate instead of * treating every `GET`/`POST` request as an unrecognized — and therefore * unsafe — statement. diff --git a/apps/desktop/src/lib/plugins/pluginFieldConditions.ts b/apps/desktop/src/lib/plugins/pluginFieldConditions.ts index 6ba8870f7d..c58275c825 100644 --- a/apps/desktop/src/lib/plugins/pluginFieldConditions.ts +++ b/apps/desktop/src/lib/plugins/pluginFieldConditions.ts @@ -2,7 +2,7 @@ import type { PluginFieldCondition, PluginFormField, PluginFormFieldValue } from /** * Canonical string form of a form value, matching the Rust host - * (`condition_value_text` in `crates/dbx-core/src/plugins/manifest.rs`). + * (`condition_value_text` in `crates/dbx-plugin-runtime/src/plugins/manifest.rs`). * `false` renders as `"false"` and `22.0` as `"22"`. */ export function pluginConditionValueText(value: PluginFormFieldValue): string { diff --git a/apps/desktop/src/lib/plugins/pluginHostBridge.ts b/apps/desktop/src/lib/plugins/pluginHostBridge.ts index 28269350d4..1e0c773120 100644 --- a/apps/desktop/src/lib/plugins/pluginHostBridge.ts +++ b/apps/desktop/src/lib/plugins/pluginHostBridge.ts @@ -243,7 +243,7 @@ export class PluginHostBridge { /** * Parse `host.network:` permission entries into CSP connect-src * origins. Must stay aligned with `parse_host_network_permission` in - * crates/dbx-core/src/plugins/manifest.rs. + * crates/dbx-plugin-runtime/src/plugins/manifest.rs. */ export function pluginNetworkOrigins(permissions: readonly string[] | undefined): string[] { const origins = new Set(); diff --git a/apps/desktop/src/lib/redis/redisDatabaseIndex.ts b/apps/desktop/src/lib/redis/redisDatabaseIndex.ts index 309c36c1ab..01870136cd 100644 --- a/apps/desktop/src/lib/redis/redisDatabaseIndex.ts +++ b/apps/desktop/src/lib/redis/redisDatabaseIndex.ts @@ -1,4 +1,4 @@ -// Mirrors dbx-core `redis_database_index()` (crates/dbx-core/src/db/redis_driver.rs): +// Mirrors dbx-core `redis_database_index()` (crates/dbx-drivers/src/db/redis_driver.rs): // a Redis database is a numeric index. Anything else (e.g. redis-cli flags like // "0 --tls --insecure" pasted into the field or imported from a dirty URL) fails // to parse on the backend and silently falls back to db 0, so every frontend diff --git a/apps/desktop/src/lib/sql/newQueryContext.ts b/apps/desktop/src/lib/sql/newQueryContext.ts index 9357adf00f..0121b02228 100644 --- a/apps/desktop/src/lib/sql/newQueryContext.ts +++ b/apps/desktop/src/lib/sql/newQueryContext.ts @@ -151,7 +151,7 @@ export function resolveNewQueryTable(input: ResolveNewQueryTableInput): NewQuery * * Time-series engines get the same rolling-window scoping the sidebar * quick-open uses (see `default_time_series_predicate` in - * `crates/dbx-core/src/sql_dialect/table_select.rs`). Without a `time` + * `crates/dbx-sql/src/sql_dialect/table_select.rs`). Without a `time` * predicate InfluxDB scans every shard (v1/v2) or every Parquet file * (v3), which turns "let me draft a query against this table" into a * full-history scan the moment the user hits Run. VictoriaMetrics diff --git a/crates/ARCHITECTURE.md b/crates/ARCHITECTURE.md new file mode 100644 index 0000000000..af7ea536ba --- /dev/null +++ b/crates/ARCHITECTURE.md @@ -0,0 +1,126 @@ +# DBX Rust 模块边界 + +## 依赖方向 + +`dbx-core` 是应用编排层，不再承载所有底层实现。桌面、Web、CLI、MCP +继续通过 core 调用连接、查询、导入导出等业务；独立工具可直接使用下层 crate。 + +```text +desktop / web / cli / mcp + │ + dbx-core + ├── dbx-drivers ── dbx-sql ── dbx-types + │ ├── dbx-types + │ ├── dbx-platform + │ └── dbx-sqlite-worker (protocol, no runtime defaults) + ├── dbx-plugin-runtime ── dbx-types + dbx-platform + ├── dbx-ai-provider ── dbx-platform + ├── dbx-formats + ├── dbx-sql / dbx-types + └── dbx-platform +``` + +下层 crate 不得反向依赖 core，包括 build 和 dev 依赖。涉及多层的测试放在 +`dbx-core/tests/`，避免为了测试把数据库连接引入 SQL、类型或格式 crate。 +`scripts/core-architecture.test.mjs` 校验 workspace 依赖、功能转发、源码归属、 +构建输入和前端测试引用路径；CI 的 Rust 检查执行此守卫。 + +## 所有权 + +| Crate | 拥有的实现 | 不应引入 | +| --- | --- | --- | +| `dbx-types` | 连接配置、数据库身份、查询与元数据 DTO、JS 安全 JSON、MQTT/GridFS 数据记录 | 连接池、应用状态、驱动 SDK | +| `dbx-sql` | 解析、方言注册与加载、SQL 风险、DDL/DML、结构差异计划 | 查询执行、凭据、core 业务 | +| `dbx-drivers` | 原生驱动、隧道、JDBC/Agent 运行与分发、超时取消、SQLite 元数据读取 | 导入/迁移编排、应用配置持久化 | +| `dbx-formats` | CSV/XLSX/文本/时间格式与 ZIP 编码 | 数据库请求、连接管理 | +| `dbx-ai-provider` | 模型与 CLI 适配器、流式协议、事件和 token 用量 | 数据库工具执行和 Agent 业务循环 | +| `dbx-plugin-runtime` | manifest、签名与安装、Marketplace、子进程会话及 host 请求 | 数据库驱动实现、core 应用状态 | +| `dbx-platform` | 进程、路径、代理、下载、版本比较与共享用户提示网关 | 数据库身份和业务规则 | + +`dbx-sql` 包含方言文件加载/监听，因此不是完全无 I/O 的函数库；边界是“不执行数据库业务”。 +平台的 `host-prompts`、`downloads` 和类型库的 `mq-admin` 等能力显式启用， +避免只使用类型或纯格式的消费者意外加载驱动或应用层。 + +## Core 业务目录 + +```text +dbx-core/src/ + lib.rs 对外模块与旧 API 兼容导出 + connection/ 连接路由、凭据、运行配置、JDBC 配置、任务监督 + query/ 查询编排、取消、文档/Redis/HBase 操作、事务与对象缓存 + schema/ 元数据编排、运行时 SQLite 表重建 + data/ 导入导出、迁移、比较、备份、文档与脚本工作流 + ai/ 数据库 Agent 循环、工具、解释、模板与 MCP 策略 + admin/ Nacos、Consul、MQ、MQTT + persistence/ 本地存储、历史、保存的 SQL、配置与云同步 + safety/ 生产安全、写入解锁和风险指标 + host/ 更新、变更日志和外部应用集成 + db/ 驱动兼容导出与 Cloudflare D1 业务接口覆盖 +``` + +目录内存放实际实现，不以 `#[path]` 把旧根目录伪装成新目录。已存在但未声明的 +实验文件不会因迁移而自动启用。 + +Cloudflare D1 的批量导入依赖迁移逻辑，保留在 `data/cloudflare_d1/`；HTTP 驱动、 +SQL 限制与词法处理位于 drivers。SQLite 表重建执行留在 core，SQL 生成位于 sql。 +数据库查询驱动的 CSV 导出编排留在 core，CSV 编码位于 formats。 + +## 兼容性 + +- 原有 `dbx_core::models`、`dbx_core::types`、`dbx_core::db`、`dbx_core::sql*`、 + `dbx_core::ai`、`dbx_core::plugins` 及业务模块路径继续通过重导出可用。 +- 同一 DTO、连接池、提示请求和插件类型不复制定义；旧路径与新 crate 具有相同类型身份。 + `public_api_compatibility` 测试覆盖这些跨层身份。 +- 协议、序列化字段、默认配置、数据库语义、安装目录与进程生命周期不因搬迁而改变。 + 原测试随实现迁移；依赖 core 的跨层测试移入 core 集成测试目录。 +- `test-support` 仅提供原有 stub/测试钩子，由 dev-dependency 启用，不在生产依赖启用。 +- 模块路径产生的日志 target 和 Rust 调试类型名会随 crate/目录变化。自定义 + `RUST_LOG=dbx_core=debug` 不会覆盖新 crate；需要加入 + `dbx_drivers=debug,dbx_sql=debug,dbx_plugin_runtime=debug,dbx_ai_provider=debug,dbx_platform=debug` + 等目标。默认桌面日志级别及 Web 默认过滤策略保持原样。 + +## Features 与资源 + +core 保留原来的 default feature 集合，并向实际实现 crate 转发 +`duckdb-sidecar`、`dynamodb`、`mq-admin`、`openapi` 与 SQLite 能力。 +`sqlite-bundled`、`sqlite-sqlcipher`、`sqlite-multiple-ciphers` 是不同的 SQLite +后端选择，不要用 workspace `--all-features` 代替合法组合测试。 + +- `dbx-types/build.rs` 消费 `plugins/connection-types/`。 +- `dbx-sql/build.rs` 消费 `plugins/dialects/`。 +- Pi MCP bridge 随 AI provider；Agent v2 协议 JSON 随 drivers，并与 Java resource 对照。 +- 数据库文档导出的 JS/CSS 仍在 `dbx-core/assets/`，由 `pnpm build:docs-export` 生成。 +- Docker 的依赖缓存阶段必须包含所有 workspace manifest 和 stub，实际构建阶段 + 必须包含方言与连接类型目录。Nix 保留整个仓库源码输入。 + +## 验证 + +本机不启动数据库/引擎或 Docker 实例。单元测试包含内存 SQLite 与本地协议模拟器； +真实数据库验证复用 SSH 测试服务器上的专用测试实例，使用 SSH 隧道访问。 +Linux 的文件权限回归测试应由非 root 用户执行；root 可绕过目录写权限，会让测试 +前提失效。测试进程的代理应排除 localhost，避免本地 HTTP 模拟器请求被转发。 + +```sh +pnpm test:architecture +cargo check -p dbx-core --no-default-features --all-targets + +env -u DOCKER_CONTEXT \ + DOCKER_HOST=unix:///tmp/dbx-disabled-docker.sock \ + RUST_MIN_STACK=33554432 \ + cargo test -p dbx-core -p dbx-drivers -p dbx-sql \ + -p dbx-types -p dbx-formats -p dbx-platform \ + -p dbx-ai-provider -p dbx-plugin-runtime \ + --no-default-features --features dbx-core/sqlite-bundled --lib + +cargo test -p dbx-core --no-default-features --features sqlite-bundled \ + --test public_api_compatibility --test connection_url_compatibility \ + --test agent_recovery_contract +``` + +交付验证还应覆盖桌面/Web/CLI/MCP 消费者、合法 feature 组合、前端类型与源码契约、 +Agent 安装/恢复以及远程数据库查询/元数据/导入导出。`#[ignore]` 的 live 测试需要其 +声明的测试环境变量；未运行的 live 用例不算通过。禁止把静态构建输入检查描述成 +Docker/Nix 镜像已构建，也不能把单元测试通过描述成所有数据库版本均已实测。 + +拆分的直接收益是可独立检查的依赖边界、实现所有权与回归范围；编译时间、内存和 +运行速度的提升需要同环境 benchmark，不能仅凭目录拆分推断。 diff --git a/crates/README.md b/crates/README.md index adaf564f12..702e60be47 100644 --- a/crates/README.md +++ b/crates/README.md @@ -4,7 +4,19 @@ Rust crates for DBX live here. ## Directories -- `dbx-core/` - shared database core, drivers, schema/query logic, import/export, transfer, and plugin support. +- `dbx-core/` - application orchestration: connections, queries, schema operations, data workflows, AI tools, administration, persistence, safety, and host services. +- `dbx-types/` - shared connection, query, metadata, and serialization contracts; generated database identities. +- `dbx-sql/` - SQL parsing, dialects, analysis, risk classification, DDL/DML generation, and schema diff planning. +- `dbx-drivers/` - native database adapters, database Agents, tunnels, execution budgets, and driver lifecycle. +- `dbx-formats/` - data formatting and file encoders, independent of database connections. +- `dbx-ai-provider/` - AI provider clients, CLI adapters, streaming, and token usage. +- `dbx-plugin-runtime/` - plugin packages, signatures, marketplace, subprocess sessions, and host requests. +- `dbx-platform/` - shared process, path, proxy, download, version, and host-prompt primitives. - `dbx-web/` - the Docker/web backend service binary published as `dbx-web`. +- `dbx-cli/` - the command-line application. +- `dbx-mcp/` - the MCP service library and binary. +- `dbx-sqlite-worker/` - the isolated SQLite file-host worker and protocol. The workspace root is defined in the repository-level `Cargo.toml`. + +See [ARCHITECTURE.md](ARCHITECTURE.md) for dependency rules, compatibility exports, feature selection, and validation commands. diff --git a/crates/dbx-ai-provider/Cargo.toml b/crates/dbx-ai-provider/Cargo.toml new file mode 100644 index 0000000000..b23a5ffeb6 --- /dev/null +++ b/crates/dbx-ai-provider/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "dbx-ai-provider" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +dbx-platform = { path = "../dbx-platform" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +futures = "0.3" +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"] } +tokio = { version = "1", features = ["process", "io-util", "sync", "time", "rt", "macros"] } +log = "0.4" +uuid = { version = "1", features = ["v4", "serde"] } +base64 = "0.22" +bytes = "1" +httpdate = "1" +tempfile = "3" + +[dev-dependencies] +tokio = { version = "1", features = ["test-util", "rt-multi-thread"] } diff --git a/crates/dbx-core/assets/pi-mcp-bridge.mjs b/crates/dbx-ai-provider/assets/pi-mcp-bridge.mjs similarity index 100% rename from crates/dbx-core/assets/pi-mcp-bridge.mjs rename to crates/dbx-ai-provider/assets/pi-mcp-bridge.mjs diff --git a/crates/dbx-core/src/agent_events.rs b/crates/dbx-ai-provider/src/agent_events.rs similarity index 100% rename from crates/dbx-core/src/agent_events.rs rename to crates/dbx-ai-provider/src/agent_events.rs diff --git a/crates/dbx-core/src/ai.rs b/crates/dbx-ai-provider/src/ai.rs similarity index 99% rename from crates/dbx-core/src/ai.rs rename to crates/dbx-ai-provider/src/ai.rs index a4e4206489..3658e4fed6 100644 --- a/crates/dbx-core/src/ai.rs +++ b/crates/dbx-ai-provider/src/ai.rs @@ -2132,7 +2132,7 @@ async fn ollama_model_completion_support( Ok(crate::ai_model_filter::ollama_completion_capability(&data)) } -pub(crate) async fn ollama_selected_model_tool_support(config: &AiConfig) -> Result, String> { +pub async fn ollama_selected_model_tool_support(config: &AiConfig) -> Result, String> { let model_id = config.model.trim(); if model_id.is_empty() { return Err("Model is required".to_string()); @@ -2724,7 +2724,7 @@ fn truncate_diagnostic(value: &str, max_chars: usize) -> String { /// its scheme because proxies often echo that token without `Bearer`/`Basic`. /// Other custom header values retain the [`MIN_SECRET_CHARS`] guard so ordinary /// diagnostics are not mangled by short values such as a version marker. -pub(crate) fn sensitive_values(config: &AiConfig) -> Vec { +pub fn sensitive_values(config: &AiConfig) -> Vec { let mut secrets: Vec = Vec::new(); push_secret(&mut secrets, normalized_api_key(config), true); for (name, value) in &config.custom_headers { @@ -2771,7 +2771,7 @@ fn push_secret(secrets: &mut Vec, value: &str, allow_short: bool) { /// body that echoes `?key=…` cannot leak either. This is the single gate every /// provider error string must pass through before it reaches the UI, a log /// target, or a shared report; callers truncate the result as needed. -pub(crate) fn redact_secrets(value: &str, secrets: &[String]) -> String { +pub fn redact_secrets(value: &str, secrets: &[String]) -> String { let mut redacted = value.to_string(); for secret in secrets { if redacted.contains(secret.as_str()) { diff --git a/crates/dbx-core/src/ai_claude_code_cli.rs b/crates/dbx-ai-provider/src/ai_claude_code_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_claude_code_cli.rs rename to crates/dbx-ai-provider/src/ai_claude_code_cli.rs diff --git a/crates/dbx-core/src/ai_cli_agent.rs b/crates/dbx-ai-provider/src/ai_cli_agent.rs similarity index 100% rename from crates/dbx-core/src/ai_cli_agent.rs rename to crates/dbx-ai-provider/src/ai_cli_agent.rs diff --git a/crates/dbx-core/src/ai_codebuddy_cli.rs b/crates/dbx-ai-provider/src/ai_codebuddy_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_codebuddy_cli.rs rename to crates/dbx-ai-provider/src/ai_codebuddy_cli.rs diff --git a/crates/dbx-core/src/ai_codex_cli.rs b/crates/dbx-ai-provider/src/ai_codex_cli.rs similarity index 99% rename from crates/dbx-core/src/ai_codex_cli.rs rename to crates/dbx-ai-provider/src/ai_codex_cli.rs index a55ecfbfe4..df994bdb77 100644 --- a/crates/dbx-core/src/ai_codex_cli.rs +++ b/crates/dbx-ai-provider/src/ai_codex_cli.rs @@ -134,7 +134,7 @@ async fn codex_process_env(config: &AiConfig, command: &CodexCommandSpec) -> Res config, command, node_runtime_dir.as_deref(), - crate::update::system_proxy_url().as_deref(), + dbx_platform::proxy::system_proxy_url().as_deref(), &inherited_env_keys, ) } diff --git a/crates/dbx-core/src/ai_cursor_cli.rs b/crates/dbx-ai-provider/src/ai_cursor_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_cursor_cli.rs rename to crates/dbx-ai-provider/src/ai_cursor_cli.rs diff --git a/crates/dbx-core/src/ai_effort.rs b/crates/dbx-ai-provider/src/ai_effort.rs similarity index 100% rename from crates/dbx-core/src/ai_effort.rs rename to crates/dbx-ai-provider/src/ai_effort.rs diff --git a/crates/dbx-core/src/ai_grok_cli.rs b/crates/dbx-ai-provider/src/ai_grok_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_grok_cli.rs rename to crates/dbx-ai-provider/src/ai_grok_cli.rs diff --git a/crates/dbx-core/src/ai_model_filter.rs b/crates/dbx-ai-provider/src/ai_model_filter.rs similarity index 100% rename from crates/dbx-core/src/ai_model_filter.rs rename to crates/dbx-ai-provider/src/ai_model_filter.rs diff --git a/crates/dbx-core/src/ai_opencode_cli.rs b/crates/dbx-ai-provider/src/ai_opencode_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_opencode_cli.rs rename to crates/dbx-ai-provider/src/ai_opencode_cli.rs diff --git a/crates/dbx-core/src/ai_pi_agent_cli.rs b/crates/dbx-ai-provider/src/ai_pi_agent_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_pi_agent_cli.rs rename to crates/dbx-ai-provider/src/ai_pi_agent_cli.rs diff --git a/crates/dbx-core/src/ai_qoder_cli.rs b/crates/dbx-ai-provider/src/ai_qoder_cli.rs similarity index 100% rename from crates/dbx-core/src/ai_qoder_cli.rs rename to crates/dbx-ai-provider/src/ai_qoder_cli.rs diff --git a/crates/dbx-ai-provider/src/lib.rs b/crates/dbx-ai-provider/src/lib.rs new file mode 100644 index 0000000000..4c1c67c2ea --- /dev/null +++ b/crates/dbx-ai-provider/src/lib.rs @@ -0,0 +1,18 @@ +#![recursion_limit = "256"] + +pub use dbx_platform::{path_utils, process}; + +pub mod agent_events; +pub mod ai; +pub mod ai_claude_code_cli; +pub mod ai_cli_agent; +pub mod ai_codebuddy_cli; +pub mod ai_codex_cli; +pub mod ai_cursor_cli; +pub mod ai_effort; +pub mod ai_grok_cli; +mod ai_model_filter; +pub mod ai_opencode_cli; +pub mod ai_pi_agent_cli; +pub mod ai_qoder_cli; +pub mod token_usage; diff --git a/crates/dbx-core/src/token_usage.rs b/crates/dbx-ai-provider/src/token_usage.rs similarity index 100% rename from crates/dbx-core/src/token_usage.rs rename to crates/dbx-ai-provider/src/token_usage.rs diff --git a/crates/dbx-core/Cargo.toml b/crates/dbx-core/Cargo.toml index 7fc4d25b3a..c8a1af0c90 100644 --- a/crates/dbx-core/Cargo.toml +++ b/crates/dbx-core/Cargo.toml @@ -6,66 +6,51 @@ license = "Apache-2.0" [features] default = ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher", "system-fonts"] -duckdb-sidecar = [] -dynamodb = ["dep:aws-sdk-dynamodb"] -mq-admin = ["dep:rumqttc"] -sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl"] -sqlite-multiple-ciphers = ["dep:libsqlite3-hotbundle"] +duckdb-sidecar = ["dbx-sql/duckdb-sidecar", "dbx-drivers/duckdb-sidecar"] +dynamodb = ["dbx-drivers/dynamodb"] +mq-admin = ["dep:rumqttc", "dbx-types/mq-admin", "dbx-drivers/mq-admin"] +sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl", "dbx-drivers/sqlite-sqlcipher"] +sqlite-multiple-ciphers = ["dbx-drivers/sqlite-multiple-ciphers"] # Keeps bundled SQLite for consumers that enable no other sqlite source # (dbx-cli, dbx-mcp); do not combine with the cipher features. -sqlite-bundled = ["rusqlite/bundled"] -openapi = ["dep:utoipa"] +sqlite-bundled = ["rusqlite/bundled", "dbx-drivers/sqlite-bundled"] +openapi = ["dbx-types/openapi", "dbx-sql/openapi"] system-fonts = ["font-kit"] [dependencies] +dbx-drivers = { path = "../dbx-drivers", default-features = false } +dbx-sql = { path = "../dbx-sql", default-features = false } +dbx-plugin-runtime = { path = "../dbx-plugin-runtime" } +dbx-formats = { path = "../dbx-formats" } +dbx-types = { path = "../dbx-types", default-features = false } +dbx-ai-provider = { path = "../dbx-ai-provider" } +dbx-platform = { path = "../dbx-platform", features = ["host-prompts", "downloads"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } -semver = "1" -ed25519-dalek = "2" -fs2 = "0.4" -utoipa = { version = "5.5", optional = true } -unicode-width = "0.2" -unicode-ident = "1" serde_yaml_ng = "0.10" -json5 = "0.4" regex = "1" -winnow = "1.0" rayon = "1" percent-encoding = "2" encoding_rs = "0.8" log = "0.4" -tracing = "0.1" -tracing-subscriber = "0.3" tokio = { version = "1", features = ["full"] } tokio-util = { version = "0.7", features = ["compat"] } chrono = { version = "0.4", features = ["serde"] } -httpdate = "1" -rust_decimal = { version = "1", features = ["serde"] } -anyhow = "1" uuid = { version = "1", features = ["v4", "serde"] } tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-uuid-1", "with-serde_json-1"] } deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } -tokio-postgres-rustls = "0.13" -postgres-openssl = "0.5" -openssl = { version = "0.10", features = ["vendored"] } webpki-roots = "0.26" rusqlite = { version = "0.32", default-features = false, features = ["load_extension", "backup", "functions", "column_decltype"] } -libsqlite3-hotbundle = { git = "https://github.com/zipg/libsqlite3-hotbundle.git", rev = "7f12f203b4f4694a273410bd9b02d77f82b6114e", features = ["cipher-sqlcipher"], optional = true } -dbx-sqlite-worker = { path = "../dbx-sqlite-worker", default-features = false } mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "client_ed25519", "chrono", "rust_decimal"] } sqlparser = { version = "0.62.0", features = ["visitor"] } redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tls-rustls-insecure", "tokio-rustls-comp", "sentinel", "cluster-async"] } rustls = { version = "0.23", features = ["aws-lc-rs"] } rustls-pemfile = "2.2" -tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] } reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "stream", "rustls-tls", "socks"] } prometheus-parse = "=0.2.5" futures = "0.3" mongodb = { version = "3.2.5", features = ["socks5-proxy"] } -aws-sdk-dynamodb = { version = "=1.104.0", optional = true } rumqttc = { version = "0.24", features = ["websocket"], optional = true } -russh = "0.60" -pageant = "0.2" portpicker = "0.1.1" csv = "1" calamine = { version = "0.30.1", features = ["dates"] } @@ -86,27 +71,21 @@ flate2 = "1" crc = "3" sha1 = "0.10" tar = "0.4" -zstd = "0.13" sysinfo = { version = "0.32", features = ["system"] } -serde_yaml = "0.9" minijinja = "2" notify = { version = "7", default-features = false, features = ["macos_kqueue"] } arc-swap = "1" -insta = { version = "1", features = ["json", "glob"] } tempfile = "3" [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem", "Win32_Foundation", "Win32_Security"] } +[dev-dependencies] +zstd = "0.13" +dbx-drivers = { path = "../dbx-drivers", default-features = false, features = ["test-support"] } +dbx-plugin-runtime = { path = "../dbx-plugin-runtime", features = ["test-support"] } +dbx-platform = { path = "../dbx-platform", features = ["test-support"] } +tokio = { version = "1", features = ["test-util"] } + [target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] nix = { version = "0.31.3", features = ["user"] } - -[dev-dependencies] -mysql_common = { git = "https://github.com/t8y2/rust_mysql_common.git", rev = "47740d85a8277167a5717afac785d37b46a36296", default-features = false } -tokio = { version = "1", features = ["test-util"] } -tokio-openssl = "0.6" - -[build-dependencies] -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -serde_yaml_ng = "0.10" diff --git a/crates/dbx-core/src/consul/acl.rs b/crates/dbx-core/src/admin/consul/acl.rs similarity index 100% rename from crates/dbx-core/src/consul/acl.rs rename to crates/dbx-core/src/admin/consul/acl.rs diff --git a/crates/dbx-core/src/consul/agent.rs b/crates/dbx-core/src/admin/consul/agent.rs similarity index 100% rename from crates/dbx-core/src/consul/agent.rs rename to crates/dbx-core/src/admin/consul/agent.rs diff --git a/crates/dbx-core/src/consul/archive.rs b/crates/dbx-core/src/admin/consul/archive.rs similarity index 100% rename from crates/dbx-core/src/consul/archive.rs rename to crates/dbx-core/src/admin/consul/archive.rs diff --git a/crates/dbx-core/src/consul/blocking.rs b/crates/dbx-core/src/admin/consul/blocking.rs similarity index 100% rename from crates/dbx-core/src/consul/blocking.rs rename to crates/dbx-core/src/admin/consul/blocking.rs diff --git a/crates/dbx-core/src/consul/bulk.rs b/crates/dbx-core/src/admin/consul/bulk.rs similarity index 100% rename from crates/dbx-core/src/consul/bulk.rs rename to crates/dbx-core/src/admin/consul/bulk.rs diff --git a/crates/dbx-core/src/consul/capabilities.rs b/crates/dbx-core/src/admin/consul/capabilities.rs similarity index 100% rename from crates/dbx-core/src/consul/capabilities.rs rename to crates/dbx-core/src/admin/consul/capabilities.rs diff --git a/crates/dbx-core/src/consul/catalog.rs b/crates/dbx-core/src/admin/consul/catalog.rs similarity index 100% rename from crates/dbx-core/src/consul/catalog.rs rename to crates/dbx-core/src/admin/consul/catalog.rs diff --git a/crates/dbx-core/src/consul/client.rs b/crates/dbx-core/src/admin/consul/client.rs similarity index 100% rename from crates/dbx-core/src/consul/client.rs rename to crates/dbx-core/src/admin/consul/client.rs diff --git a/crates/dbx-core/src/consul/config.rs b/crates/dbx-core/src/admin/consul/config.rs similarity index 100% rename from crates/dbx-core/src/consul/config.rs rename to crates/dbx-core/src/admin/consul/config.rs diff --git a/crates/dbx-core/src/consul/enterprise.rs b/crates/dbx-core/src/admin/consul/enterprise.rs similarity index 100% rename from crates/dbx-core/src/consul/enterprise.rs rename to crates/dbx-core/src/admin/consul/enterprise.rs diff --git a/crates/dbx-core/src/consul/health.rs b/crates/dbx-core/src/admin/consul/health.rs similarity index 100% rename from crates/dbx-core/src/consul/health.rs rename to crates/dbx-core/src/admin/consul/health.rs diff --git a/crates/dbx-core/src/consul/kv.rs b/crates/dbx-core/src/admin/consul/kv.rs similarity index 100% rename from crates/dbx-core/src/consul/kv.rs rename to crates/dbx-core/src/admin/consul/kv.rs diff --git a/crates/dbx-core/src/consul/mesh/config_entries.rs b/crates/dbx-core/src/admin/consul/mesh/config_entries.rs similarity index 100% rename from crates/dbx-core/src/consul/mesh/config_entries.rs rename to crates/dbx-core/src/admin/consul/mesh/config_entries.rs diff --git a/crates/dbx-core/src/consul/mesh/discovery.rs b/crates/dbx-core/src/admin/consul/mesh/discovery.rs similarity index 100% rename from crates/dbx-core/src/consul/mesh/discovery.rs rename to crates/dbx-core/src/admin/consul/mesh/discovery.rs diff --git a/crates/dbx-core/src/consul/mesh/exported_services.rs b/crates/dbx-core/src/admin/consul/mesh/exported_services.rs similarity index 100% rename from crates/dbx-core/src/consul/mesh/exported_services.rs rename to crates/dbx-core/src/admin/consul/mesh/exported_services.rs diff --git a/crates/dbx-core/src/consul/mesh/intentions.rs b/crates/dbx-core/src/admin/consul/mesh/intentions.rs similarity index 100% rename from crates/dbx-core/src/consul/mesh/intentions.rs rename to crates/dbx-core/src/admin/consul/mesh/intentions.rs diff --git a/crates/dbx-core/src/consul/mesh/mod.rs b/crates/dbx-core/src/admin/consul/mesh/mod.rs similarity index 100% rename from crates/dbx-core/src/consul/mesh/mod.rs rename to crates/dbx-core/src/admin/consul/mesh/mod.rs diff --git a/crates/dbx-core/src/consul/mesh/peering.rs b/crates/dbx-core/src/admin/consul/mesh/peering.rs similarity index 100% rename from crates/dbx-core/src/consul/mesh/peering.rs rename to crates/dbx-core/src/admin/consul/mesh/peering.rs diff --git a/crates/dbx-core/src/consul/mod.rs b/crates/dbx-core/src/admin/consul/mod.rs similarity index 100% rename from crates/dbx-core/src/consul/mod.rs rename to crates/dbx-core/src/admin/consul/mod.rs diff --git a/crates/dbx-core/src/consul/operator.rs b/crates/dbx-core/src/admin/consul/operator.rs similarity index 100% rename from crates/dbx-core/src/consul/operator.rs rename to crates/dbx-core/src/admin/consul/operator.rs diff --git a/crates/dbx-core/src/consul/response.rs b/crates/dbx-core/src/admin/consul/response.rs similarity index 100% rename from crates/dbx-core/src/consul/response.rs rename to crates/dbx-core/src/admin/consul/response.rs diff --git a/crates/dbx-core/src/consul/search.rs b/crates/dbx-core/src/admin/consul/search.rs similarity index 100% rename from crates/dbx-core/src/consul/search.rs rename to crates/dbx-core/src/admin/consul/search.rs diff --git a/crates/dbx-core/src/consul/session.rs b/crates/dbx-core/src/admin/consul/session.rs similarity index 100% rename from crates/dbx-core/src/consul/session.rs rename to crates/dbx-core/src/admin/consul/session.rs diff --git a/crates/dbx-core/src/consul/status.rs b/crates/dbx-core/src/admin/consul/status.rs similarity index 100% rename from crates/dbx-core/src/consul/status.rs rename to crates/dbx-core/src/admin/consul/status.rs diff --git a/crates/dbx-core/src/consul/test_support.rs b/crates/dbx-core/src/admin/consul/test_support.rs similarity index 100% rename from crates/dbx-core/src/consul/test_support.rs rename to crates/dbx-core/src/admin/consul/test_support.rs diff --git a/crates/dbx-core/src/consul/tools.rs b/crates/dbx-core/src/admin/consul/tools.rs similarity index 100% rename from crates/dbx-core/src/consul/tools.rs rename to crates/dbx-core/src/admin/consul/tools.rs diff --git a/crates/dbx-core/src/consul/txn.rs b/crates/dbx-core/src/admin/consul/txn.rs similarity index 100% rename from crates/dbx-core/src/consul/txn.rs rename to crates/dbx-core/src/admin/consul/txn.rs diff --git a/crates/dbx-core/src/consul/types.rs b/crates/dbx-core/src/admin/consul/types.rs similarity index 100% rename from crates/dbx-core/src/consul/types.rs rename to crates/dbx-core/src/admin/consul/types.rs diff --git a/crates/dbx-core/src/admin/mod.rs b/crates/dbx-core/src/admin/mod.rs new file mode 100644 index 0000000000..9277e5960d --- /dev/null +++ b/crates/dbx-core/src/admin/mod.rs @@ -0,0 +1,6 @@ +pub mod consul; +#[cfg(feature = "mq-admin")] +pub mod mq; +#[cfg(feature = "mq-admin")] +pub mod mqtt; +pub mod nacos; diff --git a/crates/dbx-core/src/mq/README.md b/crates/dbx-core/src/admin/mq/README.md similarity index 100% rename from crates/dbx-core/src/mq/README.md rename to crates/dbx-core/src/admin/mq/README.md diff --git a/crates/dbx-core/src/mq/adapters/kafka.rs b/crates/dbx-core/src/admin/mq/adapters/kafka.rs similarity index 100% rename from crates/dbx-core/src/mq/adapters/kafka.rs rename to crates/dbx-core/src/admin/mq/adapters/kafka.rs diff --git a/crates/dbx-core/src/mq/adapters/mod.rs b/crates/dbx-core/src/admin/mq/adapters/mod.rs similarity index 100% rename from crates/dbx-core/src/mq/adapters/mod.rs rename to crates/dbx-core/src/admin/mq/adapters/mod.rs diff --git a/crates/dbx-core/src/mq/adapters/pulsar.rs b/crates/dbx-core/src/admin/mq/adapters/pulsar.rs similarity index 100% rename from crates/dbx-core/src/mq/adapters/pulsar.rs rename to crates/dbx-core/src/admin/mq/adapters/pulsar.rs diff --git a/crates/dbx-core/src/mq/adapters/pulsar_version.rs b/crates/dbx-core/src/admin/mq/adapters/pulsar_version.rs similarity index 100% rename from crates/dbx-core/src/mq/adapters/pulsar_version.rs rename to crates/dbx-core/src/admin/mq/adapters/pulsar_version.rs diff --git a/crates/dbx-core/src/mq/adapters/rabbitmq.rs b/crates/dbx-core/src/admin/mq/adapters/rabbitmq.rs similarity index 100% rename from crates/dbx-core/src/mq/adapters/rabbitmq.rs rename to crates/dbx-core/src/admin/mq/adapters/rabbitmq.rs diff --git a/crates/dbx-core/src/mq/adapters/rocketmq.rs b/crates/dbx-core/src/admin/mq/adapters/rocketmq.rs similarity index 100% rename from crates/dbx-core/src/mq/adapters/rocketmq.rs rename to crates/dbx-core/src/admin/mq/adapters/rocketmq.rs diff --git a/crates/dbx-core/src/mq/auth.rs b/crates/dbx-core/src/admin/mq/auth.rs similarity index 100% rename from crates/dbx-core/src/mq/auth.rs rename to crates/dbx-core/src/admin/mq/auth.rs diff --git a/crates/dbx-core/src/mq/config.rs b/crates/dbx-core/src/admin/mq/config.rs similarity index 100% rename from crates/dbx-core/src/mq/config.rs rename to crates/dbx-core/src/admin/mq/config.rs diff --git a/crates/dbx-core/src/mq/mod.rs b/crates/dbx-core/src/admin/mq/mod.rs similarity index 100% rename from crates/dbx-core/src/mq/mod.rs rename to crates/dbx-core/src/admin/mq/mod.rs diff --git a/crates/dbx-core/src/mq/port.rs b/crates/dbx-core/src/admin/mq/port.rs similarity index 100% rename from crates/dbx-core/src/mq/port.rs rename to crates/dbx-core/src/admin/mq/port.rs diff --git a/crates/dbx-core/src/mq/service.rs b/crates/dbx-core/src/admin/mq/service.rs similarity index 100% rename from crates/dbx-core/src/mq/service.rs rename to crates/dbx-core/src/admin/mq/service.rs diff --git a/crates/dbx-core/src/mq/token.rs b/crates/dbx-core/src/admin/mq/token.rs similarity index 100% rename from crates/dbx-core/src/mq/token.rs rename to crates/dbx-core/src/admin/mq/token.rs diff --git a/crates/dbx-core/src/mq/types.rs b/crates/dbx-core/src/admin/mq/types.rs similarity index 100% rename from crates/dbx-core/src/mq/types.rs rename to crates/dbx-core/src/admin/mq/types.rs diff --git a/crates/dbx-core/src/mq/util.rs b/crates/dbx-core/src/admin/mq/util.rs similarity index 100% rename from crates/dbx-core/src/mq/util.rs rename to crates/dbx-core/src/admin/mq/util.rs diff --git a/crates/dbx-core/src/mqtt/client.rs b/crates/dbx-core/src/admin/mqtt/client.rs similarity index 100% rename from crates/dbx-core/src/mqtt/client.rs rename to crates/dbx-core/src/admin/mqtt/client.rs diff --git a/crates/dbx-core/src/mqtt/mod.rs b/crates/dbx-core/src/admin/mqtt/mod.rs similarity index 68% rename from crates/dbx-core/src/mqtt/mod.rs rename to crates/dbx-core/src/admin/mqtt/mod.rs index 8ae7bb7047..575f4dd87e 100644 --- a/crates/dbx-core/src/mqtt/mod.rs +++ b/crates/dbx-core/src/admin/mqtt/mod.rs @@ -8,10 +8,10 @@ //! src-tauri/src/commands/mqtt_cmd.rs (Tauri command 入口) //! │ //! ▼ -//! crates/dbx-core/src/mqtt/service.rs (共享核心逻辑) +//! crates/dbx-core/src/admin/mqtt/service.rs (共享核心逻辑) //! │ //! ▼ -//! crates/dbx-core/src/mqtt/client.rs (rumqttc 客户端封装) +//! crates/dbx-core/src/admin/mqtt/client.rs (rumqttc 客户端封装) //! │ //! ▼ //! MQTT Broker (EMQX / Mosquitto / HiveMQ / ...) @@ -19,4 +19,4 @@ pub mod client; pub mod service; -pub mod types; +pub use dbx_types::mqtt as types; diff --git a/crates/dbx-core/src/mqtt/service.rs b/crates/dbx-core/src/admin/mqtt/service.rs similarity index 100% rename from crates/dbx-core/src/mqtt/service.rs rename to crates/dbx-core/src/admin/mqtt/service.rs diff --git a/crates/dbx-core/src/nacos/access_control.rs b/crates/dbx-core/src/admin/nacos/access_control.rs similarity index 100% rename from crates/dbx-core/src/nacos/access_control.rs rename to crates/dbx-core/src/admin/nacos/access_control.rs diff --git a/crates/dbx-core/src/nacos/archive.rs b/crates/dbx-core/src/admin/nacos/archive.rs similarity index 100% rename from crates/dbx-core/src/nacos/archive.rs rename to crates/dbx-core/src/admin/nacos/archive.rs diff --git a/crates/dbx-core/src/nacos/batch.rs b/crates/dbx-core/src/admin/nacos/batch.rs similarity index 100% rename from crates/dbx-core/src/nacos/batch.rs rename to crates/dbx-core/src/admin/nacos/batch.rs diff --git a/crates/dbx-core/src/nacos/config.rs b/crates/dbx-core/src/admin/nacos/config.rs similarity index 100% rename from crates/dbx-core/src/nacos/config.rs rename to crates/dbx-core/src/admin/nacos/config.rs diff --git a/crates/dbx-core/src/nacos/http.rs b/crates/dbx-core/src/admin/nacos/http.rs similarity index 100% rename from crates/dbx-core/src/nacos/http.rs rename to crates/dbx-core/src/admin/nacos/http.rs diff --git a/crates/dbx-core/src/nacos/mod.rs b/crates/dbx-core/src/admin/nacos/mod.rs similarity index 100% rename from crates/dbx-core/src/nacos/mod.rs rename to crates/dbx-core/src/admin/nacos/mod.rs diff --git a/crates/dbx-core/src/nacos/namespace_access.rs b/crates/dbx-core/src/admin/nacos/namespace_access.rs similarity index 100% rename from crates/dbx-core/src/nacos/namespace_access.rs rename to crates/dbx-core/src/admin/nacos/namespace_access.rs diff --git a/crates/dbx-core/src/nacos/port.rs b/crates/dbx-core/src/admin/nacos/port.rs similarity index 100% rename from crates/dbx-core/src/nacos/port.rs rename to crates/dbx-core/src/admin/nacos/port.rs diff --git a/crates/dbx-core/src/nacos/prometheus.rs b/crates/dbx-core/src/admin/nacos/prometheus.rs similarity index 100% rename from crates/dbx-core/src/nacos/prometheus.rs rename to crates/dbx-core/src/admin/nacos/prometheus.rs diff --git a/crates/dbx-core/src/nacos/search.rs b/crates/dbx-core/src/admin/nacos/search.rs similarity index 100% rename from crates/dbx-core/src/nacos/search.rs rename to crates/dbx-core/src/admin/nacos/search.rs diff --git a/crates/dbx-core/src/nacos/service.rs b/crates/dbx-core/src/admin/nacos/service.rs similarity index 100% rename from crates/dbx-core/src/nacos/service.rs rename to crates/dbx-core/src/admin/nacos/service.rs diff --git a/crates/dbx-core/src/nacos/types.rs b/crates/dbx-core/src/admin/nacos/types.rs similarity index 100% rename from crates/dbx-core/src/nacos/types.rs rename to crates/dbx-core/src/admin/nacos/types.rs diff --git a/crates/dbx-core/src/agent_explain.rs b/crates/dbx-core/src/ai/agent_explain.rs similarity index 100% rename from crates/dbx-core/src/agent_explain.rs rename to crates/dbx-core/src/ai/agent_explain.rs diff --git a/crates/dbx-core/src/agent_kv.rs b/crates/dbx-core/src/ai/agent_kv.rs similarity index 100% rename from crates/dbx-core/src/agent_kv.rs rename to crates/dbx-core/src/ai/agent_kv.rs diff --git a/crates/dbx-core/src/agent_loop.rs b/crates/dbx-core/src/ai/agent_loop.rs similarity index 100% rename from crates/dbx-core/src/agent_loop.rs rename to crates/dbx-core/src/ai/agent_loop.rs diff --git a/crates/dbx-core/src/agent_tools.rs b/crates/dbx-core/src/ai/agent_tools.rs similarity index 100% rename from crates/dbx-core/src/agent_tools.rs rename to crates/dbx-core/src/ai/agent_tools.rs diff --git a/crates/dbx-core/src/mcp_policy.rs b/crates/dbx-core/src/ai/mcp_policy.rs similarity index 100% rename from crates/dbx-core/src/mcp_policy.rs rename to crates/dbx-core/src/ai/mcp_policy.rs diff --git a/crates/dbx-core/src/ai/mod.rs b/crates/dbx-core/src/ai/mod.rs new file mode 100644 index 0000000000..1b2068ed89 --- /dev/null +++ b/crates/dbx-core/src/ai/mod.rs @@ -0,0 +1,8 @@ +pub use dbx_ai_provider::ai::*; + +pub mod agent_explain; +pub mod agent_kv; +pub mod agent_loop; +pub mod agent_tools; +pub mod mcp_policy; +pub mod prompt_template; diff --git a/crates/dbx-core/src/prompt_template.rs b/crates/dbx-core/src/ai/prompt_template.rs similarity index 100% rename from crates/dbx-core/src/prompt_template.rs rename to crates/dbx-core/src/ai/prompt_template.rs diff --git a/crates/dbx-core/src/connection_secrets.rs b/crates/dbx-core/src/connection/connection_secrets.rs similarity index 100% rename from crates/dbx-core/src/connection_secrets.rs rename to crates/dbx-core/src/connection/connection_secrets.rs diff --git a/crates/dbx-core/src/driver_runtime.rs b/crates/dbx-core/src/connection/driver_runtime.rs similarity index 100% rename from crates/dbx-core/src/driver_runtime.rs rename to crates/dbx-core/src/connection/driver_runtime.rs diff --git a/crates/dbx-core/src/jdbc.rs b/crates/dbx-core/src/connection/jdbc.rs similarity index 100% rename from crates/dbx-core/src/jdbc.rs rename to crates/dbx-core/src/connection/jdbc.rs diff --git a/crates/dbx-core/src/connection.rs b/crates/dbx-core/src/connection/mod.rs similarity index 99% rename from crates/dbx-core/src/connection.rs rename to crates/dbx-core/src/connection/mod.rs index 64cc2f4153..e90e85ca17 100644 --- a/crates/dbx-core/src/connection.rs +++ b/crates/dbx-core/src/connection/mod.rs @@ -1,3 +1,10 @@ +pub mod connection_secrets; +pub mod driver_runtime; +pub mod jdbc; +pub mod runtime_config; +pub mod session_credentials; +pub mod task_supervisor; + use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/crates/dbx-core/src/runtime_config.rs b/crates/dbx-core/src/connection/runtime_config.rs similarity index 100% rename from crates/dbx-core/src/runtime_config.rs rename to crates/dbx-core/src/connection/runtime_config.rs diff --git a/crates/dbx-core/src/session_credentials.rs b/crates/dbx-core/src/connection/session_credentials.rs similarity index 100% rename from crates/dbx-core/src/session_credentials.rs rename to crates/dbx-core/src/connection/session_credentials.rs diff --git a/crates/dbx-core/src/task_supervisor.rs b/crates/dbx-core/src/connection/task_supervisor.rs similarity index 100% rename from crates/dbx-core/src/task_supervisor.rs rename to crates/dbx-core/src/connection/task_supervisor.rs diff --git a/crates/dbx-core/src/db/cloudflare_d1/import.rs b/crates/dbx-core/src/data/cloudflare_d1/import.rs similarity index 100% rename from crates/dbx-core/src/db/cloudflare_d1/import.rs rename to crates/dbx-core/src/data/cloudflare_d1/import.rs diff --git a/crates/dbx-core/src/data/cloudflare_d1/mod.rs b/crates/dbx-core/src/data/cloudflare_d1/mod.rs new file mode 100644 index 0000000000..91d1d108ba --- /dev/null +++ b/crates/dbx-core/src/data/cloudflare_d1/mod.rs @@ -0,0 +1,7 @@ +pub use dbx_drivers::db::cloudflare_d1::*; + +mod import; +#[cfg(test)] +mod transfer_tests; + +pub(crate) use import::{build_import_insert_batches, build_streaming_import_insert_batch}; diff --git a/crates/dbx-core/src/db/cloudflare_d1/transfer_tests.rs b/crates/dbx-core/src/data/cloudflare_d1/transfer_tests.rs similarity index 100% rename from crates/dbx-core/src/db/cloudflare_d1/transfer_tests.rs rename to crates/dbx-core/src/data/cloudflare_d1/transfer_tests.rs diff --git a/crates/dbx-core/src/correction.rs b/crates/dbx-core/src/data/correction.rs similarity index 100% rename from crates/dbx-core/src/correction.rs rename to crates/dbx-core/src/data/correction.rs diff --git a/crates/dbx-core/src/data/csv_export.rs b/crates/dbx-core/src/data/csv_export.rs new file mode 100644 index 0000000000..cdb79bd3b4 --- /dev/null +++ b/crates/dbx-core/src/data/csv_export.rs @@ -0,0 +1,111 @@ +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::{BufWriter, Write}; + +use crate::connection::AppState; +use crate::models::connection::DatabaseType; +use crate::query::{execute_sql_statement_with_options, QueryExecutionOptions}; +use crate::sql_dialect::{build_table_data_select_sql, TableDataSelectSqlOptions}; + +pub use dbx_formats::csv_export::*; + +const TABLE_DATA_EXPORT_PAGE_SIZE: usize = 10_000; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableCsvExportOptions { + pub file_path: String, + pub connection_id: String, + pub database: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + pub table_name: String, + #[serde(default)] + pub columns: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub page_size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_secs: Option, + #[serde(default)] + pub csv_quote_mode: CsvQuoteMode, +} + +async fn connection_database_type(state: &AppState, connection_id: &str) -> Result { + state + .configs + .read() + .await + .get(connection_id) + .map(|config| config.db_type) + .ok_or_else(|| format!("Connection config not found: {connection_id}")) +} + +pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExportOptions) -> Result { + let database_type = connection_database_type(state, &options.connection_id).await?; + let page_size = options.page_size.unwrap_or(TABLE_DATA_EXPORT_PAGE_SIZE).max(1); + let mut writer = + BufWriter::new(File::create(&options.file_path).map_err(|err| format!("Failed to write CSV file: {err}"))?); + writer.write_all("\u{FEFF}".as_bytes()).map_err(|err| err.to_string())?; + + let mut offset = 0usize; + let mut rows_exported = 0u64; + let mut wrote_header = false; + + loop { + let sql = build_table_data_select_sql(TableDataSelectSqlOptions { + database_type: Some(database_type), + schema: options.schema.clone(), + table_name: options.table_name.clone(), + table_type: None, + primary_keys: Vec::new(), + columns: options.columns.clone(), + fallback_order_columns: Vec::new(), + order_by: None, + limit: Some(page_size), + offset: Some(offset), + where_input: None, + include_row_id: false, + ..Default::default() + }); + let result = execute_sql_statement_with_options( + state, + &options.connection_id, + &options.database, + &sql, + options.schema.as_deref(), + None, + QueryExecutionOptions { + max_rows: Some(page_size), + timeout_secs: options.timeout_secs, + ..Default::default() + }, + ) + .await?; + + if !wrote_header { + write_csv_text_row(&mut writer, result.columns, options.csv_quote_mode)?; + wrote_header = true; + } + + let fetched = result.rows.len(); + if fetched == 0 { + break; + } + for row in result.rows { + writer.write_all(b"\n").map_err(|err| err.to_string())?; + write_csv_value_row(&mut writer, row, options.csv_quote_mode)?; + } + + rows_exported += fetched as u64; + if fetched < page_size { + break; + } + offset += fetched; + } + + if rows_exported == 0 { + writer.write_all(b"\n").map_err(|err| err.to_string())?; + } + writer.flush().map_err(|err| err.to_string())?; + Ok(rows_exported) +} diff --git a/crates/dbx-core/src/data_compare.rs b/crates/dbx-core/src/data/data_compare.rs similarity index 100% rename from crates/dbx-core/src/data_compare.rs rename to crates/dbx-core/src/data/data_compare.rs diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/data/database_export.rs similarity index 100% rename from crates/dbx-core/src/database_export.rs rename to crates/dbx-core/src/data/database_export.rs diff --git a/crates/dbx-core/src/docs/annotations.rs b/crates/dbx-core/src/data/docs/annotations.rs similarity index 100% rename from crates/dbx-core/src/docs/annotations.rs rename to crates/dbx-core/src/data/docs/annotations.rs diff --git a/crates/dbx-core/src/docs/collector.rs b/crates/dbx-core/src/data/docs/collector.rs similarity index 100% rename from crates/dbx-core/src/docs/collector.rs rename to crates/dbx-core/src/data/docs/collector.rs diff --git a/crates/dbx-core/src/docs/color.rs b/crates/dbx-core/src/data/docs/color.rs similarity index 100% rename from crates/dbx-core/src/docs/color.rs rename to crates/dbx-core/src/data/docs/color.rs diff --git a/crates/dbx-core/src/docs/dbml.rs b/crates/dbx-core/src/data/docs/dbml.rs similarity index 100% rename from crates/dbx-core/src/docs/dbml.rs rename to crates/dbx-core/src/data/docs/dbml.rs diff --git a/crates/dbx-core/src/docs/export.rs b/crates/dbx-core/src/data/docs/export.rs similarity index 97% rename from crates/dbx-core/src/docs/export.rs rename to crates/dbx-core/src/data/docs/export.rs index fc76629319..8f911c753a 100644 --- a/crates/dbx-core/src/docs/export.rs +++ b/crates/dbx-core/src/data/docs/export.rs @@ -8,8 +8,8 @@ use crate::docs::snapshot::SchemaSnapshot; /// Committed rather than built by cargo because Rust cannot run Vite and /// `dbx docs` must work from a plain `cargo install` on a machine with no /// Node. `docs_export_bundle_is_current` is what keeps it honest. -const EXPORT_JS: &str = include_str!("../../assets/docs-export.js"); -const EXPORT_CSS: &str = include_str!("../../assets/docs-export.css"); +const EXPORT_JS: &str = include_str!("../../../assets/docs-export.js"); +const EXPORT_CSS: &str = include_str!("../../../assets/docs-export.css"); pub const EXPORT_LANGUAGES: [&str; 8] = ["en", "es", "it", "ja", "ko", "pt-BR", "zh-CN", "zh-TW"]; @@ -186,8 +186,9 @@ mod tests { return; // packaged crate: the sources are genuinely absent } - let manifest: serde_json::Value = serde_json::from_str(include_str!("../../assets/docs-export.manifest.json")) - .expect("manifest is valid JSON"); + let manifest: serde_json::Value = + serde_json::from_str(include_str!("../../../assets/docs-export.manifest.json")) + .expect("manifest is valid JSON"); let sources = manifest["sources"].as_object().expect("manifest.sources"); assert!(sources.len() > 5, "manifest lists only {} sources — the build emitted an empty graph", sources.len()); diff --git a/crates/dbx-core/src/docs/keys.rs b/crates/dbx-core/src/data/docs/keys.rs similarity index 100% rename from crates/dbx-core/src/docs/keys.rs rename to crates/dbx-core/src/data/docs/keys.rs diff --git a/crates/dbx-core/src/docs/mod.rs b/crates/dbx-core/src/data/docs/mod.rs similarity index 100% rename from crates/dbx-core/src/docs/mod.rs rename to crates/dbx-core/src/data/docs/mod.rs diff --git a/crates/dbx-core/src/docs/relations.rs b/crates/dbx-core/src/data/docs/relations.rs similarity index 100% rename from crates/dbx-core/src/docs/relations.rs rename to crates/dbx-core/src/data/docs/relations.rs diff --git a/crates/dbx-core/src/docs/snapshot.rs b/crates/dbx-core/src/data/docs/snapshot.rs similarity index 100% rename from crates/dbx-core/src/docs/snapshot.rs rename to crates/dbx-core/src/data/docs/snapshot.rs diff --git a/crates/dbx-core/src/export_runtime.rs b/crates/dbx-core/src/data/export_runtime.rs similarity index 100% rename from crates/dbx-core/src/export_runtime.rs rename to crates/dbx-core/src/data/export_runtime.rs diff --git a/crates/dbx-core/src/data/mod.rs b/crates/dbx-core/src/data/mod.rs new file mode 100644 index 0000000000..4d4a6dfcf3 --- /dev/null +++ b/crates/dbx-core/src/data/mod.rs @@ -0,0 +1,17 @@ +pub mod cloudflare_d1; +pub mod correction; +pub mod csv_export; +pub mod data_compare; +pub mod database_export; +pub mod docs; +pub mod export_runtime; +pub mod mongodb_dump; +pub mod mongodb_import_export; +pub mod query_result_export; +pub mod script_generator; +pub mod sql_file_import; +pub mod sqlite_backup; +pub mod table_export; +pub mod table_import; +pub mod transfer; +pub mod transfer_rebuild; diff --git a/crates/dbx-core/src/mongodb_dump.rs b/crates/dbx-core/src/data/mongodb_dump.rs similarity index 100% rename from crates/dbx-core/src/mongodb_dump.rs rename to crates/dbx-core/src/data/mongodb_dump.rs diff --git a/crates/dbx-core/src/mongodb_dump/archive.rs b/crates/dbx-core/src/data/mongodb_dump/archive.rs similarity index 100% rename from crates/dbx-core/src/mongodb_dump/archive.rs rename to crates/dbx-core/src/data/mongodb_dump/archive.rs diff --git a/crates/dbx-core/src/mongodb_dump/directory.rs b/crates/dbx-core/src/data/mongodb_dump/directory.rs similarity index 100% rename from crates/dbx-core/src/mongodb_dump/directory.rs rename to crates/dbx-core/src/data/mongodb_dump/directory.rs diff --git a/crates/dbx-core/src/mongodb_dump/metadata.rs b/crates/dbx-core/src/data/mongodb_dump/metadata.rs similarity index 100% rename from crates/dbx-core/src/mongodb_dump/metadata.rs rename to crates/dbx-core/src/data/mongodb_dump/metadata.rs diff --git a/crates/dbx-core/src/mongodb_dump/source.rs b/crates/dbx-core/src/data/mongodb_dump/source.rs similarity index 100% rename from crates/dbx-core/src/mongodb_dump/source.rs rename to crates/dbx-core/src/data/mongodb_dump/source.rs diff --git a/crates/dbx-core/src/mongodb_dump/stream.rs b/crates/dbx-core/src/data/mongodb_dump/stream.rs similarity index 100% rename from crates/dbx-core/src/mongodb_dump/stream.rs rename to crates/dbx-core/src/data/mongodb_dump/stream.rs diff --git a/crates/dbx-core/src/mongodb_import_export.rs b/crates/dbx-core/src/data/mongodb_import_export.rs similarity index 100% rename from crates/dbx-core/src/mongodb_import_export.rs rename to crates/dbx-core/src/data/mongodb_import_export.rs diff --git a/crates/dbx-core/src/query_result_export.rs b/crates/dbx-core/src/data/query_result_export.rs similarity index 100% rename from crates/dbx-core/src/query_result_export.rs rename to crates/dbx-core/src/data/query_result_export.rs diff --git a/crates/dbx-core/src/script_generator.rs b/crates/dbx-core/src/data/script_generator.rs similarity index 98% rename from crates/dbx-core/src/script_generator.rs rename to crates/dbx-core/src/data/script_generator.rs index 1194626902..39f1e999cd 100644 --- a/crates/dbx-core/src/script_generator.rs +++ b/crates/dbx-core/src/data/script_generator.rs @@ -2023,7 +2023,7 @@ mod tests { sync_sql: None, }], sync_sql: "ALTER TABLE users ADD COLUMN name VARCHAR(100);".to_string(), - ..Default::default() + ..empty_schema_diff() }; let options = JointScriptOptions { @@ -2072,29 +2072,27 @@ mod tests { }], sync_sql: "CREATE TABLE test_table (id INT PRIMARY KEY);".to_string(), rollback_sync_sql: Some("DROP TABLE IF EXISTS test_table;".to_string()), - ..Default::default() + ..empty_schema_diff() } } - impl Default for SchemaDiffPreparation { - fn default() -> Self { - Self { - diffs: vec![], - function_diffs: vec![], - sequence_diffs: vec![], - rule_diffs: vec![], - owner_diffs: vec![], - sync_sql: String::new(), - rollback_sync_sql: None, - rollback_completeness: crate::schema_diff::RollbackCompleteness::Complete, - missing_rollback_objects: vec![], - rename_candidates: vec![], - rollback_graph: None, - compatibility_warnings: vec![], - permission_diffs: vec![], - permission_sync_sql: None, - dependency_graph: None, - } + fn empty_schema_diff() -> SchemaDiffPreparation { + SchemaDiffPreparation { + diffs: vec![], + function_diffs: vec![], + sequence_diffs: vec![], + rule_diffs: vec![], + owner_diffs: vec![], + sync_sql: String::new(), + rollback_sync_sql: None, + rollback_completeness: crate::schema_diff::RollbackCompleteness::Complete, + missing_rollback_objects: vec![], + rename_candidates: vec![], + rollback_graph: None, + compatibility_warnings: vec![], + permission_diffs: vec![], + permission_sync_sql: None, + dependency_graph: None, } } @@ -2231,7 +2229,7 @@ mod tests { #[test] fn generate_enhanced_sync_sql_with_idempotent() { let schema_diff = - SchemaDiffPreparation { sync_sql: "CREATE TABLE test (id INT);".to_string(), ..Default::default() }; + SchemaDiffPreparation { sync_sql: "CREATE TABLE test (id INT);".to_string(), ..empty_schema_diff() }; let result = generate_enhanced_sync_sql(&schema_diff, DatabaseType::Mysql, None, IdempotentStrategy::IfNotExists); @@ -2295,7 +2293,7 @@ mod tests { sync_sql: "CREATE TABLE users (id INT);".to_string(), rollback_sync_sql: Some("DROP TABLE IF EXISTS users;".to_string()), rollback_graph: Some(graph), - ..Default::default() + ..empty_schema_diff() }; let result = generate_enhanced_rollback_sql( @@ -2315,7 +2313,7 @@ mod tests { sync_sql: "CREATE TABLE test (id INT);".to_string(), rollback_sync_sql: Some("DROP TABLE IF EXISTS test;".to_string()), permission_sync_sql: Some("GRANT SELECT ON test TO reader;".to_string()), - ..Default::default() + ..empty_schema_diff() }; let script = generate_complete_script( @@ -2432,7 +2430,7 @@ mod tests { ..Default::default() }; - let prep = SchemaDiffPreparation { diffs: vec![table_diff], ..Default::default() }; + let prep = SchemaDiffPreparation { diffs: vec![table_diff], ..empty_schema_diff() }; // Seed a pair-specific rule so the test does not depend on CWD for YAML files. { diff --git a/crates/dbx-core/src/sql_file_import.rs b/crates/dbx-core/src/data/sql_file_import.rs similarity index 100% rename from crates/dbx-core/src/sql_file_import.rs rename to crates/dbx-core/src/data/sql_file_import.rs diff --git a/crates/dbx-core/src/sql_file_import/table_restore.rs b/crates/dbx-core/src/data/sql_file_import/table_restore.rs similarity index 98% rename from crates/dbx-core/src/sql_file_import/table_restore.rs rename to crates/dbx-core/src/data/sql_file_import/table_restore.rs index e8a55e18e1..367dfdfc3f 100644 --- a/crates/dbx-core/src/sql_file_import/table_restore.rs +++ b/crates/dbx-core/src/data/sql_file_import/table_restore.rs @@ -1,16 +1,9 @@ use std::collections::BTreeSet; -use serde::{Deserialize, Serialize}; use sqlparser::dialect::MySqlDialect; use sqlparser::tokenizer::{Token, Tokenizer}; -/// The database is the scope in the dump, not the destination selected in the UI. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SqlFileTable { - pub database: Option, - pub name: String, -} +pub use dbx_types::sql_file::SqlFileTable; #[derive(Default)] pub(super) struct TableRestoreFilter { diff --git a/crates/dbx-core/src/sqlite_backup.rs b/crates/dbx-core/src/data/sqlite_backup.rs similarity index 100% rename from crates/dbx-core/src/sqlite_backup.rs rename to crates/dbx-core/src/data/sqlite_backup.rs diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/data/table_export.rs similarity index 100% rename from crates/dbx-core/src/table_export.rs rename to crates/dbx-core/src/data/table_export.rs diff --git a/crates/dbx-core/src/table_import.rs b/crates/dbx-core/src/data/table_import.rs similarity index 99% rename from crates/dbx-core/src/table_import.rs rename to crates/dbx-core/src/data/table_import.rs index be04c8c3cb..5adb23de6f 100644 --- a/crates/dbx-core/src/table_import.rs +++ b/crates/dbx-core/src/data/table_import.rs @@ -7818,7 +7818,7 @@ mod tests { #[test] fn legacy_xls_rejects_numeric_to_text_without_affecting_numeric_targets() { let path = std::env::temp_dir().join(format!("dbx-table-import-formatted-{}.xls", uuid::Uuid::new_v4())); - std::fs::write(&path, include_bytes!("../tests/fixtures/issue3683-formatted-numbers.xls")).unwrap(); + std::fs::write(&path, include_bytes!("../../tests/fixtures/issue3683-formatted-numbers.xls")).unwrap(); let options = TableImportParseOptions { has_header: Some(false), ..TableImportParseOptions::default() }; let numeric = @@ -7846,7 +7846,7 @@ mod tests { #[tokio::test] async fn legacy_xls_preview_reads_sheet_names_without_zip_parser() { let path = std::env::temp_dir().join(format!("dbx-table-import-preview-{}.xls", uuid::Uuid::new_v4())); - std::fs::write(&path, include_bytes!("../tests/fixtures/issue3683-formatted-numbers.xls")).unwrap(); + std::fs::write(&path, include_bytes!("../../tests/fixtures/issue3683-formatted-numbers.xls")).unwrap(); let options = TableImportParseOptions { has_header: Some(false), ..TableImportParseOptions::default() }; let (parsed, non_streaming, sheets) = parse_import_preview_file_with_options( @@ -7916,8 +7916,9 @@ mod tests { write!(sheet, "").unwrap(); sheet.flush().unwrap(); + let helper_test = format!("{}::xlsx_style_rss_helper", module_path!().split_once("::").unwrap().1); let mut child = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", "table_import::tests::xlsx_style_rss_helper", "--nocapture"]) + .args(["--exact", &helper_test, "--nocapture"]) .env("DBX_XLSX_STYLE_RSS_PATH", &sheet_path) .env("DBX_XLSX_STYLE_RSS_READY", &ready_path) .env("DBX_XLSX_STYLE_RSS_GO", &go_path) diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/data/transfer.rs similarity index 99% rename from crates/dbx-core/src/transfer.rs rename to crates/dbx-core/src/data/transfer.rs index b6b4bf250f..7d57be5406 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/data/transfer.rs @@ -1,3 +1,10 @@ +pub use dbx_sql::value_literals::quote_string_literal; +pub use dbx_sql::value_literals::{ + format_ch_array_element, format_ch_array_sql_literal, format_pg_array_element, format_pg_array_sql_literal, + format_postgres_vector_element, format_postgres_vector_sql_literal, is_postgres_vector_type, + quote_postgres_string_literal, +}; + use regex::Regex; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; @@ -962,10 +969,6 @@ async fn resolve_transfer_target_table_name( } } -fn quote_string_literal(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - /// Returns a SQL statement selecting 1 row when `name` exists in `schema` /// on the target side; None-kind support per family mirrors /// `transfer_object_kinds`. @@ -1318,37 +1321,6 @@ pub fn rewrite_oracle_schema_qualifier(ddl: &str, source_schema: &str, target_sc rewrite_double_quoted_schema_qualifier(ddl, source_schema, target_schema) } -pub(crate) fn quote_postgres_string_literal(value: &str) -> String { - if !value.contains('\\') && !value.chars().any(|character| character.is_ascii_control()) { - return quote_string_literal(value); - } - - let mut escaped = String::with_capacity(value.len()); - for character in value.chars() { - match character { - '\\' => escaped.push_str("\\\\"), - '\n' => escaped.push_str("\\n"), - '\r' => escaped.push_str("\\r"), - '\t' => escaped.push_str("\\t"), - '\x08' => escaped.push_str("\\b"), - '\x0c' => escaped.push_str("\\f"), - '\'' => escaped.push_str("''"), - character if character.is_ascii_control() => { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - let byte = character as u8; - escaped.push_str("\\x"); - escaped.push(HEX[(byte >> 4) as usize] as char); - escaped.push(HEX[(byte & 0x0F) as usize] as char); - } - character => escaped.push(character), - } - } - - // Escape string constants keep control characters out of the physical - // script and remain correct regardless of standard_conforming_strings. - format!("E'{escaped}'") -} - fn postgres_schema_exists_sql(schema: &str) -> String { format!("SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = {} LIMIT 1", quote_string_literal(schema)) } @@ -2242,7 +2214,7 @@ fn postgres_index_column_sql( ) -> String { // The base key text: a real column is quoted as an identifier; an expression/functional // key part arrives as raw expression text (the per-column `pg_get_indexdef` omits the - // opclass — see `crates/dbx-core/src/db/postgres.rs`), so quoting the whole thing as + // opclass — see `crates/dbx-drivers/src/db/postgres.rs`), so quoting the whole thing as // an identifier would turn it into a nonexistent column reference (#6295). let base = if is_expression { column.to_string() } else { quote_identifier(column, &DatabaseType::Postgres) }; // The opclass is read separately from `pg_index.indclass` for every key position @@ -2428,7 +2400,7 @@ fn generate_mysql_foreign_key_alter_statements( // and callers that leave `source_schema` empty (the common case for MySQL // transfers) still need something to compare `ForeignKeyInfo.ref_schema` // against. Mirrors `mysql_table_metadata_catalog`'s schema-or-database - // fallback (crates/dbx-core/src/schema.rs), which is private to that module. + // fallback (crates/dbx-core/src/schema/mod.rs), which is private to that module. let source_database = if request.source_schema.trim().is_empty() { request.source_database.as_str() } else { @@ -3101,122 +3073,6 @@ fn oracle_export_date_parts(value: &str) -> Option> { None } -pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String { - if arr.is_empty() { - return "'{}'".to_string(); - } - let elements: Vec = arr.iter().map(format_pg_array_element).collect(); - let inner = format!("{{{}}}", elements.join(",")); - format!("'{}'", inner.replace('\\', "\\\\").replace('\'', "''")) -} - -pub(crate) fn is_postgres_vector_type(column_type: Option<&str>) -> bool { - column_type - .map(|column_type| { - let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase(); - if normalized.trim_end().ends_with("[]") { - return false; - } - let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"'); - matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec") - }) - .unwrap_or(false) -} - -pub(crate) fn format_postgres_vector_sql_literal(value: &serde_json::Value) -> String { - if value.is_null() { - return "NULL".to_string(); - } - let text = match value { - // pgvector vector/halfvec are scalar extension types whose importable - // literal grammar uses square brackets, unlike PostgreSQL arrays. - serde_json::Value::Array(arr) => { - let elements = arr.iter().map(format_postgres_vector_element).collect::>(); - format!("[{}]", elements.join(",")) - } - serde_json::Value::String(text) => text.to_string(), - _ => value.to_string(), - }; - quote_postgres_string_literal(&text) -} - -fn format_postgres_vector_element(value: &serde_json::Value) -> String { - match value { - serde_json::Value::String(text) => text.trim().to_string(), - serde_json::Value::Number(number) => number.to_string(), - serde_json::Value::Bool(value) => value.to_string(), - serde_json::Value::Null => "NULL".to_string(), - _ => value.to_string(), - } -} - -fn format_pg_array_element(val: &serde_json::Value) -> String { - match val { - serde_json::Value::Null => "NULL".to_string(), - serde_json::Value::Array(arr) => { - if arr.is_empty() { - return "{}".to_string(); - } - let elements: Vec = arr.iter().map(format_pg_array_element).collect(); - format!("{{{}}}", elements.join(",")) - } - serde_json::Value::String(s) => { - let escaped = s.replace('\\', "\\\\").replace('"', "\\\""); - format!("\"{}\"", escaped) - } - serde_json::Value::Number(n) => n.to_string(), - serde_json::Value::Bool(b) => { - if *b { - "true".to_string() - } else { - "false".to_string() - } - } - serde_json::Value::Object(o) => { - let json = serde_json::to_string(o).unwrap_or_default(); - let escaped = json.replace('\\', "\\\\").replace('"', "\\\""); - format!("\"{}\"", escaped) - } - } -} - -pub fn format_ch_array_sql_literal(arr: &[serde_json::Value]) -> String { - if arr.is_empty() { - return "[]".to_string(); - } - let elements: Vec = arr.iter().map(format_ch_array_element).collect(); - format!("[{}]", elements.join(",")) -} - -fn format_ch_array_element(val: &serde_json::Value) -> String { - match val { - serde_json::Value::Null => "NULL".to_string(), - serde_json::Value::Array(arr) => { - if arr.is_empty() { - return "[]".to_string(); - } - let elements: Vec = arr.iter().map(format_ch_array_element).collect(); - format!("[{}]", elements.join(",")) - } - serde_json::Value::String(s) => { - let escaped = s.replace('\\', "\\\\").replace('\'', "''"); - format!("'{}'", escaped) - } - serde_json::Value::Number(n) => n.to_string(), - serde_json::Value::Bool(b) => { - if *b { - "true".to_string() - } else { - "false".to_string() - } - } - serde_json::Value::Object(o) => { - let json = serde_json::to_string(o).unwrap_or_default(); - format!("'{}'", json.replace('\\', "\\\\").replace('\'', "''")) - } - } -} - fn format_literal_string(value: &str, db_type: &DatabaseType, column_type: Option<&str>) -> String { if *db_type == DatabaseType::SqlServer { crate::sqlserver_temporal::normalize_sqlserver_temporal_literal(value, column_type) diff --git a/crates/dbx-core/src/transfer/ddl_plan.rs b/crates/dbx-core/src/data/transfer/ddl_plan.rs similarity index 100% rename from crates/dbx-core/src/transfer/ddl_plan.rs rename to crates/dbx-core/src/data/transfer/ddl_plan.rs diff --git a/crates/dbx-core/src/transfer/rebuild_tests.rs b/crates/dbx-core/src/data/transfer/rebuild_tests.rs similarity index 100% rename from crates/dbx-core/src/transfer/rebuild_tests.rs rename to crates/dbx-core/src/data/transfer/rebuild_tests.rs diff --git a/crates/dbx-core/src/transfer_rebuild.rs b/crates/dbx-core/src/data/transfer_rebuild.rs similarity index 100% rename from crates/dbx-core/src/transfer_rebuild.rs rename to crates/dbx-core/src/data/transfer_rebuild.rs diff --git a/crates/dbx-core/src/db/mod.rs b/crates/dbx-core/src/db/mod.rs index c788de078b..531ffdb5a0 100644 --- a/crates/dbx-core/src/db/mod.rs +++ b/crates/dbx-core/src/db/mod.rs @@ -1,459 +1,4 @@ -pub mod agent_driver; -pub mod clickhouse_driver; -pub mod cloudberry; -pub mod cloudflare_d1; +pub use dbx_drivers::db::*; + +pub use crate::data::cloudflare_d1; pub use cloudflare_d1 as cloudflare_d1_driver; -pub(crate) mod ddl_scan; -pub mod document_result; -pub mod dolt; -pub mod doris; -pub mod duckdb_sql; -#[cfg(feature = "duckdb-sidecar")] -pub mod duckdb_worker_process; -#[cfg(feature = "duckdb-sidecar")] -pub mod duckdb_worker_protocol; -#[cfg(feature = "dynamodb")] -#[path = "dynamodb_driver.rs"] -pub mod dynamodb_driver; -#[cfg(not(feature = "dynamodb"))] -#[path = "dynamodb_driver_disabled.rs"] -pub mod dynamodb_driver; -pub mod easysearch_driver; -pub mod elasticsearch_driver; -pub mod elasticsearch_sql; -pub mod file_validator; -pub mod hbase_driver; -pub mod http_tunnel; -pub mod influxdb3_driver; -pub mod influxdb_driver; -pub mod manticoresearch; -pub mod meilisearch_driver; -pub mod mongo_driver; -pub mod mysql; -pub mod mysql_compatible; -pub mod ob_oracle; -pub mod oceanbase_mysql; -pub mod opentenbase; -pub mod postgres; -pub mod proxy_tunnel; -pub mod questdb; -pub mod redis_driver; -pub mod rqlite_driver; -pub mod sqlite; -pub mod sqlite_worker; -pub mod sqlserver; -pub mod ssh_host_key; -pub mod ssh_prompt; -pub mod ssh_tunnel; -pub mod starrocks; -pub mod tdsql_mysql; -pub mod tidb; -pub mod transport_layer_tunnel; -pub mod turso_driver; -pub mod vector_driver; -pub mod victoriametrics_driver; -pub mod wkb; - -use crate::path_utils::expand_tilde; -use reqwest::ClientBuilder; -use reqwest::{Certificate, Identity}; -use std::fmt; -use std::future::Future; -use std::time::Duration; - -// Re-export types so that `db::QueryResult` etc. work within dbx-core -pub use crate::mysql_event_sql::MysqlEventInfo; -pub use crate::types::*; -pub use file_validator::validate_file_path; - -pub const CONNECTION_TIMEOUT_SECS: u64 = 5; -pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PoolCheckoutStage { - Wait, - Create, - Recycle, - Unknown, -} - -impl PoolCheckoutStage { - pub fn as_str(self) -> &'static str { - match self { - Self::Wait => "wait", - Self::Create => "create", - Self::Recycle => "recycle", - Self::Unknown => "unknown", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PoolCheckoutError { - Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration }, - Failed { database: &'static str, stage: PoolCheckoutStage, detail: String }, - Canceled, -} - -impl PoolCheckoutError { - pub fn stage(&self) -> Option { - match self { - Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage), - Self::Canceled => None, - } - } - - pub fn is_pool_saturation(&self) -> bool { - matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. }) - } -} - -impl fmt::Display for PoolCheckoutError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Timeout { database, stage, timeout } => write!( - formatter, - "{database} connection pool checkout timed out [stage={}, timeout_ms={}]", - stage.as_str(), - timeout.as_millis() - ), - Self::Failed { database, stage, detail } => { - write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str()) - } - Self::Canceled => formatter.write_str(crate::query::QUERY_CANCELED), - } - } -} - -pub fn connection_timeout() -> Duration { - Duration::from_secs(CONNECTION_TIMEOUT_SECS) -} - -pub fn http_client_builder(timeout: Duration) -> ClientBuilder { - reqwest::Client::builder().connect_timeout(timeout).no_proxy() -} - -/// Augments an HTTP client builder with custom TLS material for drivers that -/// need to trust a non-system CA or present a client certificate (mTLS). -/// -/// * `ca_cert_path` adds an extra root certificate (PEM bundle) alongside the -/// system roots, so self-signed server certificates are trusted. -/// * `client_cert_path` + `client_key_path` (required together) are concatenated -/// into a single PEM identity, as expected by reqwest's rustls backend. -/// -/// Returns the augmented builder, or a descriptive error if a configured -/// certificate/key file cannot be read or parsed. Callers surface that error -/// instead of silently falling back to the system trust store, so a typo in a -/// cert path fails the connection loudly rather than being ignored. -pub(crate) fn apply_tls_certificates( - mut builder: ClientBuilder, - ca_cert_path: Option<&str>, - client_cert_path: Option<&str>, - client_key_path: Option<&str>, -) -> Result { - // The desktop build unifies reqwest with both `default-tls` and `rustls-tls`, - // and `TlsBackend` defaults to native-tls there, which rejects the PEM identity - // below with `incompatible TLS identity type`. Pin rustls like the Consul and - // etcd-metrics clients so custom CA roots and PEM client identities work in - // every target. - builder = builder.use_rustls_tls(); - if let Some(ca) = ca_cert_path.filter(|path| !path.is_empty()) { - let ca = expand_tilde(ca); - let contents = std::fs::read(&ca) - .map_err(|error| format!("Failed to read Elasticsearch CA certificate at {ca}: {error}"))?; - let certificates = Certificate::from_pem_bundle(&contents) - .map_err(|error| format!("Failed to parse Elasticsearch CA certificate at {ca}: {error}"))?; - for certificate in certificates { - builder = builder.add_root_certificate(certificate); - } - } - if let (Some(cert_path), Some(key_path)) = - (client_cert_path.filter(|path| !path.is_empty()), client_key_path.filter(|path| !path.is_empty())) - { - let cert_path = expand_tilde(cert_path); - let key_path = expand_tilde(key_path); - let mut pem = std::fs::read(&cert_path) - .map_err(|error| format!("Failed to read Elasticsearch client certificate at {cert_path}: {error}"))?; - // reqwest's rustls backend concatenates the cert and key as two PEM - // blocks; ensure they are newline-separated so parsing succeeds even - // when the cert file does not end with a trailing newline. - if !pem.ends_with(b"\n") { - pem.push(b'\n'); - } - pem.extend( - std::fs::read(&key_path) - .map_err(|error| format!("Failed to read Elasticsearch client key at {key_path}: {error}"))?, - ); - let identity = Identity::from_pem(&pem) - .map_err(|error| format!("Failed to parse Elasticsearch client identity: {error}"))?; - builder = builder.identity(identity); - } - Ok(builder) -} - -pub(crate) const JS_MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991; - -pub fn safe_i64_to_json(v: i64) -> serde_json::Value { - if !(-JS_MAX_SAFE_INTEGER..=JS_MAX_SAFE_INTEGER).contains(&v) { - serde_json::Value::String(v.to_string()) - } else { - serde_json::Value::Number(v.into()) - } -} - -pub fn safe_u64_to_json(v: u64) -> serde_json::Value { - if v > JS_MAX_SAFE_INTEGER as u64 { - serde_json::Value::String(v.to_string()) - } else { - serde_json::Value::Number(v.into()) - } -} - -pub fn json_value_for_js(value: serde_json::Value) -> serde_json::Value { - match value { - serde_json::Value::Number(number) => { - if let Some(value) = number.as_i64() { - safe_i64_to_json(value) - } else if let Some(value) = number.as_u64() { - safe_u64_to_json(value) - } else { - let value = number.to_string(); - if value.bytes().all(|byte| byte == b'-' || byte.is_ascii_digit()) { - serde_json::Value::String(value) - } else { - serde_json::Value::Number(number) - } - } - } - serde_json::Value::Array(values) => { - serde_json::Value::Array(values.into_iter().map(json_value_for_js).collect()) - } - serde_json::Value::Object(entries) => { - serde_json::Value::Object(entries.into_iter().map(|(key, value)| (key, json_value_for_js(value))).collect()) - } - value => value, - } -} - -pub(crate) fn hex_encode(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut out = String::with_capacity(bytes.len() * 2); - for &byte in bytes { - out.push(HEX[(byte >> 4) as usize] as char); - out.push(HEX[(byte & 0x0f) as usize] as char); - } - out -} - -pub(crate) fn binary_value_to_json(bytes: &[u8]) -> serde_json::Value { - serde_json::Value::String(format!("0x{}", hex_encode(bytes))) -} - -pub fn tcp_probe_timeout() -> Duration { - Duration::from_secs(TCP_PROBE_TIMEOUT_SECS) -} - -pub fn parse_connect_timeout(url: &str) -> Duration { - parse_connect_timeout_with_fallback(url, connection_timeout()) -} - -pub fn parse_connect_timeout_with_fallback(url: &str, fallback: Duration) -> Duration { - let Some(query) = url.split('?').nth(1) else { - return fallback; - }; - for param in query.split('&') { - let trimmed = param.trim(); - if trimmed.is_empty() { - continue; - } - let (key, value) = match trimmed.split_once('=') { - Some(pair) => pair, - None => continue, - }; - if key.eq_ignore_ascii_case("connect_timeout") - || key.eq_ignore_ascii_case("connectTimeout") - || key.eq_ignore_ascii_case("connection_timeout") - || key.eq_ignore_ascii_case("connectionTimeout") - { - if let Ok(v) = value.parse::() { - if (1..=300).contains(&v) { - return Duration::from_secs(v); - } - } - } - } - fallback -} - -pub async fn with_connection_timeout(label: &str, timeout: Duration, future: F) -> Result -where - F: Future>, -{ - tokio::time::timeout(timeout, future) - .await - .map_err(|_| format!("{label} connection timed out ({}s)", timeout.as_secs()))? -} - -pub async fn probe_tcp_endpoint(label: &str, host: &str, port: u16, timeout: Duration) -> Result<(), String> { - tokio::time::timeout(timeout, tokio::net::TcpStream::connect((host, port))) - .await - .map_err(|_| format!("{label} TCP connection timed out ({}s)", timeout.as_secs()))? - .map(|_| ()) - .map_err(|e| format!("{label} TCP connection failed: {e}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn binary_values_are_displayed_as_prefixed_hex() { - assert_eq!(binary_value_to_json(&[0x00, 0x01, 0xab, 0xff]), serde_json::json!("0x0001abff")); - } - - #[test] - fn arbitrary_precision_integers_are_strings_for_js() { - let value = serde_json::from_str( - r#"{ - "positive": 20240302001417986771, - "negative": -20240302001417986771 - }"#, - ) - .unwrap(); - - assert_eq!( - json_value_for_js(value), - serde_json::json!({ - "positive": "20240302001417986771", - "negative": "-20240302001417986771" - }) - ); - } - - #[test] - fn json_value_for_js_preserves_existing_value_behavior_recursively() { - let value = serde_json::from_str( - r#"{ - "safe": [9007199254740991, -9007199254740991], - "unsafe": [9007199254740992, -9007199254740992], - "non_integer": [1.25, 1e-2], - "other": [null, true, "text"] - }"#, - ) - .unwrap(); - let expected: serde_json::Value = serde_json::from_str( - r#"{ - "safe": [9007199254740991, -9007199254740991], - "unsafe": ["9007199254740992", "-9007199254740992"], - "non_integer": [1.25, 1e-2], - "other": [null, true, "text"] - }"#, - ) - .unwrap(); - - assert_eq!(json_value_for_js(value), expected); - } -} - -#[cfg(test)] -mod tls_tests { - use super::{apply_tls_certificates, http_client_builder}; - use std::path::Path; - use std::process::Command; - use std::time::Duration; - - /// Generates a throwaway CA + client certificate/key (PEM) via openssl and - /// returns their paths. If openssl is unavailable the files stay empty, which - /// makes `apply_tls_certificates` surface a read error instead of degrading. - fn gen_certs(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) { - let ca_key = dir.join("ca.key"); - let ca_pem = dir.join("ca.pem"); - let csr = dir.join("client.csr"); - let client_key = dir.join("client.key"); - let client_crt = dir.join("client.crt"); - // Extensions force an X.509 v3 client cert; some toolchains (e.g. - // OpenSSL 3.0 on CI) emit v1 without one, which rustls rejects. - let client_ext = dir.join("client.ext"); - let _ = std::fs::write(&client_ext, "basicConstraints=critical,CA:FALSE\n"); - let _ = Command::new("openssl") - .args([ - "req", - "-x509", - "-newkey", - "rsa:2048", - "-nodes", - "-keyout", - ca_key.to_str().unwrap(), - "-out", - ca_pem.to_str().unwrap(), - "-days", - "1", - "-subj", - "/CN=dbx-test-ca", - ]) - .output(); - let _ = Command::new("openssl") - .args([ - "req", - "-newkey", - "rsa:2048", - "-nodes", - "-keyout", - client_key.to_str().unwrap(), - "-out", - csr.to_str().unwrap(), - "-subj", - "/CN=dbx-test-client", - ]) - .output(); - let _ = Command::new("openssl") - .args([ - "x509", - "-req", - "-in", - csr.to_str().unwrap(), - "-CA", - ca_pem.to_str().unwrap(), - "-CAkey", - ca_key.to_str().unwrap(), - "-CAcreateserial", - "-out", - client_crt.to_str().unwrap(), - "-days", - "1", - "-extfile", - client_ext.to_str().unwrap(), - ]) - .output(); - (ca_pem, client_crt, client_key) - } - - #[test] - fn apply_tls_certificates_builds_with_ca_and_client_cert() { - let dir = std::env::temp_dir().join(format!("dbx-tls-test-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let (ca, client_crt, client_key) = gen_certs(&dir); - - let built = apply_tls_certificates( - http_client_builder(Duration::from_secs(5)), - ca.to_str(), - client_crt.to_str(), - client_key.to_str(), - ) - .expect("client should build with CA + client cert") - .build(); - assert!(built.is_ok(), "built client should be usable with CA + client cert: {:?}", built.err()); - - // A misconfigured CA path must now surface a descriptive error instead of - // silently degrading to the system trust store. - let missing = apply_tls_certificates( - http_client_builder(Duration::from_secs(5)), - Some("/nonexistent/ca.pem"), - None, - None, - ); - assert!(missing.is_err(), "missing CA path must surface a read error"); - - let _ = std::fs::remove_dir_all(&dir); - } -} diff --git a/crates/dbx-core/src/changelog.rs b/crates/dbx-core/src/host/changelog.rs similarity index 100% rename from crates/dbx-core/src/changelog.rs rename to crates/dbx-core/src/host/changelog.rs diff --git a/crates/dbx-core/src/external/mod.rs b/crates/dbx-core/src/host/external/mod.rs similarity index 100% rename from crates/dbx-core/src/external/mod.rs rename to crates/dbx-core/src/host/external/mod.rs diff --git a/crates/dbx-core/src/external/traits.rs b/crates/dbx-core/src/host/external/traits.rs similarity index 100% rename from crates/dbx-core/src/external/traits.rs rename to crates/dbx-core/src/host/external/traits.rs diff --git a/crates/dbx-core/src/external/types.rs b/crates/dbx-core/src/host/external/types.rs similarity index 100% rename from crates/dbx-core/src/external/types.rs rename to crates/dbx-core/src/host/external/types.rs diff --git a/crates/dbx-core/src/host/mod.rs b/crates/dbx-core/src/host/mod.rs new file mode 100644 index 0000000000..1c13c5ae73 --- /dev/null +++ b/crates/dbx-core/src/host/mod.rs @@ -0,0 +1,3 @@ +pub mod changelog; +pub mod external; +pub mod update; diff --git a/crates/dbx-core/src/update.rs b/crates/dbx-core/src/host/update.rs similarity index 61% rename from crates/dbx-core/src/update.rs rename to crates/dbx-core/src/host/update.rs index 8ba810b681..fe148e8c43 100644 --- a/crates/dbx-core/src/update.rs +++ b/crates/dbx-core/src/host/update.rs @@ -1,3 +1,5 @@ +pub use dbx_platform::version::{is_newer_version, normalize_version, parse_version}; + use serde::{Deserialize, Serialize}; const LATEST_JSON_GITHUB_PATH: &str = "https://github.com/t8y2/dbx/releases/latest/download/latest.json"; @@ -140,121 +142,7 @@ fn build_update_http_client() -> Result { builder.build().map_err(|e| format!("Failed to create HTTP client: {e}")) } -pub fn system_proxy_url() -> Option { - system_proxy_url_from_platform() -} - -#[cfg(target_os = "macos")] -fn system_proxy_url_from_platform() -> Option { - let output = crate::process::new_std_command("scutil").arg("--proxy").output().ok()?; - if !output.status.success() { - return None; - } - let stdout = String::from_utf8(output.stdout).ok()?; - system_proxy_url_from_scutil_output(&stdout) -} - -#[cfg(target_os = "windows")] -fn system_proxy_url_from_platform() -> Option { - let key = r"HKCU\Software\Microsoft\Windows\CurrentVersion\Internet Settings"; - let proxy_enable = - crate::process::new_std_command("reg").args(["query", key, "/v", "ProxyEnable"]).output().ok()?; - let proxy_server = - crate::process::new_std_command("reg").args(["query", key, "/v", "ProxyServer"]).output().ok()?; - if !proxy_enable.status.success() || !proxy_server.status.success() { - return None; - } - let proxy_enable = String::from_utf8(proxy_enable.stdout).ok()?; - let proxy_server = String::from_utf8(proxy_server.stdout).ok()?; - system_proxy_url_from_windows_registry_output(&proxy_enable, &proxy_server) -} - -#[cfg(not(any(target_os = "macos", target_os = "windows")))] -fn system_proxy_url_from_platform() -> Option { - None -} - -#[cfg_attr(not(test), allow(dead_code))] -fn system_proxy_url_from_scutil_output(output: &str) -> Option { - let value = |key: &str| { - output.lines().find_map(|line| { - let (line_key, line_value) = line.split_once(':')?; - (line_key.trim() == key).then(|| line_value.trim()) - }) - }; - - if value("HTTPSEnable") == Some("1") { - if let Some(url) = proxy_url(value("HTTPSProxy")?, value("HTTPSPort")?) { - return Some(url); - } - } - - if value("HTTPEnable") == Some("1") { - if let Some(url) = proxy_url(value("HTTPProxy")?, value("HTTPPort")?) { - return Some(url); - } - } - - None -} - -#[cfg_attr(not(test), allow(dead_code))] -fn system_proxy_url_from_windows_registry_output(proxy_enable: &str, proxy_server: &str) -> Option { - let enabled = proxy_enable - .lines() - .find(|line| line.contains("ProxyEnable"))? - .split_whitespace() - .last() - .is_some_and(|value| value == "0x1" || value == "1"); - if !enabled { - return None; - } - - let server = proxy_server.lines().find(|line| line.contains("ProxyServer"))?.split_whitespace().last()?; - - proxy_url_from_windows_proxy_server(server) -} - -fn proxy_url_from_windows_proxy_server(server: &str) -> Option { - let entries = server.split(';').map(str::trim).filter(|entry| !entry.is_empty()).collect::>(); - - for key in ["https=", "http="] { - if let Some(entry) = entries.iter().find_map(|entry| entry.strip_prefix(key)) { - if let Some(url) = proxy_url_from_host_port(entry) { - return Some(url); - } - } - } - - entries.iter().find(|entry| !entry.contains('=')).and_then(|entry| proxy_url_from_host_port(entry)) -} - -fn proxy_url_from_host_port(value: &str) -> Option { - let value = value.trim(); - if value.starts_with("http://") || value.starts_with("https://") { - return Some(value.to_string()); - } - if value.starts_with("socks://") || value.starts_with("socks5://") || value.starts_with("socks5h://") { - return None; - } - - let (host, port) = if let Some(rest) = value.strip_prefix('[') { - let (host, rest) = rest.split_once(']')?; - let port = rest.strip_prefix(':')?; - (host, port) - } else { - value.rsplit_once(':')? - }; - proxy_url(host, port) -} - -fn proxy_url(host: &str, port: &str) -> Option { - if host.is_empty() || port.parse::().is_err() { - return None; - } - let host = if host.contains(':') && !host.starts_with('[') { format!("[{host}]") } else { host.to_string() }; - Some(format!("http://{host}:{port}")) -} +pub use dbx_platform::proxy::system_proxy_url; async fn fetch_github_release_metadata( client: &reqwest::Client, @@ -322,110 +210,9 @@ fn non_empty(value: Option<&str>) -> Option<&str> { }) } -pub fn normalize_version(version: &str) -> String { - version.trim().trim_start_matches('v').to_string() -} - -pub fn parse_version(version: &str) -> Vec { - normalize_version(version).split(['.', '-', '+']).map(|part| part.parse::().unwrap_or(0)).collect() -} - -pub fn is_newer_version(latest: &str, current: &str) -> bool { - let latest_parts = parse_version(latest); - let current_parts = parse_version(current); - let max_len = latest_parts.len().max(current_parts.len()); - - for i in 0..max_len { - let latest_part = *latest_parts.get(i).unwrap_or(&0); - let current_part = *current_parts.get(i).unwrap_or(&0); - if latest_part > current_part { - return true; - } - if latest_part < current_part { - return false; - } - } - - false -} - #[cfg(test)] mod tests { - use super::{ - build_update_info, is_newer_version, normalize_version, system_proxy_url_from_scutil_output, - system_proxy_url_from_windows_registry_output, GithubReleaseMetadata, TauriRelease, - }; - - #[test] - fn normalizes_tag_versions() { - assert_eq!(normalize_version("v1.2.3"), "1.2.3"); - assert_eq!(normalize_version(" 0.2.0 "), "0.2.0"); - } - - #[test] - fn compares_semver_like_versions() { - assert!(is_newer_version("0.2.1", "0.2.0")); - assert!(is_newer_version("1.0.0", "0.9.9")); - assert!(!is_newer_version("0.2.0", "0.2.0")); - assert!(!is_newer_version("0.1.9", "0.2.0")); - } - - #[test] - fn parses_macos_https_system_proxy() { - let output = r#" { - HTTPEnable : 1 - HTTPPort : 7890 - HTTPProxy : 127.0.0.1 - HTTPSEnable : 1 - HTTPSPort : 7891 - HTTPSProxy : 127.0.0.1 -}"#; - - assert_eq!(system_proxy_url_from_scutil_output(output), Some("http://127.0.0.1:7891".to_string())); - } - - #[test] - fn falls_back_to_macos_http_system_proxy() { - let output = r#" { - HTTPEnable : 1 - HTTPPort : 7890 - HTTPProxy : 127.0.0.1 - HTTPSEnable : 0 -}"#; - - assert_eq!(system_proxy_url_from_scutil_output(output), Some("http://127.0.0.1:7890".to_string())); - } - - #[test] - fn ignores_disabled_or_incomplete_macos_system_proxy() { - assert_eq!(system_proxy_url_from_scutil_output("HTTPEnable : 0\nHTTPProxy : 127.0.0.1\nHTTPPort : 7890"), None); - assert_eq!(system_proxy_url_from_scutil_output("HTTPEnable : 1\nHTTPProxy : 127.0.0.1"), None); - } - - #[test] - fn parses_windows_system_proxy() { - let enabled = r#" -HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Internet Settings - ProxyEnable REG_DWORD 0x1 -"#; - let server = r#" -HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Internet Settings - ProxyServer REG_SZ http=127.0.0.1:7890;https=127.0.0.1:7891 -"#; - - assert_eq!( - system_proxy_url_from_windows_registry_output(enabled, server), - Some("http://127.0.0.1:7891".to_string()) - ); - } - - #[test] - fn ignores_disabled_windows_system_proxy() { - let disabled = "ProxyEnable REG_DWORD 0x0"; - let server = "ProxyServer REG_SZ 127.0.0.1:7890"; - - assert_eq!(system_proxy_url_from_windows_registry_output(disabled, server), None); - } + use super::{build_update_info, GithubReleaseMetadata, TauriRelease}; #[test] fn parses_jdbc_plugin_metadata_from_latest_json() { diff --git a/crates/dbx-core/src/lib.rs b/crates/dbx-core/src/lib.rs index 01b8881ef4..70b353b0f7 100644 --- a/crates/dbx-core/src/lib.rs +++ b/crates/dbx-core/src/lib.rs @@ -1,239 +1,120 @@ #![recursion_limit = "256"] -pub mod agent_catalog; -pub mod agent_connection; -pub mod agent_events; -pub mod agent_explain; -pub mod agent_kv; -pub mod agent_loop; -pub mod agent_manager; -pub mod agent_offline_export; -pub mod agent_recovery; -pub mod agent_runtime; -pub mod agent_service; -pub mod agent_tools; +pub mod admin; pub mod ai; -pub mod ai_claude_code_cli; -pub mod ai_cli_agent; -pub mod ai_codebuddy_cli; -pub mod ai_codex_cli; -pub mod ai_cursor_cli; -pub mod ai_effort; -pub mod ai_grok_cli; -mod ai_model_filter; -pub mod ai_opencode_cli; -pub mod ai_pi_agent_cli; -pub mod ai_qoder_cli; -pub mod backend_error; -pub mod changelog; -pub mod cloud_sync; -pub mod config; +pub mod data; +pub mod host; +pub mod persistence; +pub mod safety; + +pub use dbx_ai_provider::{ + agent_events, ai_claude_code_cli, ai_cli_agent, ai_codebuddy_cli, ai_codex_cli, ai_cursor_cli, ai_effort, + ai_grok_cli, ai_opencode_cli, ai_pi_agent_cli, ai_qoder_cli, token_usage, +}; +pub use dbx_platform::{path_utils, process}; + +pub use ai::agent_explain; +pub use ai::agent_kv; +pub use ai::agent_loop; +pub use ai::agent_tools; +pub use dbx_drivers::agent_catalog; +pub use dbx_drivers::agent_connection; +pub use dbx_drivers::agent_manager; +pub use dbx_drivers::agent_offline_export; +pub use dbx_drivers::agent_recovery; +pub use dbx_drivers::agent_runtime; +pub use dbx_drivers::agent_service; +pub use dbx_drivers::backend_error; +pub use host::changelog; +pub use persistence::cloud_sync; +pub use persistence::config; pub mod connection; -pub mod connection_secrets; -pub mod consul; -pub mod correction; -pub mod csv_export; -pub mod data_compare; -pub mod data_grid_extractors; -pub mod data_grid_sql; -pub mod database_capabilities; -pub mod database_export; -pub mod database_manifest; -pub mod database_search_sql; +pub use admin::consul; +pub use connection::connection_secrets; +pub use data::correction; +pub use data::csv_export; +pub use data::data_compare; +pub use data::database_export; +pub use dbx_drivers::database_capabilities; +pub use dbx_sql::data_grid_extractors; +pub use dbx_sql::data_grid_sql; +pub use dbx_sql::database_search_sql; +pub use dbx_types::database_manifest; pub mod db; -pub mod db_admin_sql; -pub mod dml_binding; -pub mod dml_preview_sql; -pub mod docs; -pub mod document_ops; -pub mod driver_runtime; -pub mod export_runtime; -pub mod export_split_zip; -pub mod external; -pub mod hbase_ops; -pub mod history; -pub mod jdbc; -pub mod mcp_policy; -pub mod models; -pub mod mongo_oidc; -pub mod mongo_ops; -pub mod mongo_shell; -pub mod mongodb_dump; -pub mod mongodb_import_export; #[cfg(feature = "mq-admin")] -pub mod mq; +pub use admin::mq; #[cfg(feature = "mq-admin")] -pub mod mqtt; -pub(crate) mod mysql_ddl_normalize; -pub mod mysql_event_sql; -pub mod nacos; -#[cfg(all(target_os = "windows", target_env = "gnu"))] -mod nanosleep_stub; -pub mod object_cache; -pub mod object_source_sql; -pub mod path_utils; -pub mod plugins; -pub mod process; -pub mod production_safety; -pub mod prompt_template; +pub use admin::mqtt; +pub use admin::nacos; +pub use ai::mcp_policy; +pub use ai::prompt_template; +pub use connection::driver_runtime; +pub use connection::jdbc; +pub use data::docs; +pub use data::export_runtime; +pub use data::mongodb_dump; +pub use data::mongodb_import_export; +pub use dbx_drivers::mongo_oidc; +pub use dbx_drivers::mongo_shell; +pub use dbx_formats::export_split_zip; +pub use dbx_plugin_runtime::plugins; +pub use dbx_sql::db_admin_sql; +pub use dbx_sql::dml_binding; +pub use dbx_sql::dml_preview_sql; +pub(crate) use dbx_sql::mysql_ddl_normalize; +pub use dbx_sql::mysql_event_sql; +pub use dbx_sql::object_source_sql; +pub use dbx_types::models; +pub use host::external; +pub use persistence::history; +pub use query::document_ops; +pub use query::hbase_ops; +pub use query::mongo_ops; +pub use query::object_cache; +pub use safety::production_safety; pub mod query; -pub mod query_cancel; -pub mod query_execution_sql; -pub mod query_result_export; -pub mod query_result_sql; -pub mod redis_ops; -pub mod risk_metrics; -pub mod runtime_config; -pub mod saved_sql; +pub use connection::runtime_config; +pub use data::query_result_export; +pub use dbx_sql::query_execution_sql; +pub use dbx_sql::query_result_sql; +pub use persistence::saved_sql; +pub use query::query_cancel; +pub use query::redis_ops; +pub use safety::risk_metrics; pub mod schema; -pub mod schema_diff; -pub mod script_generator; -pub mod session_credentials; -pub mod sql; -pub mod sql_analysis; -pub mod sql_diagnostics; -pub mod sql_dialect; -pub mod sql_editability; -pub mod sql_error_position; -pub mod sql_file_import; -pub mod sql_file_zip_package; -pub mod sql_parser; -pub mod sql_risk; -pub mod sqlite_backup; -pub(crate) mod sqlserver_temporal; -pub mod ssh_config; -pub mod state_persistence; -pub mod storage; -pub mod table_export; -pub mod table_import; -pub mod table_structure_sql; -pub mod task_supervisor; -pub mod temporal_format; -pub mod text_export; -pub mod token_usage; -pub mod transfer; -pub mod transfer_rebuild; -pub mod two_phase_commit; -pub mod types; -pub mod update; -pub mod write_unlock; -pub mod xlsx_export; +pub use connection::session_credentials; +pub use connection::task_supervisor; +pub use data::script_generator; +pub use data::sql_file_import; +pub use data::sqlite_backup; +pub use data::table_export; +pub use data::table_import; +pub use data::transfer; +pub use data::transfer_rebuild; +pub use dbx_drivers::ssh_config; +pub use dbx_formats::sql_file_zip_package; +pub use dbx_formats::temporal_format; +pub use dbx_formats::text_export; +pub use dbx_formats::xlsx_export; +pub use dbx_sql::schema_diff; +pub use dbx_sql::sql; +pub use dbx_sql::sql_analysis; +pub use dbx_sql::sql_diagnostics; +pub use dbx_sql::sql_dialect; +pub use dbx_sql::sql_editability; +pub use dbx_sql::sql_error_position; +pub use dbx_sql::sql_parser; +pub use dbx_sql::sql_risk; +pub(crate) use dbx_sql::sqlserver_temporal; +pub use dbx_types::types; +pub use host::update; +pub use persistence::state_persistence; +pub use persistence::storage; +pub use query::two_phase_commit; +pub use safety::write_unlock; +pub use schema::table_structure_sql; -pub const R2_CDN_BASE: &str = "https://dl.dbxio.com/"; -pub const GITHUB_RELEASE_DOWNLOAD_PREFIX: &str = "https://github.com/t8y2/dbx/releases/download/"; -pub const CNB_RELEASE_DOWNLOAD_PREFIX: &str = "https://cnb.cool/dbxio.com/dbx/-/releases/download/"; - -#[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq, Hash)] -#[serde(rename_all = "lowercase")] -pub enum DownloadSource { - #[default] - Official, - Cnb, -} - -impl DownloadSource { - pub fn download_candidate_urls(self, github_url: &str, r2_path: &str) -> Result, String> { - match self { - Self::Official => Ok(download_candidate_urls(github_url, r2_path)), - Self::Cnb => Ok(mirror_download_candidate_urls( - r2_path, - rewrite_github_release_url(github_url, CNB_RELEASE_DOWNLOAD_PREFIX)?, - )), - } - } -} - -fn mirror_download_candidate_urls(r2_path: &str, mirror_url: String) -> Vec { - let r2_url = format!("{R2_CDN_BASE}{r2_path}"); - vec![mirror_url, r2_url] -} - -fn rewrite_github_release_url(url: &str, target_prefix: &str) -> Result { - if url.starts_with(target_prefix) { - return Ok(url.to_string()); - } - url.strip_prefix(GITHUB_RELEASE_DOWNLOAD_PREFIX) - .map(|path| format!("{target_prefix}{path}")) - .ok_or_else(|| format!("Unsupported DBX release download URL: {url}")) -} - -pub fn download_candidate_urls(github_url: &str, r2_path: &str) -> Vec { - vec![format!("{R2_CDN_BASE}{r2_path}"), github_url.to_string()] -} - -use std::pin::Pin; - -type ResponseFuture = Pin> + Send>>; - -pub async fn race_download( - client: &reqwest::Client, - github_url: &str, - r2_path: &str, - user_agent: &str, -) -> Result { - race_download_urls(client, &download_candidate_urls(github_url, r2_path), user_agent).await -} - -pub async fn race_download_urls( - client: &reqwest::Client, - urls: &[String], - user_agent: &str, -) -> Result { - use futures::future::select_ok; - - let mut futs: Vec = Vec::with_capacity(urls.len()); - - for url in urls { - let client = client.clone(); - let url = url.clone(); - let ua = user_agent.to_string(); - futs.push(Box::pin(async move { - client - .get(url) - .header(reqwest::header::USER_AGENT, ua) - .header(reqwest::header::ACCEPT_ENCODING, "identity") - .send() - .await - .and_then(|r| r.error_for_status()) - .map_err(|e| format!("{e}")) - }) as ResponseFuture); - } - - match select_ok(futs).await { - Ok((resp, _)) => Ok(resp), - Err(last_err) => Err(last_err), - } -} - -#[cfg(test)] -mod tests { - use super::{download_candidate_urls, DownloadSource}; - - #[test] - fn download_candidates_exclude_third_party_github_proxy() { - let urls = download_candidate_urls( - "https://github.com/t8y2/dbx/releases/latest/download/latest.json", - "releases/latest/latest.json", - ); - - assert_eq!( - urls, - vec![ - "https://dl.dbxio.com/releases/latest/latest.json", - "https://github.com/t8y2/dbx/releases/latest/download/latest.json", - ] - ); - } - - #[test] - fn mirror_download_candidates_prefer_selected_source() { - let github_url = "https://github.com/t8y2/dbx/releases/download/agents-latest/agent-registry.json"; - assert_eq!( - DownloadSource::Cnb.download_candidate_urls(github_url, "agents/agent-registry.json").unwrap(), - vec![ - "https://cnb.cool/dbxio.com/dbx/-/releases/download/agents-latest/agent-registry.json", - "https://dl.dbxio.com/agents/agent-registry.json", - ] - ); - } -} +pub use dbx_platform::download::{ + download_candidate_urls, race_download, race_download_urls, DownloadSource, CNB_RELEASE_DOWNLOAD_PREFIX, + GITHUB_RELEASE_DOWNLOAD_PREFIX, R2_CDN_BASE, +}; diff --git a/crates/dbx-core/src/cloud_sync.rs b/crates/dbx-core/src/persistence/cloud_sync.rs similarity index 100% rename from crates/dbx-core/src/cloud_sync.rs rename to crates/dbx-core/src/persistence/cloud_sync.rs diff --git a/crates/dbx-core/src/config/expression.rs b/crates/dbx-core/src/persistence/config/expression.rs similarity index 100% rename from crates/dbx-core/src/config/expression.rs rename to crates/dbx-core/src/persistence/config/expression.rs diff --git a/crates/dbx-core/src/config/governance.rs b/crates/dbx-core/src/persistence/config/governance.rs similarity index 100% rename from crates/dbx-core/src/config/governance.rs rename to crates/dbx-core/src/persistence/config/governance.rs diff --git a/crates/dbx-core/src/config/layer.rs b/crates/dbx-core/src/persistence/config/layer.rs similarity index 100% rename from crates/dbx-core/src/config/layer.rs rename to crates/dbx-core/src/persistence/config/layer.rs diff --git a/crates/dbx-core/src/config/mod.rs b/crates/dbx-core/src/persistence/config/mod.rs similarity index 100% rename from crates/dbx-core/src/config/mod.rs rename to crates/dbx-core/src/persistence/config/mod.rs diff --git a/crates/dbx-core/src/config/tag.rs b/crates/dbx-core/src/persistence/config/tag.rs similarity index 100% rename from crates/dbx-core/src/config/tag.rs rename to crates/dbx-core/src/persistence/config/tag.rs diff --git a/crates/dbx-core/src/config/trace.rs b/crates/dbx-core/src/persistence/config/trace.rs similarity index 100% rename from crates/dbx-core/src/config/trace.rs rename to crates/dbx-core/src/persistence/config/trace.rs diff --git a/crates/dbx-core/src/history.rs b/crates/dbx-core/src/persistence/history.rs similarity index 100% rename from crates/dbx-core/src/history.rs rename to crates/dbx-core/src/persistence/history.rs diff --git a/crates/dbx-core/src/persistence/mod.rs b/crates/dbx-core/src/persistence/mod.rs new file mode 100644 index 0000000000..d9e5caf1bd --- /dev/null +++ b/crates/dbx-core/src/persistence/mod.rs @@ -0,0 +1,6 @@ +pub mod cloud_sync; +pub mod config; +pub mod history; +pub mod saved_sql; +pub mod state_persistence; +pub mod storage; diff --git a/crates/dbx-core/src/saved_sql.rs b/crates/dbx-core/src/persistence/saved_sql.rs similarity index 100% rename from crates/dbx-core/src/saved_sql.rs rename to crates/dbx-core/src/persistence/saved_sql.rs diff --git a/crates/dbx-core/src/state_persistence.rs b/crates/dbx-core/src/persistence/state_persistence.rs similarity index 100% rename from crates/dbx-core/src/state_persistence.rs rename to crates/dbx-core/src/persistence/state_persistence.rs diff --git a/crates/dbx-core/src/storage.rs b/crates/dbx-core/src/persistence/storage.rs similarity index 99% rename from crates/dbx-core/src/storage.rs rename to crates/dbx-core/src/persistence/storage.rs index 195e1d07f0..95c42e538e 100644 --- a/crates/dbx-core/src/storage.rs +++ b/crates/dbx-core/src/persistence/storage.rs @@ -1,3 +1,8 @@ +pub use dbx_drivers::runtime_config::{ + normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT, DUCKDB_WORKER_MAX_PROCESSES_MAX, + DUCKDB_WORKER_MAX_PROCESSES_MIN, +}; + use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -634,10 +639,6 @@ fn default_sidebar_table_page_size() -> usize { 1000 } -pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1; -pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16; -pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4; - pub const METADATA_CACHE_MEMORY_MIN_MB: usize = 16; pub const METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB: usize = 256; pub const METADATA_CACHE_MEMORY_HARD_MAX_MB: usize = 512; @@ -667,10 +668,6 @@ pub fn default_duckdb_worker_max_processes() -> usize { DUCKDB_WORKER_MAX_PROCESSES_DEFAULT } -pub fn normalize_duckdb_worker_max_processes(value: usize) -> usize { - value.clamp(DUCKDB_WORKER_MAX_PROCESSES_MIN, DUCKDB_WORKER_MAX_PROCESSES_MAX) -} - impl Default for DesktopSettings { fn default() -> Self { Self { diff --git a/crates/dbx-core/src/document_ops.rs b/crates/dbx-core/src/query/document_ops.rs similarity index 98% rename from crates/dbx-core/src/document_ops.rs rename to crates/dbx-core/src/query/document_ops.rs index f3bc89fa96..68fcdd8a90 100644 --- a/crates/dbx-core/src/document_ops.rs +++ b/crates/dbx-core/src/query/document_ops.rs @@ -5,27 +5,7 @@ use crate::db::{dynamodb_driver, easysearch_driver, elasticsearch_driver, mongo_ pub use crate::db::vector_driver::CollectionInfo; -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MongoGridFsFileInfo { - pub id: String, - pub filename: Option, - pub length: i64, - pub chunk_size: i32, - pub upload_date: Option, - pub metadata: Option, - pub md5: Option, - pub content_type: Option, - pub aliases: Option>, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MongoGridFsBucketInfo { - pub name: String, - pub file_count: u64, - pub total_bytes: i64, -} +pub use dbx_types::document::{MongoGridFsBucketInfo, MongoGridFsFileInfo}; /// 侧边栏「查看索引 Mapping / 索引配置 / 索引统计」请求的只读元数据端点。 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] diff --git a/crates/dbx-core/src/hbase_ops.rs b/crates/dbx-core/src/query/hbase_ops.rs similarity index 100% rename from crates/dbx-core/src/hbase_ops.rs rename to crates/dbx-core/src/query/hbase_ops.rs diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query/mod.rs similarity index 97% rename from crates/dbx-core/src/query.rs rename to crates/dbx-core/src/query/mod.rs index 165ec5ce67..a78bebf277 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query/mod.rs @@ -1,3 +1,18 @@ +pub mod document_ops; +pub mod hbase_ops; +pub mod mongo_ops; +pub mod object_cache; +pub mod query_cancel; +pub mod redis_ops; +pub mod two_phase_commit; + +pub use dbx_drivers::execution::{ + await_stream_with_progress_timeout, canceled_error, is_canceled, lock_shared_client_with_wait, + query_timeout_duration, resolve_query_timeout, timeout_error, timeout_error_for, wait_for_query, + wait_for_query_opt, wait_for_query_with_timeout, wait_for_result_opt, wait_for_result_with_timeout, + wait_for_value_opt, DbOperationBudget, StreamProgressClock, MAX_ROWS, QUERY_CANCELED, QUERY_TIMEOUT, +}; + use futures::StreamExt; use mysql_async::prelude::Queryable; use serde::{Deserialize, Serialize}; @@ -7,14 +22,9 @@ use sqlparser::ast::{ use sqlparser::dialect::{GenericDialect, MsSqlDialect, PostgreSqlDialect}; use sqlparser::parser::Parser; use std::collections::{HashMap, HashSet}; -use std::future::Future; use std::ops::ControlFlow; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Condvar, Mutex, -}; +use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; -use tokio::time::timeout; use tokio_util::sync::CancellationToken; use crate::agent_recovery::{RecoveryDecision, RecoveryPolicy, RecoveryScope}; @@ -28,10 +38,7 @@ use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable use crate::sql_dialect::{quote_iris_identifier, resolve_for_db, CAP_TRANSACTIONAL_DDL}; use crate::sql_risk::{classify_sql_risk_for_database, SqlRisk}; -pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30); -pub const MAX_ROWS: usize = 10000; pub const AGENT_PROTOCOL_MAX_ROWS: usize = i32::MAX as usize; -pub const QUERY_CANCELED: &str = "Query canceled"; pub const METADATA_POOL_BUSY_ERROR: &str = "DBX metadata pool is busy; please retry"; pub const MANUAL_TRANSACTION_IDLE_TIMEOUT_SECS: u64 = 300; pub const MANUAL_TRANSACTION_SESSION_NOT_FOUND_ERROR: &str = @@ -713,56 +720,6 @@ fn error_from_query_result(result: &db::QueryResult) -> Option, - pub cancel_timeout: Duration, - pub cleanup_timeout: Duration, -} - -impl DbOperationBudget { - /// Build an execution budget from connection config. - /// checkout/connect/recycle use connect_timeout_secs (clamped to 1s minimum, 300s maximum). - /// query_timeout follows resolve_query_timeout semantics (Some(0) -> None). - /// cancel/cleanup are fixed values and cannot be disabled. - pub fn from_config(connect_timeout_secs: u64, query_timeout_secs: Option) -> Self { - let infra_timeout = Duration::from_secs(connect_timeout_secs.clamp(1, 300)); - Self { - checkout_timeout: infra_timeout, - connect_timeout: infra_timeout, - recycle_timeout: infra_timeout, - query_timeout: resolve_query_timeout(query_timeout_secs), - cancel_timeout: Duration::from_secs(5), - cleanup_timeout: Duration::from_secs(3), - } - } - - pub fn from_connection_config(config: &ConnectionConfig) -> Self { - // effective_query_timeout_secs, not the raw field: it preserves 0 ("no limit") - // and applies the per-database-type floor Cloud Spanner needs for DDL. - Self::from_config(config.effective_connect_timeout_secs(), Some(config.effective_query_timeout_secs())) - } - - /// Use global default values (when no connection config is available). - pub fn with_defaults() -> Self { - let default_infra = db::connection_timeout(); - Self { - checkout_timeout: default_infra, - connect_timeout: default_infra, - recycle_timeout: default_infra, - query_timeout: Some(QUERY_TIMEOUT), - cancel_timeout: Duration::from_secs(5), - cleanup_timeout: Duration::from_secs(3), - } - } -} - /// Check read-only protection for a connection, blocking write SQL statements. /// Only clones the connection name when read-only mode is active, avoiding /// unnecessary allocations otherwise. @@ -1578,19 +1535,6 @@ fn is_os_connection_error(lower: &str) -> bool { false } -pub fn timeout_error() -> String { - timeout_error_for(QUERY_TIMEOUT) -} - -fn timeout_error_for(timeout_duration: Duration) -> String { - let seconds = timeout_duration.as_secs().max(1); - format!("Query timed out after {seconds} seconds") -} - -pub fn canceled_error() -> String { - QUERY_CANCELED.to_string() -} - fn canceled_query_execution_error() -> QueryExecutionError { QueryExecutionError::Canceled { stage: AgentErrorStage::Cancel, operation_outcome: AgentOperationOutcome::Unknown } } @@ -1621,228 +1565,6 @@ fn postgres_transaction_statement_error( } } -pub(crate) struct StreamProgressClock { - started_at: tokio::time::Instant, - last_progress_ms: AtomicU64, - #[cfg(test)] - marked: std::sync::atomic::AtomicBool, -} - -impl StreamProgressClock { - pub(crate) fn new() -> Self { - Self { - started_at: tokio::time::Instant::now(), - last_progress_ms: AtomicU64::new(0), - #[cfg(test)] - marked: std::sync::atomic::AtomicBool::new(false), - } - } - - pub(crate) fn mark(&self) { - self.last_progress_ms.store(self.started_at.elapsed().as_millis() as u64, Ordering::Relaxed); - #[cfg(test)] - self.marked.store(true, Ordering::Relaxed); - } - - /// Whether any progress has been recorded yet. Test-only: production code - /// only needs the derived inactivity window. A row read within the first - /// millisecond records a zero timestamp, so this cannot be derived from - /// `last_progress_ms`. - #[cfg(test)] - pub(crate) fn marked(&self) -> bool { - self.marked.load(Ordering::Relaxed) - } - - fn elapsed_since_progress(&self) -> Duration { - let last_progress_ms = self.last_progress_ms.load(Ordering::Relaxed); - let elapsed_ms = self.started_at.elapsed().as_millis() as u64; - Duration::from_millis(elapsed_ms.saturating_sub(last_progress_ms)) - } -} - -pub(crate) async fn await_stream_with_progress_timeout( - stream_future: F, - timeout: Option, - progress_clock: Arc, - cancel_token: Option<&CancellationToken>, - timeout_message: String, -) -> Result -where - F: Future>, -{ - let Some(timeout) = timeout else { - return match cancel_token { - Some(token) => { - tokio::select! { - biased; - _ = token.cancelled() => Err(canceled_error()), - result = stream_future => result, - } - } - None => stream_future.await, - }; - }; - - tokio::pin!(stream_future); - loop { - // Query timeout is an inactivity budget, not a cap on total stream duration. - let remaining = timeout.saturating_sub(progress_clock.elapsed_since_progress()); - if remaining.is_zero() { - return Err(timeout_message.clone()); - } - let sleep = tokio::time::sleep(remaining); - tokio::pin!(sleep); - - match cancel_token { - Some(token) => { - tokio::select! { - biased; - _ = token.cancelled() => return Err(canceled_error()), - result = &mut stream_future => return result, - _ = &mut sleep => {}, - } - } - None => { - tokio::select! { - biased; - result = &mut stream_future => return result, - _ = &mut sleep => {}, - } - } - } - - if progress_clock.elapsed_since_progress() >= timeout { - return Err(timeout_message); - } - } -} - -pub fn is_canceled(cancel_token: &Option) -> bool { - cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false) -} - -pub async fn wait_for_query(cancel_token: Option, future: F) -> Result -where - F: Future>, -{ - wait_for_query_with_timeout(cancel_token, QUERY_TIMEOUT, future).await -} - -pub async fn wait_for_query_with_timeout( - cancel_token: Option, - timeout_duration: Duration, - future: F, -) -> Result -where - F: Future>, -{ - wait_for_result_with_timeout(cancel_token, timeout_duration, future).await -} - -async fn wait_for_result_with_timeout( - cancel_token: Option, - timeout_duration: Duration, - future: F, -) -> Result -where - F: Future>, -{ - if let Some(token) = cancel_token { - tokio::select! { - biased; - _ = token.cancelled() => Err(canceled_error()), - result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration))?, - } - } else { - timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration))? - } -} - -/// Like `wait_for_query_with_timeout` but with an optional timeout. -/// `None` means no timeout (only cancellation can stop the query). -pub async fn wait_for_query_opt( - cancel_token: Option, - timeout_duration: Option, - future: F, -) -> Result -where - F: Future>, -{ - wait_for_result_opt(cancel_token, timeout_duration, future).await -} - -async fn wait_for_result_opt( - cancel_token: Option, - timeout_duration: Option, - future: F, -) -> Result -where - F: Future>, -{ - match timeout_duration { - Some(d) => wait_for_result_with_timeout(cancel_token, d, future).await, - None => match cancel_token { - Some(token) => { - tokio::select! { - biased; - _ = token.cancelled() => Err(canceled_error()), - result = future => result, - } - } - None => future.await, - }, - } -} - -async fn wait_for_value_opt( - cancel_token: Option, - timeout_duration: Option, - future: F, -) -> Result -where - F: Future, -{ - match timeout_duration { - Some(timeout_duration) => { - if let Some(token) = cancel_token { - tokio::select! { - biased; - _ = token.cancelled() => Err(canceled_error()), - result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration)), - } - } else { - timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration)) - } - } - None => match cancel_token { - Some(token) => { - tokio::select! { - biased; - _ = token.cancelled() => Err(canceled_error()), - result = future => Ok(result), - } - } - None => Ok(future.await), - }, - } -} - -/// Locks a mutex-guarded shared connection (e.g. SQL Server's single connection -/// per pool key) and reports how long the caller waited for the lock alongside -/// the guard. Callers should fold the returned wait time into any execution-time -/// metric they report, since a driver-level timer that only starts once the lock -/// is held cannot see time spent queued behind another operation on the same -/// connection. -async fn lock_shared_client_with_wait<'a, T>( - client: &'a Arc>, - cancel_token: Option, - timeout_duration: Option, -) -> Result<(tokio::sync::MutexGuard<'a, T>, u128), String> { - let started_at = std::time::Instant::now(); - let guard = wait_for_value_opt(cancel_token, timeout_duration, client.lock()).await?; - Ok((guard, started_at.elapsed().as_millis())) -} - async fn sqlserver_pool_is_current( state: &AppState, pool_key: &str, @@ -1851,18 +1573,6 @@ async fn sqlserver_pool_is_current( matches!(state.pool_handle(pool_key).await, Some(PoolKind::SqlServer(current)) if Arc::ptr_eq(¤t, client)) } -pub fn query_timeout_duration(timeout_secs: Option) -> Option { - match timeout_secs { - Some(0) => None, - Some(n) => Some(Duration::from_secs(n)), - None => Some(QUERY_TIMEOUT), - } -} - -fn resolve_query_timeout(timeout_secs: Option) -> Option { - query_timeout_duration(timeout_secs) -} - fn query_pool_database<'a>(database: &'a str, catalog: Option<&str>) -> Option<&'a str> { if database.is_empty() || catalog.is_some() { None @@ -6575,6 +6285,8 @@ pub async fn rollback_manual_transaction(state: &AppState, txn_session_id: &str) mod tests { use super::*; use crate::query_cancel::RunningTaskMetadata; + use std::sync::atomic::Ordering; + use tokio::time::timeout; mod manual_transaction_snapshot_tests { use super::*; diff --git a/crates/dbx-core/src/mongo_ops.rs b/crates/dbx-core/src/query/mongo_ops.rs similarity index 100% rename from crates/dbx-core/src/mongo_ops.rs rename to crates/dbx-core/src/query/mongo_ops.rs diff --git a/crates/dbx-core/src/object_cache.rs b/crates/dbx-core/src/query/object_cache.rs similarity index 100% rename from crates/dbx-core/src/object_cache.rs rename to crates/dbx-core/src/query/object_cache.rs diff --git a/crates/dbx-core/src/query_cancel.rs b/crates/dbx-core/src/query/query_cancel.rs similarity index 100% rename from crates/dbx-core/src/query_cancel.rs rename to crates/dbx-core/src/query/query_cancel.rs diff --git a/crates/dbx-core/src/redis_ops.rs b/crates/dbx-core/src/query/redis_ops.rs similarity index 100% rename from crates/dbx-core/src/redis_ops.rs rename to crates/dbx-core/src/query/redis_ops.rs diff --git a/crates/dbx-core/src/two_phase_commit.rs b/crates/dbx-core/src/query/two_phase_commit.rs similarity index 100% rename from crates/dbx-core/src/two_phase_commit.rs rename to crates/dbx-core/src/query/two_phase_commit.rs diff --git a/crates/dbx-core/src/safety/mod.rs b/crates/dbx-core/src/safety/mod.rs new file mode 100644 index 0000000000..ca6058e09c --- /dev/null +++ b/crates/dbx-core/src/safety/mod.rs @@ -0,0 +1,3 @@ +pub mod production_safety; +pub mod risk_metrics; +pub mod write_unlock; diff --git a/crates/dbx-core/src/production_safety.rs b/crates/dbx-core/src/safety/production_safety.rs similarity index 99% rename from crates/dbx-core/src/production_safety.rs rename to crates/dbx-core/src/safety/production_safety.rs index 66d38c76f2..9d18b3e021 100644 --- a/crates/dbx-core/src/production_safety.rs +++ b/crates/dbx-core/src/safety/production_safety.rs @@ -962,7 +962,7 @@ mod tests { #[test] fn matches_shared_sql_target_safety_corpus() { let corpus: Vec = - serde_json::from_str(include_str!("../../../tests/fixtures/production-safety-corpus.json")) + serde_json::from_str(include_str!("../../../../tests/fixtures/production-safety-corpus.json")) .expect("production safety corpus is valid JSON"); for corpus_case in corpus { diff --git a/crates/dbx-core/src/risk_metrics.rs b/crates/dbx-core/src/safety/risk_metrics.rs similarity index 100% rename from crates/dbx-core/src/risk_metrics.rs rename to crates/dbx-core/src/safety/risk_metrics.rs diff --git a/crates/dbx-core/src/safety_report.rs b/crates/dbx-core/src/safety/safety_report.rs similarity index 100% rename from crates/dbx-core/src/safety_report.rs rename to crates/dbx-core/src/safety/safety_report.rs diff --git a/crates/dbx-core/src/write_unlock.rs b/crates/dbx-core/src/safety/write_unlock.rs similarity index 100% rename from crates/dbx-core/src/write_unlock.rs rename to crates/dbx-core/src/safety/write_unlock.rs diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema/mod.rs similarity index 99% rename from crates/dbx-core/src/schema.rs rename to crates/dbx-core/src/schema/mod.rs index 41c9d64662..8c5877f673 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema/mod.rs @@ -1,3 +1,7 @@ +pub mod table_structure_sql; + +pub use dbx_drivers::metadata::sqlite_ddl; + use crate::agent_recovery::{RecoveryDecision, RecoveryPolicy, RecoveryScope}; use crate::connection::{ connection_url_for_endpoint, database_connection_config, gaussdb_uses_m_jdbc_driver, task_client_session_id, @@ -96,78 +100,9 @@ const TDENGINE_COMMENT_SEARCH_TIMEOUT: Duration = Duration::from_secs(5); const TDENGINE_COMMENT_SEARCH_CACHE_TTL: Duration = Duration::from_secs(10); const TDENGINE_LIKE_PATTERN_MAX_BYTES: usize = 100; -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct TableNameFilter { - pub include_patterns: Vec, - pub exclude_patterns: Vec, -} - -impl TableNameFilter { - pub fn is_empty(&self) -> bool { - self.include_patterns.iter().all(|pattern| pattern.trim().is_empty()) - && self.exclude_patterns.iter().all(|pattern| pattern.trim().is_empty()) - } -} - -fn sql_like_pattern_matches_case_insensitive(pattern: &str, value: &str) -> bool { - #[derive(Clone, Copy)] - enum LikeToken { - Any, - One, - Literal(char), - } - - let mut tokens = Vec::new(); - let pattern = pattern.trim().to_lowercase(); - let mut pattern_chars = pattern.chars().peekable(); - while let Some(ch) = pattern_chars.next() { - match ch { - '%' => tokens.push(LikeToken::Any), - '_' => tokens.push(LikeToken::One), - '\\' => tokens.push(LikeToken::Literal(pattern_chars.next().unwrap_or('\\'))), - literal => tokens.push(LikeToken::Literal(literal)), - } - } - let value_chars: Vec = value.to_lowercase().chars().collect(); - - let mut previous = vec![false; value_chars.len() + 1]; - previous[0] = true; - for token in tokens { - let mut current = vec![false; value_chars.len() + 1]; - match token { - LikeToken::Any => { - current[0] = previous[0]; - for value_index in 1..=value_chars.len() { - current[value_index] = previous[value_index] || current[value_index - 1]; - } - } - LikeToken::One => { - current[1..].copy_from_slice(&previous[..value_chars.len()]); - } - LikeToken::Literal(literal) => { - for value_index in 1..=value_chars.len() { - current[value_index] = previous[value_index - 1] && value_chars[value_index - 1] == literal; - } - } - } - previous = current; - } - previous[value_chars.len()] -} - -pub fn table_name_filter_matches(name: &str, filter: Option<&TableNameFilter>) -> bool { - let Some(filter) = filter.filter(|filter| !filter.is_empty()) else { - return true; - }; - let include_patterns: Vec<&str> = - filter.include_patterns.iter().map(|pattern| pattern.trim()).filter(|pattern| !pattern.is_empty()).collect(); - let exclude_patterns: Vec<&str> = - filter.exclude_patterns.iter().map(|pattern| pattern.trim()).filter(|pattern| !pattern.is_empty()).collect(); - let included = include_patterns.is_empty() - || include_patterns.iter().any(|pattern| sql_like_pattern_matches_case_insensitive(pattern, name)); - included && !exclude_patterns.iter().any(|pattern| sql_like_pattern_matches_case_insensitive(pattern, name)) -} +pub use dbx_types::metadata_filter::{ + sql_like_pattern_matches_case_insensitive, table_name_filter_matches, TableNameFilter, +}; fn clickhouse_metadata_database<'a>(database: &'a str, schema: &'a str) -> &'a str { if schema.trim().is_empty() { @@ -4314,7 +4249,7 @@ done fn mysql_object_source_sql_emits_show_create_materialized_view() { // Regression for the review comment: Doris / StarRocks ride on the MySQL // protocol, so the MV branch of mysql_object_source_sql must produce a - // real statement (used at crates/dbx-core/src/schema.rs:5395-5404 by + // real statement (used at crates/dbx-core/src/schema/mod.rs:5395-5404 by // get_table_ddl_core). Returning an empty string silently broke the UI. assert_eq!( mysql_object_source_sql("shop", "daily_sales_mv", &db::ObjectSourceKind::MaterializedView), @@ -12325,21 +12260,6 @@ fn ensure_display_ddl_terminated(sql: String) -> String { } } -pub async fn sqlite_ddl(pool: &db::sqlite::SqliteHandle, schema: &str, table: &str) -> Result { - let pool = pool.clone(); - let schema = schema.to_string(); - let table = table.to_string(); - tokio::task::spawn_blocking(move || { - pool.with_connection(|conn| { - let schema = db::sqlite::sqlite_quote_schema_ident_for_connection(conn, &schema)?; - let sql = format!("SELECT sql FROM {}.sqlite_master WHERE type='table' AND name=?1", schema); - conn.query_row(&sql, [table], |row| row.get::<_, String>(0)).map_err(|e| e.to_string()) - }) - }) - .await - .map_err(|e| e.to_string())? -} - /// DDL for a single relation. Callers that already iterate a relation set /// themselves (database export, table transfer) must use this rather than /// `pg_ddl_with_partitions` — recursing into partition children here would @@ -13365,7 +13285,7 @@ fn render_postgres_table_ddl_with_constraints_and_partition_info( .map(|(i, c)| { // A real column is quoted via `pg_ident`; an expression/functional key part // arrives as raw expression text (the per-column `pg_get_indexdef` omits the - // opclass — see `crates/dbx-core/src/db/postgres.rs`), so quoting the whole + // opclass — see `crates/dbx-drivers/src/db/postgres.rs`), so quoting the whole // thing as an identifier would turn it into a nonexistent column reference // (#6295). let is_expr = idx.key_is_expression.get(i).copied().unwrap_or(false); diff --git a/crates/dbx-core/src/osc_probe.rs b/crates/dbx-core/src/schema/osc_probe.rs similarity index 100% rename from crates/dbx-core/src/osc_probe.rs rename to crates/dbx-core/src/schema/osc_probe.rs diff --git a/crates/dbx-core/src/state_calibrator.rs b/crates/dbx-core/src/schema/state_calibrator.rs similarity index 100% rename from crates/dbx-core/src/state_calibrator.rs rename to crates/dbx-core/src/schema/state_calibrator.rs diff --git a/crates/dbx-core/src/schema/table_structure_sql.rs b/crates/dbx-core/src/schema/table_structure_sql.rs new file mode 100644 index 0000000000..aeaa08cbdf --- /dev/null +++ b/crates/dbx-core/src/schema/table_structure_sql.rs @@ -0,0 +1,4 @@ +mod sqlite_rebuild; + +pub use dbx_sql::table_structure_sql::*; +pub use sqlite_rebuild::{apply_sqlite_table_structure_change, preview_sqlite_table_structure_change}; diff --git a/crates/dbx-core/src/table_structure_sql/sqlite_rebuild.rs b/crates/dbx-core/src/schema/table_structure_sql/sqlite_rebuild.rs similarity index 99% rename from crates/dbx-core/src/table_structure_sql/sqlite_rebuild.rs rename to crates/dbx-core/src/schema/table_structure_sql/sqlite_rebuild.rs index b03c50dae1..8e4e84a93f 100644 --- a/crates/dbx-core/src/table_structure_sql/sqlite_rebuild.rs +++ b/crates/dbx-core/src/schema/table_structure_sql/sqlite_rebuild.rs @@ -4,9 +4,9 @@ use std::time::Instant; use rusqlite::{Connection, OptionalExtension}; use sha2::{Digest, Sha256}; -use super::indexes::has_existing_index_change; -use super::types::{SqliteTableStructurePreview, TableStructureSqlOptions}; -use super::util::{clean, normalize_default, original_comment, original_default}; +use super::has_existing_index_change; +use super::{clean, normalize_default, original_comment, original_default}; +use super::{SqliteTableStructurePreview, TableStructureSqlOptions}; use crate::connection::{AppState, PoolKind}; use crate::db; use crate::models::connection::DatabaseType; @@ -134,7 +134,7 @@ fn build_change_plan(conn: &mut Connection, options: &TableStructureSqlOptions) } let snapshot = load_schema_snapshot(conn, schema, &options.table_name)?; - let mut warnings = super::validation::validate_draft(options); + let mut warnings = super::validate_draft(options); let mut type_changes = BTreeMap::new(); let live_columns = snapshot.columns.iter().map(|column| (column.name.to_ascii_lowercase(), column)).collect::>(); diff --git a/crates/dbx-core/tests/agent_recovery_contract.rs b/crates/dbx-core/tests/agent_recovery_contract.rs new file mode 100644 index 0000000000..c1f718baab --- /dev/null +++ b/crates/dbx-core/tests/agent_recovery_contract.rs @@ -0,0 +1,18 @@ +#[cfg(test)] +mod tests { + #[test] + fn business_modules_do_not_consume_legacy_marker_fields() { + let business_sources = [ + ("query.rs", include_str!("../src/query/mod.rs")), + ("schema.rs", include_str!("../src/schema/mod.rs")), + ("connection.rs", include_str!("../src/connection/mod.rs")), + ]; + let retired_consumers = ["agent_rpc_error_category", "agent_rpc_error_session_id", "agent_session_disposition"]; + + for (name, source) in business_sources { + for consumer in retired_consumers { + assert!(!source.contains(consumer), "{name} still consumes {consumer}"); + } + } + } +} diff --git a/crates/dbx-core/tests/connection_url_compatibility.rs b/crates/dbx-core/tests/connection_url_compatibility.rs new file mode 100644 index 0000000000..35900a96fd --- /dev/null +++ b/crates/dbx-core/tests/connection_url_compatibility.rs @@ -0,0 +1,213 @@ +use dbx_core::models::connection::{ + default_connect_timeout_secs, default_idle_timeout_secs, default_keepalive_interval_secs, + default_query_timeout_secs, default_redis_key_separator, ConnectionConfig, DatabaseType, +}; +use std::str::FromStr; + +fn mysql_config(username: &str, password: &str, database: Option<&str>) -> ConnectionConfig { + ConnectionConfig { + docs_notes_path: None, + id: "id".to_string(), + name: "name".to_string(), + note: String::new(), + db_type: DatabaseType::Mysql, + driver_profile: None, + driver_label: None, + url_params: None, + agent_java_options: Vec::new(), + host: "10.1.2.3".to_string(), + port: 2883, + username: username.to_string(), + password: password.to_string(), + database: database.map(str::to_string), + default_schema: None, + visible_databases: None, + visible_database_patterns: None, + visible_schemas: None, + show_system_schemas: false, + attached_databases: Vec::new(), + init_script: None, + color: None, + transport_layers: Vec::new(), + connect_timeout_secs: default_connect_timeout_secs(), + query_timeout_secs: default_query_timeout_secs(), + idle_timeout_secs: default_idle_timeout_secs(), + keepalive_interval_secs: default_keepalive_interval_secs(), + ssl: false, + ca_cert_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + sysdba: false, + oracle_connection_type: None, + connection_string: None, + redis_connection_mode: None, + redis_sentinel_master: String::new(), + redis_sentinel_nodes: String::new(), + redis_sentinel_username: String::new(), + redis_sentinel_password: String::new(), + redis_sentinel_tls: false, + redis_cluster_nodes: String::new(), + redis_key_separator: default_redis_key_separator(), + redis_scan_page_size: None, + redis_database_aliases: Default::default(), + redis_key_templates: Vec::new(), + redis_key_grouping: None, + etcd_endpoints: String::new(), + gbase_server: String::new(), + informix_server: String::new(), + external_config: None, + plugin_id: None, + plugin_connection_provider: None, + plugin_connection_type: None, + connection_secrets: Default::default(), + jdbc_driver_class: None, + jdbc_driver_paths: Vec::new(), + one_time: false, + save_password: true, + read_only: false, + is_production: false, + production_databases: vec![], + database_info: None, + } +} + +#[test] +fn doris_database_type_enables_cleartext_password_auth_for_direct_and_tunneled_urls() { + let mut config = mysql_config("root", "secret", Some("analytics")); + config.db_type = DatabaseType::Doris; + config.port = 9030; + + let direct_url = config.connection_url(); + assert_eq!(direct_url, "mysql://root:secret@10.1.2.3:9030/analytics?enable_cleartext_plugin=true"); + assert!(mysql_async::Opts::from_url(&direct_url).unwrap().enable_cleartext_plugin()); + assert_eq!( + config.connection_url_with_host("127.0.0.1", 19030), + "mysql://root:secret@127.0.0.1:19030/analytics?enable_cleartext_plugin=true" + ); +} + +#[test] +fn postgres_url_normalizes_timezone_param_into_options() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("sslmode=require&timezone=Asia/Shanghai".to_string()); + + assert_eq!( + config.connection_url(), + "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=require&options=%2Dc%20TimeZone%3DAsia%2FShanghai" + ); + let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); + assert_eq!(pg_config.get_options(), Some("-c TimeZone=Asia/Shanghai")); +} + +#[test] +fn postgres_url_maps_schema_param_into_search_path_options() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("schema=public".to_string()); + + assert_eq!( + config.connection_url(), + "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dpublic" + ); + let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); + assert_eq!(pg_config.get_options(), Some("-c search_path=public")); +} + +#[test] +fn postgres_url_maps_current_schema_param_into_search_path_options() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("currentSchema=app".to_string()); + + assert_eq!( + config.connection_url(), + "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dapp" + ); + let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); + assert_eq!(pg_config.get_options(), Some("-c search_path=app")); +} + +#[test] +fn postgres_url_ignores_jdbc_stringtype_param() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("currentSchema=public&stringtype=unspecified".to_string()); + + assert_eq!(config.validate_native_url_params(), Ok(())); + assert_eq!( + config.connection_url(), + "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dpublic" + ); + let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); + assert_eq!(pg_config.get_options(), Some("-c search_path=public")); +} + +#[test] +fn postgres_url_accepts_encoded_jdbc_varchar_stringtype_param() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("currentSchema=app&%73tringtype=%76aRcHaR".to_string()); + + assert_eq!(config.validate_native_url_params(), Ok(())); + assert_eq!( + config.connection_url(), + "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dapp" + ); + let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); + assert_eq!(pg_config.get_options(), Some("-c search_path=app")); +} + +#[test] +fn postgres_url_uses_only_the_structured_endpoint() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some( + "HOST=origin.example.com&%68ostaddr=203.0.113.10&%70ort=6432¤tSchema=app&application_name=dbx" + .to_string(), + ); + + let url = config.connection_url_with_host("127.0.0.1", 6543); + + assert_eq!( + url, + "postgres://postgres:secret@127.0.0.1:6543/test?sslmode=prefer&application_name=dbx&options=%2Dc%20search%5Fpath%3Dapp" + ); + let pg_config = tokio_postgres::Config::from_str(&url).unwrap(); + assert_eq!(pg_config.get_hosts().len(), 1); + assert_eq!(pg_config.get_ports(), &[6543]); + assert!(pg_config.get_hostaddrs().is_empty()); + assert_eq!(pg_config.get_options(), Some("-c search_path=app")); +} + +#[test] +fn postgres_url_query_password_keeps_priority_but_is_redacted() { + let mut config = mysql_config("postgres", "field-secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("%70assword=query-secret&application_name=dbx".to_string()); + + let url = config.connection_url(); + let pg_config = tokio_postgres::Config::from_str(&url).unwrap(); + assert_eq!(pg_config.get_password(), Some(b"query-secret".as_slice())); + assert_eq!(config.redacted_connection_url(), "postgres://10.1.2.3:2883/test?sslmode=prefer&application_name=dbx"); +} + +#[test] +fn postgres_url_ignores_mysql_only_params_from_saved_connections() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("ssl-mode=preferred&charset=utf8mb4".to_string()); + + assert_eq!(config.connection_url(), "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer"); + tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); +} + +#[test] +fn postgres_url_maps_mysql_ssl_mode_require_to_sslmode() { + let mut config = mysql_config("postgres", "secret", Some("test")); + config.db_type = DatabaseType::Postgres; + config.url_params = Some("ssl-mode=required&verify_ca=false&verify_identity=false".to_string()); + + assert_eq!(config.connection_url(), "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=require"); + tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); +} diff --git a/crates/dbx-core/tests/live_postgres_schema_contracts.rs b/crates/dbx-core/tests/live_postgres_schema_contracts.rs new file mode 100644 index 0000000000..101c1848ad --- /dev/null +++ b/crates/dbx-core/tests/live_postgres_schema_contracts.rs @@ -0,0 +1,651 @@ +use dbx_core::db::postgres::*; +use std::time::Duration; + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] +async fn postgres_owned_serial_table_ddl_round_trips_without_external_sequences() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let suffix = uuid::Uuid::new_v4().simple(); + let schema = format!("dbx 6405 \"{suffix}"); + let schema_ident = pg_quote_ident(&schema); + let table_name = "order\"items"; + let table_ident = pg_quote_ident(table_name); + let table = format!("{schema_ident}.{table_ident}"); + let custom_sequence_name = "custom\"source"; + let custom_sequence = format!("{schema_ident}.{}", pg_quote_ident(custom_sequence_name)); + let custom_table_name = "custom defaults"; + let custom_table = format!("{schema_ident}.{}", pg_quote_ident(custom_table_name)); + let owned_custom_sequence_name = "owned custom source"; + let owned_custom_sequence = format!("{schema_ident}.{}", pg_quote_ident(owned_custom_sequence_name)); + let owned_custom_table_name = "owned custom defaults"; + let owned_custom_table = format!("{schema_ident}.{}", pg_quote_ident(owned_custom_table_name)); + let dropped_default_table_name = "dropped default"; + let dropped_default_table = format!("{schema_ident}.{}", pg_quote_ident(dropped_default_table_name)); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE SCHEMA {schema_ident}; \ + CREATE TABLE {table} (\ + \"small\"\"id\" smallserial, \ + \"regular id\" serial, \ + \"large\"\"id\" bigserial, \ + identity_id bigint GENERATED BY DEFAULT AS IDENTITY, \ + base bigint DEFAULT 2, \ + generated bigint GENERATED ALWAYS AS (base * 2) STORED\ + ); \ + CREATE SEQUENCE {custom_sequence}; \ + CREATE TABLE {custom_table} (id bigint DEFAULT nextval({}::regclass)); \ + CREATE TABLE {owned_custom_table} (id bigint NOT NULL); \ + CREATE SEQUENCE {owned_custom_sequence} OWNED BY {owned_custom_table}.id; \ + ALTER TABLE {owned_custom_table} ALTER COLUMN id \ + SET DEFAULT nextval({}::regclass) + 7; \ + CREATE TABLE {dropped_default_table} (id serial); \ + ALTER TABLE {dropped_default_table} ALTER COLUMN id DROP DEFAULT", + pg_quote_literal(&custom_sequence), + pg_quote_literal(&owned_custom_sequence), + )) + .await + .expect("create serial probe objects"); + + let ddl = dbx_core::schema::pg_ddl(&pool, &schema, table_name).await.expect("read serial table ddl"); + let custom_ddl = + dbx_core::schema::pg_ddl(&pool, &schema, custom_table_name).await.expect("read custom-default table ddl"); + let owned_custom_ddl = dbx_core::schema::pg_ddl(&pool, &schema, owned_custom_table_name) + .await + .expect("read owned custom-default table ddl"); + let dropped_default_ddl = dbx_core::schema::pg_ddl(&pool, &schema, dropped_default_table_name) + .await + .expect("read dropped-default table ddl"); + + assert!(ddl.contains("\"small\"\"id\" smallserial NOT NULL"), "ddl: {ddl}"); + assert!(ddl.contains("\"regular id\" serial NOT NULL"), "ddl: {ddl}"); + assert!(ddl.contains("\"large\"\"id\" bigserial NOT NULL"), "ddl: {ddl}"); + assert!(ddl.contains("\"identity_id\" bigint generated by default as identity"), "ddl: {ddl}"); + assert!(ddl.contains("\"generated\" bigint generated always as ((base * 2)) stored"), "ddl: {ddl}"); + assert!(!ddl.contains("nextval("), "ddl: {ddl}"); + assert!(custom_ddl.contains("bigint DEFAULT nextval("), "ddl: {custom_ddl}"); + assert!(!custom_ddl.contains("bigserial"), "ddl: {custom_ddl}"); + assert!(owned_custom_ddl.contains("bigint NOT NULL DEFAULT (nextval("), "ddl: {owned_custom_ddl}"); + assert!(owned_custom_ddl.contains(" + 7)"), "ddl: {owned_custom_ddl}"); + assert!(!owned_custom_ddl.contains("bigserial"), "ddl: {owned_custom_ddl}"); + assert!(dropped_default_ddl.contains("\"id\" integer NOT NULL"), "ddl: {dropped_default_ddl}"); + assert!(!dropped_default_ddl.contains("serial"), "ddl: {dropped_default_ddl}"); + assert!(!dropped_default_ddl.contains("DEFAULT"), "ddl: {dropped_default_ddl}"); + + client.batch_execute(&format!("DROP TABLE {table} CASCADE; {ddl}")).await.expect("replay copied table ddl"); + let inserted = client + .query_one( + &format!( + "INSERT INTO {table} DEFAULT VALUES RETURNING \"small\"\"id\", \"regular id\", \"large\"\"id\", identity_id, generated" + ), + &[], + ) + .await + .expect("insert with replayed sequence defaults"); + assert_eq!(inserted.get::<_, i16>(0), 1); + assert_eq!(inserted.get::<_, i32>(1), 1); + assert_eq!(inserted.get::<_, i64>(2), 1); + assert_eq!(inserted.get::<_, i64>(3), 1); + assert_eq!(inserted.get::<_, i64>(4), 4); + + client.batch_execute(&format!("DROP SCHEMA {schema_ident} CASCADE")).await.expect("drop serial probe schema"); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] +async fn postgres_column_metadata_dependency_joins_return_each_column_once() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let suffix = uuid::Uuid::new_v4().simple(); + let schema = format!("dbx 6547 \"{suffix}"); + let schema_ident = pg_quote_ident(&schema); + let table_name = "issue6547_repro"; + let table_ident = pg_quote_ident(table_name); + let table = format!("{schema_ident}.{table_ident}"); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE SCHEMA {schema_ident}; \ + CREATE TABLE {table} (\ + id serial PRIMARY KEY, \ + value text, \ + small_id smallserial, \ + big_id bigserial, \ + ident_id bigint GENERATED BY DEFAULT AS IDENTITY, \ + plain_int integer, \ + vc varchar(32), \ + num numeric(10, 2), \ + ts timestamp\ + ); \ + CREATE SEQUENCE {schema_ident}.serial_decoy OWNED BY {table}.id; \ + CREATE SEQUENCE {schema_ident}.identity_decoy OWNED BY {table}.ident_id; \ + CREATE INDEX idx_issue6547_repro_id ON {table}(id); \ + CREATE INDEX idx_issue6547_repro_big ON {table}(big_id); \ + CREATE UNIQUE INDEX uniq_issue6547_repro_vc ON {table}(vc);" + )) + .await + .expect("create repro table"); + + let columns = get_columns(&pool, &schema, table_name).await.expect("get columns"); + // One pg_attribute row must produce exactly one ColumnInfo even when + // the column has multiple sequence dependencies and several indexes. + // Only the sequence used by the default, or the internal identity + // sequence, is semantic metadata for the column. + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for column in &columns { + assert!(seen.insert(column.name.as_str()), "duplicate column {:?} in {columns:?}", column.name); + } + assert_eq!(columns.len(), 9, "expected 9 columns, got {columns:?}"); + + for (name, expected_extra) in [("id", "serial"), ("small_id", "smallserial"), ("big_id", "bigserial")] { + let column = columns.iter().find(|c| c.name == name).unwrap_or_else(|| panic!("missing {name}")); + assert_eq!(column.extra.as_deref(), Some(expected_extra), "{name} extra mismatch"); + } + let ident = columns.iter().find(|c| c.name == "ident_id").expect("ident_id column"); + assert!( + ident.extra.as_deref().is_some_and(|extra| extra.starts_with("generated by default as identity")), + "ident_id extra: {:?}", + ident.extra + ); + // Ordinary columns keep their plain metadata. + let vc = columns.iter().find(|c| c.name == "vc").expect("vc column"); + assert_eq!(vc.data_type, "character varying(32)"); + let num = columns.iter().find(|c| c.name == "num").expect("num column"); + assert_eq!(num.data_type, "numeric(10,2)"); + assert_eq!(num.numeric_precision, Some(10)); + assert_eq!(num.numeric_scale, Some(2)); + let ts = columns.iter().find(|c| c.name == "ts").expect("ts column"); + assert_eq!(ts.data_type, "timestamp without time zone"); + + // The rendered column section of the DDL must contain each column + // definition exactly once. Before the dependency-join fix these + // queries could return "id" (serial + PRIMARY KEY + secondary index + // = several pg_depend rows) any number of times. + let ddl = dbx_core::schema::pg_ddl(&pool, &schema, table_name).await.expect("read repro table ddl"); + for needle in [r#" "id" serial NOT NULL"#, r#" "value" text"#, r#" "vc" character varying(32)"#] { + assert_eq!(ddl.matches(needle).count(), 1, "needle {needle:?} not unique in ddl:\n{ddl}"); + } + + drop(client); + execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")).await.expect("drop repro schema"); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL 9.x database"] +async fn postgres_legacy_table_ddl_uses_compatible_partition_probe() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let version = execute_query(&pool, "SHOW server_version_num").await.expect("query PostgreSQL version"); + let version_num = version.rows[0][0] + .as_str() + .expect("server_version_num should be text") + .parse::() + .expect("server_version_num should be numeric"); + assert!((90000..100000).contains(&version_num), "expected PostgreSQL 9.x, got {version_num}"); + + let schema = format!("dbx_legacy_ddl_{}", uuid::Uuid::new_v4().simple()); + let schema_ident = pg_quote_ident(&schema); + let parent = format!("{schema_ident}.parent"); + let child = format!("{schema_ident}.child"); + let function = format!("{schema_ident}.fill_child_name"); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE SCHEMA {schema_ident}; \ + CREATE TABLE {parent} (id integer PRIMARY KEY); \ + CREATE TABLE {child} (parent_id integer REFERENCES {parent}(id), name text) INHERITS ({parent}); \ + ALTER TABLE {child} ADD PRIMARY KEY (id); \ + CREATE INDEX child_name_idx ON {child}(name); \ + CREATE FUNCTION {function}() RETURNS trigger LANGUAGE plpgsql AS $$BEGIN NEW.name := COALESCE(NEW.name, chr(120)); RETURN NEW; END$$; \ + CREATE TRIGGER child_bi BEFORE INSERT ON {child} FOR EACH ROW EXECUTE PROCEDURE {function}()" + )) + .await + .expect("create PostgreSQL 9.x DDL fixtures"); + drop(client); + + let partition_info = + get_table_partition_info(&pool, &schema, "child").await.expect("classify PostgreSQL 9.x inherited table"); + let ddl = dbx_core::schema::pg_ddl(&pool, &schema, "child").await; + // 查看表 DDL（DISPLAY 路径）在 9.x 上不能因 relispartition 报错， + // INHERITS 子表也不能被当成声明式分区渲染。 + // The view-DDL path must not fail on the missing relispartition + // column pre-10, and INHERITS children must render as plain tables. + let ddl_with_partitions = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "child").await; + let parent_ddl = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "parent").await; + execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")) + .await + .expect("drop PostgreSQL 9.x DDL fixtures"); + let ddl = ddl.expect("generate PostgreSQL 9.x table DDL"); + let ddl_with_partitions = ddl_with_partitions.expect("generate PostgreSQL 9.x partition-tree DDL"); + let parent_ddl = parent_ddl.expect("generate PostgreSQL 9.x partition-tree DDL for parent"); + + assert!(ddl.contains("CREATE TABLE"), "ddl: {ddl}"); + assert!(ddl.contains("PRIMARY KEY"), "ddl: {ddl}"); + assert!(ddl.contains("FOREIGN KEY"), "ddl: {ddl}"); + assert!(ddl.contains("CREATE INDEX \"child_name_idx\""), "ddl: {ddl}"); + assert!(ddl.contains("CREATE TRIGGER child_bi"), "ddl: {ddl}"); + assert_eq!(partition_info, PostgresTablePartitionInfo::default()); + assert!(!ddl.contains("PARTITION OF"), "ddl: {ddl}"); + assert!(ddl_with_partitions.contains("CREATE TABLE"), "ddl_with_partitions: {ddl_with_partitions}"); + assert!(!ddl_with_partitions.contains("PARTITION OF"), "ddl_with_partitions: {ddl_with_partitions}"); + // 9.x 没有声明式分区：父表的分区树只含父表自身（子表按普通表）。 + assert_eq!(parent_ddl.matches("CREATE TABLE").count(), 1, "parent_ddl: {parent_ddl}"); + assert!(!parent_ddl.contains("PARTITION OF"), "parent_ddl: {parent_ddl}"); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] +async fn postgres_legacy_batched_ddl_metadata_preserves_relation_pairs_across_compat_tiers() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let schema = format!("dbx_batch_fkeys_{}", uuid::Uuid::new_v4().simple()); + let schema_ident = pg_quote_ident(&schema); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE SCHEMA {schema_ident}; \ + CREATE TABLE {schema_ident}.parent_a (id integer PRIMARY KEY); \ + CREATE TABLE {schema_ident}.parent_b (id integer PRIMARY KEY); \ + CREATE TABLE {schema_ident}.child_a ( \ + id integer PRIMARY KEY, parent_id integer, \ + CONSTRAINT child_a_parent_fk FOREIGN KEY (parent_id) REFERENCES {schema_ident}.parent_a(id) \ + ); \ + CREATE TABLE {schema_ident}.child_b ( \ + id integer PRIMARY KEY, parent_id integer, \ + CONSTRAINT child_b_parent_fk FOREIGN KEY (parent_id) REFERENCES {schema_ident}.parent_b(id) \ + ); \ + CREATE TABLE {schema_ident}.without_fk (id integer PRIMARY KEY)" + )) + .await + .expect("create batched foreign-key fixtures"); + + let version_num = client + .query_one("SHOW server_version_num", &[]) + .await + .expect("query PostgreSQL version") + .get::<_, String>(0) + .parse::() + .expect("server_version_num should be numeric"); + let has_conparentid = client + .query_one( + "SELECT EXISTS ( \ + SELECT 1 FROM pg_catalog.pg_attribute \ + WHERE attrelid = 'pg_catalog.pg_constraint'::pg_catalog.regclass \ + AND attname = 'conparentid' AND NOT attisdropped \ + )", + &[], + ) + .await + .expect("probe pg_constraint.conparentid") + .get::<_, bool>(0); + if version_num < 110000 { + assert!(!has_conparentid, "pre-11 PostgreSQL must not have parent-linked constraint clones"); + } + + let empty = list_foreign_keys_for_relations(&pool, &[]).await.expect("empty relation batch"); + assert!(empty.is_empty()); + + let single_relations = vec![(schema.clone(), "child_a".to_string())]; + let single = list_foreign_keys_for_relations(&pool, &single_relations).await.expect("single relation batch"); + let single_fk = &single[&(schema.clone(), "child_a".to_string())][0]; + assert_eq!(single_fk.name, "child_a_parent_fk"); + assert_eq!(single_fk.ref_table, "parent_a"); + + // Keep the pairs deliberately out of name order. A cross product or + // independently unnested array would duplicate or misattribute keys. + let relations = vec![ + (schema.clone(), "child_b".to_string()), + (schema.clone(), "without_fk".to_string()), + (schema.clone(), "child_a".to_string()), + ]; + let batched = list_foreign_keys_for_relations(&pool, &relations).await.expect("multi-relation batch"); + assert_eq!(batched.len(), 2, "batched keys: {batched:?}"); + assert!(!batched.contains_key(&(schema.clone(), "without_fk".to_string()))); + assert_eq!(batched[&(schema.clone(), "child_a".to_string())][0].ref_table, "parent_a"); + assert_eq!(batched[&(schema.clone(), "child_b".to_string())][0].ref_table, "parent_b"); + + let schemas: Vec<&str> = relations.iter().map(|(schema, _)| schema.as_str()).collect(); + let tables: Vec<&str> = relations.iter().map(|(_, table)| table.as_str()).collect(); + let compat = list_foreign_keys_for_relations_with_sql( + &client, + postgres_foreign_keys_for_relations_compat_sql(), + &schemas, + &tables, + ) + .await + .expect("subscript-paired compatibility query"); + assert_eq!(compat[&(schema.clone(), "child_a".to_string())][0].ref_table, "parent_a"); + assert_eq!(compat[&(schema.clone(), "child_b".to_string())][0].ref_table, "parent_b"); + + let modern = + list_foreign_keys_for_relations_with_sql(&client, postgres_foreign_keys_for_relations_sql(), &schemas, &tables) + .await; + if version_num >= 90400 { + let modern = modern.expect("multi-array unnest primary query"); + assert_eq!(modern[&(schema.clone(), "child_a".to_string())][0].ref_table, "parent_a"); + assert_eq!(modern[&(schema.clone(), "child_b".to_string())][0].ref_table, "parent_b"); + } else { + modern.expect_err("pre-9.4 server must reject multi-array unnest"); + } + + let relation_oids: Vec = client + .query( + "SELECT c.oid::bigint \ + FROM pg_catalog.pg_class c \ + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = $1 ORDER BY c.relname", + &[&schema], + ) + .await + .expect("query fixture relation oids") + .into_iter() + .map(|row| row.get(0)) + .collect(); + get_table_partition_local_objects_for_relations_with_sql( + &client, + postgres_table_partition_local_objects_for_relations_compat_sql(), + &relation_oids, + ) + .await + .expect("pre-parent-clone local-object query"); + let modern_local_objects = get_table_partition_local_objects_for_relations_with_sql( + &client, + postgres_table_partition_local_objects_for_relations_sql(), + &relation_oids, + ) + .await; + if version_num >= 90300 { + modern_local_objects.expect("JSON-extraction local-object query"); + } else { + modern_local_objects.expect_err("PostgreSQL 9.2 must reject the JSON ->> operator"); + } + drop(client); + + let child_ddl = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "child_a") + .await + .expect("display DDL with foreign key"); + let without_fk_ddl = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "without_fk") + .await + .expect("display DDL without foreign key"); + execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")) + .await + .expect("drop batched foreign-key fixtures"); + + assert!(child_ddl.contains("FOREIGN KEY"), "child ddl: {child_ddl}"); + assert!(!without_fk_ddl.contains("FOREIGN KEY"), "without-fk ddl: {without_fk_ddl}"); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] +async fn postgres_ddl_emits_not_valid_check_constraints_as_separate_replayable_alter() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let pid = std::process::id(); + let schema = format!("dbx_not_valid_check_{pid}"); + let schema_ident = pg_quote_ident(&schema); + let table = format!("{schema_ident}.events"); + + execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema"); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE TABLE {table} (id integer, amount integer); \ + ALTER TABLE {table} ADD CONSTRAINT c_pos CHECK (amount > 0) NOT VALID" + )) + .await + .expect("create table with unvalidated check constraint"); + + let ddl = dbx_core::schema::pg_ddl(&pool, &schema, "events").await.expect("events ddl"); + + // A `NOT VALID` CHECK constraint is illegal inline in a `CREATE + // TABLE` column list — Postgres only accepts it after `ALTER TABLE + // ADD CONSTRAINT`. Replaying the generated DDL onto a fresh schema + // exercises that this is real, executable syntax, not just a string + // match. + let replay_schema = format!("{schema}_replay"); + let replay_schema_ident = pg_quote_ident(&replay_schema); + execute_query(&pool, &format!("CREATE SCHEMA {replay_schema_ident}")).await.expect("create replay schema"); + client + .batch_execute(&ddl.replace(&schema, &replay_schema)) + .await + .unwrap_or_else(|error| panic!("replay not-valid-check ddl failed: {error:?}; ddl: {ddl}")); + + client + .batch_execute(&format!("DROP SCHEMA {schema_ident} CASCADE; DROP SCHEMA {replay_schema_ident} CASCADE")) + .await + .expect("drop schemas"); + + assert!(!ddl.contains("NOT VALID)"), "NOT VALID must not appear inline in the column list: {ddl}"); + assert!(ddl.contains("CREATE TABLE"), "ddl: {ddl}"); + assert!(!ddl.contains("\"amount\" integer,"), "check constraint must not be inlined into columns: {ddl}"); + assert!(ddl.contains("ALTER TABLE"), "ddl: {ddl}"); + assert!( + ddl.contains("ADD CONSTRAINT \"c_pos\" CHECK (amount > 0) NOT VALID;"), + "expected a separate ALTER TABLE ADD CONSTRAINT ... NOT VALID statement: {ddl}" + ); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] +async fn postgres_partition_metadata_renders_replayable_children_and_subpartitions() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let pid = std::process::id(); + let schema = format!("dbx_partition_meta_{pid}"); + let schema_ident = pg_quote_ident(&schema); + let replay_schema = format!("{schema}_replay"); + let replay_schema_ident = pg_quote_ident(&replay_schema); + let parent = format!("{schema_ident}.parent"); + let child = format!("{schema_ident}.child"); + let default_child = format!("{schema_ident}.child_default"); + let subpartition = format!("{schema_ident}.subpartition"); + let inherited_parent = format!("{schema_ident}.inherited_parent"); + let inherited_child = format!("{schema_ident}.inherited_child"); + // Declared in "public" (not the per-test schema) so the same + // qualified name keeps resolving after `schema_ident` substrings are + // rewritten to `replay_schema_ident` for the replay pass below. + let trigger_fn = format!("public.dbx_partition_meta_trigger_fn_{pid}"); + + execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema"); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE OR REPLACE FUNCTION {trigger_fn}() RETURNS trigger LANGUAGE plpgsql AS \ + $$ BEGIN RETURN NEW; END $$; \ + CREATE TABLE {parent} (id integer, payload text, status text DEFAULT 'pending') PARTITION BY RANGE (id); \ + CREATE TABLE {child} PARTITION OF {parent} ( \ + PRIMARY KEY (id), \ + CONSTRAINT child_payload_not_blank CHECK (payload <> ''), \ + status WITH OPTIONS DEFAULT 'child-status' \ + ) FOR VALUES FROM (1) TO (10); \ + CREATE INDEX child_payload_idx ON {child} (payload); \ + CREATE TABLE {default_child} PARTITION OF {parent} DEFAULT; \ + CREATE TABLE {subpartition} PARTITION OF {parent} FOR VALUES FROM (10) TO (20) PARTITION BY HASH (payload); \ + ALTER TABLE {parent} ADD CONSTRAINT parent_id_positive CHECK (id > 0); \ + CREATE TRIGGER parent_biu BEFORE INSERT ON {parent} FOR EACH ROW EXECUTE FUNCTION {trigger_fn}(); \ + CREATE TABLE {inherited_parent} (id integer, bucket integer, PRIMARY KEY (id, bucket)) PARTITION BY RANGE (bucket); \ + CREATE TABLE {inherited_child} PARTITION OF {inherited_parent} FOR VALUES FROM (1) TO (10)" + )) + .await + .expect("create partitioned tables"); + + let parent_info = get_table_partition_info(&pool, &schema, "parent").await.expect("parent metadata"); + let child_info = get_table_partition_info(&pool, &schema, "child").await.expect("child metadata"); + let default_info = get_table_partition_info(&pool, &schema, "child_default").await.expect("default metadata"); + let subpartition_info = + get_table_partition_info(&pool, &schema, "subpartition").await.expect("subpartition metadata"); + let child_local_objects = + get_table_partition_local_objects(&pool, &schema, "child").await.expect("child local objects"); + let inherited_child_local_objects = get_table_partition_local_objects(&pool, &schema, "inherited_child") + .await + .expect("inherited child local objects"); + let parent_ddl = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "parent").await.expect("parent ddl"); + let child_ddl = dbx_core::schema::pg_ddl(&pool, &schema, "child").await.expect("child ddl"); + let default_ddl = dbx_core::schema::pg_ddl(&pool, &schema, "child_default").await.expect("default ddl"); + let subpartition_ddl = dbx_core::schema::pg_ddl(&pool, &schema, "subpartition").await.expect("subpartition ddl"); + let inherited_parent_ddl = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "inherited_parent") + .await + .expect("inherited parent ddl"); + let inherited_child_ddl = + dbx_core::schema::pg_ddl(&pool, &schema, "inherited_child").await.expect("inherited child ddl"); + // The bare (non-recursive) parent DDL, exactly what database export + // and table transfer produce per-relation — used below to give the + // single-relation partition ddls (child_ddl, default_ddl, + // subpartition_ddl, inherited_child_ddl) a real parent shell to + // replay against, so their syntax is actually exercised by + // PostgreSQL rather than only checked with string assertions. + let bare_parent_ddl = dbx_core::schema::pg_ddl(&pool, &schema, "parent").await.expect("bare parent ddl"); + let bare_inherited_parent_ddl = + dbx_core::schema::pg_ddl(&pool, &schema, "inherited_parent").await.expect("bare inherited parent ddl"); + + // `parent_ddl`/`inherited_parent_ddl` already embed their full + // partition subtree (that's the fix under test), so replaying the + // per-partition ddl strings on top would recreate the same relations + // twice. Only the two top-level requests need replaying; the + // individually-fetched partition ddls above are replayed separately + // below, on top of a bare (childless) copy of their parent. + execute_query(&pool, &format!("CREATE SCHEMA {replay_schema_ident}")).await.expect("create replay schema"); + for ddl in [&parent_ddl, &inherited_parent_ddl] { + client + .batch_execute(&ddl.replace(&schema, &replay_schema)) + .await + .unwrap_or_else(|error| panic!("replay partition ddl failed: {error:?}; ddl: {ddl}")); + } + + // Single-relation partition ddls are now a real production code + // path (database export/table transfer fetch each partition this + // way), so replay them for real too: a fresh schema gets just the + // bare parents, then each single-relation partition ddl on top. + let bare_schema = format!("{schema}_bare"); + let bare_schema_ident = pg_quote_ident(&bare_schema); + execute_query(&pool, &format!("CREATE SCHEMA {bare_schema_ident}")).await.expect("create bare schema"); + for ddl in [&bare_parent_ddl, &bare_inherited_parent_ddl] { + client + .batch_execute(&ddl.replace(&schema, &bare_schema)) + .await + .unwrap_or_else(|error| panic!("replay bare parent ddl failed: {error:?}; ddl: {ddl}")); + } + for ddl in [&child_ddl, &default_ddl, &subpartition_ddl, &inherited_child_ddl] { + client + .batch_execute(&ddl.replace(&schema, &bare_schema)) + .await + .unwrap_or_else(|error| panic!("replay single-relation partition ddl failed: {error:?}; ddl: {ddl}")); + } + + client + .batch_execute(&format!( + "DROP SCHEMA {schema_ident} CASCADE; DROP SCHEMA {replay_schema_ident} CASCADE; \ + DROP SCHEMA {bare_schema_ident} CASCADE; DROP FUNCTION IF EXISTS {trigger_fn}()" + )) + .await + .expect("drop schemas and trigger function"); + + assert_eq!(parent_info.key.as_deref(), Some("RANGE (id)")); + assert!(!parent_info.is_partition); + assert_eq!(child_info.parent_schema.as_deref(), Some(schema.as_str())); + assert_eq!(child_info.parent_table.as_deref(), Some("parent")); + assert_eq!(child_info.bound.as_deref(), Some("FOR VALUES FROM (1) TO (10)")); + assert!(child_local_objects.has_primary_key); + assert!(child_local_objects.indexes.contains("child_payload_idx")); + assert!(child_local_objects.check_constraints.contains("child_payload_not_blank")); + assert!(!child_local_objects.check_constraints.contains("parent_id_positive")); + assert_eq!(child_local_objects.column_defaults.get("status"), Some(&PostgresColumnDefaultState::Overridden)); + assert!(!inherited_child_local_objects.has_primary_key); + assert!(inherited_child_local_objects.indexes.is_empty()); + assert_eq!(default_info.bound.as_deref(), Some("DEFAULT")); + assert_eq!(subpartition_info.key.as_deref(), Some("HASH (payload)")); + assert!(parent_ddl.contains(") PARTITION BY RANGE (id);"), "ddl: {parent_ddl}"); + assert!(parent_ddl.contains("CONSTRAINT \"parent_id_positive\" CHECK (id > 0)"), "ddl: {parent_ddl}"); + // The parent's trigger is automatically cloned onto every partition + // by PostgreSQL itself when the partition is created/attached, so it + // must appear exactly once in the combined DDL, not once per relation. + assert_eq!(parent_ddl.matches("CREATE TRIGGER").count(), 1, "ddl: {parent_ddl}"); + assert!(child_ddl.contains("CREATE TABLE"), "ddl: {child_ddl}"); + assert!(child_ddl.contains("PARTITION OF"), "ddl: {child_ddl}"); + assert!(child_ddl.contains("PRIMARY KEY (\"id\")"), "ddl: {child_ddl}"); + assert!(child_ddl.contains("FOR VALUES FROM (1) TO (10);"), "ddl: {child_ddl}"); + assert!(child_ddl.contains("CREATE INDEX \"child_payload_idx\""), "ddl: {child_ddl}"); + assert!(child_ddl.contains("CONSTRAINT \"child_payload_not_blank\" CHECK (payload"), "ddl: {child_ddl}"); + assert!(!child_ddl.contains("parent_id_positive"), "ddl: {child_ddl}"); + assert!(child_ddl.contains("\"status\" WITH OPTIONS DEFAULT 'child-status'"), "ddl: {child_ddl}"); + assert!(!child_ddl.contains("CREATE TRIGGER"), "child must not redeclare the parent's cloned trigger: {child_ddl}"); + assert!(default_ddl.contains(" DEFAULT;"), "ddl: {default_ddl}"); + assert!( + subpartition_ddl.contains("FOR VALUES FROM (10) TO (20) PARTITION BY HASH (payload);"), + "ddl: {subpartition_ddl}" + ); + assert!(!inherited_child_ddl.contains("PRIMARY KEY"), "ddl: {inherited_child_ddl}"); + assert!(!inherited_child_ddl.contains("CREATE INDEX"), "ddl: {inherited_child_ddl}"); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database with postgres_fdw available"] +async fn postgres_partition_ddl_uses_foreign_table_syntax_for_foreign_partitions() { + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); + let pid = std::process::id(); + let schema = format!("dbx_partition_fdw_{pid}"); + let schema_ident = pg_quote_ident(&schema); + let server = format!("dbx_partition_fdw_server_{pid}"); + let server_ident = pg_quote_ident(&server); + let replay_schema = format!("{schema}_replay"); + let replay_schema_ident = pg_quote_ident(&replay_schema); + let parent = format!("{schema_ident}.parent"); + let local_child = format!("{schema_ident}.local_child"); + let foreign_child = format!("{schema_ident}.foreign_child"); + + execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema"); + let client = pool.get().await.expect("get postgres client"); + client + .batch_execute(&format!( + "CREATE EXTENSION IF NOT EXISTS postgres_fdw; \ + CREATE SERVER {server_ident} FOREIGN DATA WRAPPER postgres_fdw \ + OPTIONS (host 'localhost', dbname 'postgres', port '5432'); \ + CREATE USER MAPPING FOR CURRENT_USER SERVER {server_ident} \ + OPTIONS (user 'postgres', password 'postgres'); \ + CREATE TABLE {parent} (id integer, payload text) PARTITION BY RANGE (id); \ + CREATE TABLE {local_child} PARTITION OF {parent} FOR VALUES FROM (1) TO (10); \ + CREATE FOREIGN TABLE {foreign_child} PARTITION OF {parent} \ + FOR VALUES FROM (10) TO (20) \ + SERVER {server_ident} OPTIONS (schema_name 'public', table_name 'foreign_child_data')" + )) + .await + .expect("create partitioned tables with a foreign partition"); + + let foreign_info = get_table_partition_info(&pool, &schema, "foreign_child").await.expect("foreign child info"); + let local_info = get_table_partition_info(&pool, &schema, "local_child").await.expect("local child info"); + let parent_ddl = dbx_core::schema::pg_ddl_with_partitions(&pool, &schema, "parent").await.expect("parent ddl"); + let foreign_ddl = dbx_core::schema::pg_ddl(&pool, &schema, "foreign_child").await.expect("foreign child ddl"); + + execute_query(&pool, &format!("CREATE SCHEMA {replay_schema_ident}")).await.expect("create replay schema"); + client + .batch_execute(&parent_ddl.replace(&schema, &replay_schema)) + .await + .unwrap_or_else(|error| panic!("replay foreign partition ddl failed: {error:?}; ddl: {parent_ddl}")); + + client + .batch_execute(&format!( + "DROP SCHEMA {schema_ident} CASCADE; DROP SCHEMA {replay_schema_ident} CASCADE; \ + DROP SERVER {server_ident} CASCADE" + )) + .await + .expect("drop schemas and foreign server"); + + assert!(!local_info.is_foreign); + assert!(foreign_info.is_foreign); + assert_eq!(foreign_info.foreign_server.as_deref(), Some(server.as_str())); + assert!( + foreign_info.foreign_options.contains(&("schema_name".to_string(), "public".to_string())), + "options: {:?}", + foreign_info.foreign_options + ); + assert!(parent_ddl.contains("CREATE FOREIGN TABLE"), "ddl: {parent_ddl}"); + assert!(foreign_ddl.starts_with("CREATE FOREIGN TABLE"), "ddl: {foreign_ddl}"); + assert!(foreign_ddl.contains("PARTITION OF"), "ddl: {foreign_ddl}"); + assert!(foreign_ddl.contains(&format!("SERVER {}", pg_quote_ident(&server))), "ddl: {foreign_ddl}"); + assert!(foreign_ddl.contains("\"schema_name\" 'public'"), "ddl: {foreign_ddl}"); + assert!(foreign_ddl.contains("\"table_name\" 'foreign_child_data'"), "ddl: {foreign_ddl}"); + assert!(!foreign_ddl.contains("PRIMARY KEY"), "ddl: {foreign_ddl}"); + assert!(!foreign_ddl.contains("CREATE INDEX"), "ddl: {foreign_ddl}"); +} diff --git a/crates/dbx-core/tests/live_schema_diff_regressions.rs b/crates/dbx-core/tests/live_schema_diff_regressions.rs new file mode 100644 index 0000000000..52c6e1674c --- /dev/null +++ b/crates/dbx-core/tests/live_schema_diff_regressions.rs @@ -0,0 +1,346 @@ +use dbx_core::models::connection::DatabaseType; +use dbx_core::schema_diff::*; +use dbx_core::sql_dialect::descriptor::DialectKind; +use dbx_core::types::*; + +fn table_info(name: &str, table_type: &str) -> TableInfo { + TableInfo { + name: name.to_string(), + table_type: table_type.to_string(), + valid: None, + comment: None, + parent_schema: None, + parent_name: None, + } +} + +#[tokio::test] +#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"] +async fn live_sqlserver_alter_column_preserves_unchanged_runtime_dependencies() { + let database = std::env::var("DBX_LIVE_SQLSERVER_DATABASE").unwrap_or_else(|_| "tempdb".to_string()); + let host = std::env::var("DBX_LIVE_SQLSERVER_HOST").expect("DBX_LIVE_SQLSERVER_HOST"); + let port = std::env::var("DBX_LIVE_SQLSERVER_PORT") + .expect("DBX_LIVE_SQLSERVER_PORT") + .parse() + .expect("valid DBX_LIVE_SQLSERVER_PORT"); + let user = std::env::var("DBX_LIVE_SQLSERVER_USER").expect("DBX_LIVE_SQLSERVER_USER"); + let password = std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"); + let mut client = dbx_core::db::sqlserver::connect( + &host, + port, + &user, + &password, + Some(&database), + None, + std::time::Duration::from_secs(10), + ) + .await + .expect("connect SQL Server"); + + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let lookup_table = format!("dbx_dep_lookup_{suffix}"); + let parent_table = format!("dbx_dep_parent_{suffix}"); + let child_table = format!("dbx_dep_child_{suffix}"); + let primary_key = format!("PK_dbx_dep_parent_{suffix}"); + let unique_constraint = format!("UQ_dbx_dep_parent_code_{suffix}"); + let lookup_unique_constraint = format!("UQ_dbx_dep_lookup_code_{suffix}"); + let check_constraint = format!("CK_dbx_dep_parent_values_{suffix}"); + let outbound_foreign_key = format!("FK_dbx_dep_parent_lookup_{suffix}"); + let inbound_foreign_key = format!("FK_dbx_dep_child_parent_{suffix}"); + let ordinary_index = format!("IX_dbx_dep_parent_code_{suffix}"); + let cleanup = format!( + "DROP TABLE IF EXISTS [dbo].[{child_table}];\ + DROP TABLE IF EXISTS [dbo].[{parent_table}];\ + DROP TABLE IF EXISTS [dbo].[{lookup_table}];" + ); + let _ = dbx_core::db::sqlserver::execute_batch(&mut client, &cleanup).await; + + let exercise = async { + let setup = format!( + "SET ANSI_NULLS ON;\ + SET QUOTED_IDENTIFIER ON;\ + SET ANSI_PADDING ON;\ + SET ANSI_WARNINGS ON;\ + SET CONCAT_NULL_YIELDS_NULL ON;\ + SET ARITHABORT ON;\ + SET NUMERIC_ROUNDABORT OFF;\ + CREATE TABLE [dbo].[{lookup_table}] (\ + [code] INT NOT NULL,\ + CONSTRAINT [{lookup_unique_constraint}] UNIQUE NONCLUSTERED ([code] ASC)\ + );\ + CREATE TABLE [dbo].[{parent_table}] (\ + [id] INT NOT NULL,\ + [code] INT NOT NULL,\ + [payload] NVARCHAR(64) NULL,\ + CONSTRAINT [{primary_key}] PRIMARY KEY CLUSTERED ([id] ASC),\ + CONSTRAINT [{unique_constraint}] UNIQUE NONCLUSTERED ([code] ASC),\ + CONSTRAINT [{check_constraint}] CHECK ([id] > 0 AND [code] >= 0),\ + CONSTRAINT [{outbound_foreign_key}] FOREIGN KEY ([code])\ + REFERENCES [dbo].[{lookup_table}] ([code])\ + );\ + CREATE TABLE [dbo].[{child_table}] (\ + [id] INT NOT NULL,\ + [parent_code] INT NULL,\ + CONSTRAINT [{inbound_foreign_key}] FOREIGN KEY ([parent_code])\ + REFERENCES [dbo].[{parent_table}] ([code])\ + ON UPDATE CASCADE ON DELETE SET NULL NOT FOR REPLICATION\ + );\ + CREATE NONCLUSTERED INDEX [{ordinary_index}]\ + ON [dbo].[{parent_table}] ([code] DESC, [id] ASC)\ + INCLUDE ([payload])\ + WHERE [code] IS NOT NULL\ + WITH (PAD_INDEX = ON, FILLFACTOR = 80, STATISTICS_NORECOMPUTE = ON,\ + ALLOW_ROW_LOCKS = OFF, ALLOW_PAGE_LOCKS = ON)\ + ON [PRIMARY];" + ); + dbx_core::db::sqlserver::execute_batch(&mut client, &setup).await?; + + let target_detail = TableSchemaDetail { + name: parent_table.clone(), + columns: dbx_core::db::sqlserver::get_columns(&mut client, "dbo", &parent_table).await?, + indexes: dbx_core::db::sqlserver::list_indexes(&mut client, "dbo", &parent_table).await?, + foreign_keys: dbx_core::db::sqlserver::list_foreign_keys(&mut client, "dbo", &parent_table).await?, + triggers: Vec::new(), + ddl: None, + }; + let mut source_detail = target_detail.clone(); + let source_id = source_detail + .columns + .iter_mut() + .find(|column| column.name == "id") + .ok_or_else(|| "live target metadata did not contain id".to_string())?; + source_id.data_type = "bigint".to_string(); + source_id.numeric_precision = Some(19); + let source_code = source_detail + .columns + .iter_mut() + .find(|column| column.name == "code") + .ok_or_else(|| "live target metadata did not contain code".to_string())?; + source_code.is_nullable = true; + + let prepared = prepare_schema_diff(SchemaDiffPreparationOptions { + source_tables: vec![table_info(&parent_table, "BASE TABLE")], + target_tables: vec![table_info(&parent_table, "BASE TABLE")], + source_details: vec![source_detail], + target_details: vec![target_detail], + database_type: DatabaseType::SqlServer, + target_schema: Some("dbo".to_string()), + source_dialect: Some(DialectKind::SqlServer), + target_dialect: Some(DialectKind::SqlServer), + ..Default::default() + }); + + dbx_core::db::sqlserver::execute_batch(&mut client, "SET QUOTED_IDENTIFIER OFF;").await?; + dbx_core::db::sqlserver::execute_batch(&mut client, &prepared.sync_sql).await?; + let verification_sql = format!( + "DECLARE @parent_id int = OBJECT_ID(N'[dbo].[{parent_table}]');\ + SELECT\ + CONVERT(int, CASE WHEN EXISTS (\ + SELECT 1\ + FROM sys.indexes AS idx\ + JOIN sys.stats AS stats\ + ON stats.object_id = idx.object_id AND stats.stats_id = idx.index_id\ + JOIN sys.data_spaces AS data_space ON data_space.data_space_id = idx.data_space_id\ + JOIN sys.index_columns AS key_ic\ + ON key_ic.object_id = idx.object_id AND key_ic.index_id = idx.index_id\ + JOIN sys.columns AS key_column\ + ON key_column.object_id = key_ic.object_id AND key_column.column_id = key_ic.column_id\ + WHERE idx.object_id = @parent_id\ + AND idx.name = N'{ordinary_index}'\ + AND idx.type_desc = N'NONCLUSTERED'\ + AND idx.is_primary_key = 0 AND idx.is_unique_constraint = 0\ + AND idx.is_disabled = 0 AND idx.has_filter = 1\ + AND idx.is_padded = 1 AND idx.fill_factor = 80\ + AND idx.allow_row_locks = 0 AND idx.allow_page_locks = 1\ + AND stats.no_recompute = 1 AND data_space.name = N'PRIMARY'\ + AND key_column.name = N'code' AND key_ic.key_ordinal = 1\ + AND key_ic.is_descending_key = 1\ + AND CHARINDEX(N'[code]', idx.filter_definition) > 0\ + AND EXISTS (\ + SELECT 1\ + FROM sys.index_columns AS include_ic\ + JOIN sys.columns AS include_column\ + ON include_column.object_id = include_ic.object_id\ + AND include_column.column_id = include_ic.column_id\ + WHERE include_ic.object_id = idx.object_id\ + AND include_ic.index_id = idx.index_id\ + AND include_ic.is_included_column = 1\ + AND include_column.name = N'payload'\ + )\ + ) THEN 1 ELSE 0 END) AS ordinary_index_ok,\ + CONVERT(int, CASE WHEN EXISTS (\ + SELECT 1\ + FROM sys.key_constraints AS key_constraint\ + JOIN sys.indexes AS idx\ + ON idx.object_id = key_constraint.parent_object_id\ + AND idx.index_id = key_constraint.unique_index_id\ + JOIN sys.index_columns AS ic\ + ON ic.object_id = idx.object_id AND ic.index_id = idx.index_id\ + JOIN sys.columns AS column_info\ + ON column_info.object_id = ic.object_id AND column_info.column_id = ic.column_id\ + JOIN sys.types AS column_type ON column_type.user_type_id = column_info.user_type_id\ + WHERE key_constraint.parent_object_id = @parent_id\ + AND key_constraint.name = N'{primary_key}'\ + AND key_constraint.type = N'PK'\ + AND idx.is_primary_key = 1 AND idx.type_desc = N'CLUSTERED'\ + AND ic.key_ordinal = 1 AND column_info.name = N'id'\ + AND column_type.name = N'bigint' AND column_info.is_nullable = 0\ + ) THEN 1 ELSE 0 END) AS primary_key_ok,\ + CONVERT(int, CASE WHEN EXISTS (\ + SELECT 1\ + FROM sys.check_constraints AS check_constraint\ + WHERE check_constraint.parent_object_id = @parent_id\ + AND check_constraint.name = N'{check_constraint}'\ + AND check_constraint.is_disabled = 0\ + AND check_constraint.is_not_trusted = 0\ + AND CHARINDEX(N'[id]', check_constraint.definition) > 0\ + AND CHARINDEX(N'[code]', check_constraint.definition) > 0\ + ) THEN 1 ELSE 0 END) AS check_constraint_ok,\ + CONVERT(int, CASE WHEN EXISTS (\ + SELECT 1\ + FROM sys.foreign_keys AS foreign_key\ + JOIN sys.foreign_key_columns AS fkc\ + ON fkc.constraint_object_id = foreign_key.object_id\ + JOIN sys.columns AS child_column\ + ON child_column.object_id = fkc.parent_object_id\ + AND child_column.column_id = fkc.parent_column_id\ + JOIN sys.columns AS parent_column\ + ON parent_column.object_id = fkc.referenced_object_id\ + AND parent_column.column_id = fkc.referenced_column_id\ + WHERE foreign_key.name = N'{inbound_foreign_key}'\ + AND foreign_key.parent_object_id = OBJECT_ID(N'[dbo].[{child_table}]')\ + AND foreign_key.referenced_object_id = @parent_id\ + AND foreign_key.type = N'F'\ + AND foreign_key.is_disabled = 0 AND foreign_key.is_not_trusted = 1\ + AND foreign_key.is_not_for_replication = 1\ + AND foreign_key.update_referential_action_desc = N'CASCADE'\ + AND foreign_key.delete_referential_action_desc = N'SET_NULL'\ + AND child_column.name = N'parent_code' AND parent_column.name = N'code'\ + AND EXISTS (\ + SELECT 1 FROM sys.key_constraints AS uq\ + WHERE uq.parent_object_id = @parent_id\ + AND uq.name = N'{unique_constraint}' AND uq.type = N'UQ'\ + )\ + ) THEN 1 ELSE 0 END) AS inbound_foreign_key_ok,\ + CONVERT(int, CASE WHEN\ + EXISTS (\ + SELECT 1 FROM sys.columns AS column_info\ + JOIN sys.types AS column_type ON column_type.user_type_id = column_info.user_type_id\ + WHERE column_info.object_id = @parent_id AND column_info.name = N'id'\ + AND column_type.name = N'bigint' AND column_info.is_nullable = 0\ + )\ + AND EXISTS (\ + SELECT 1 FROM sys.columns AS column_info\ + JOIN sys.types AS column_type ON column_type.user_type_id = column_info.user_type_id\ + WHERE column_info.object_id = @parent_id AND column_info.name = N'code'\ + AND column_type.name = N'int' AND column_info.is_nullable = 1\ + )\ + THEN 1 ELSE 0 END) AS altered_columns_ok;" + ); + let verification = dbx_core::db::sqlserver::execute_query(&mut client, &verification_sql).await?; + Ok::<_, String>((prepared, verification)) + } + .await; + + let cleanup_result = dbx_core::db::sqlserver::execute_batch(&mut client, &cleanup).await; + cleanup_result.expect("drop live SQL Server dependency test tables"); + let (prepared, verification) = exercise.expect("exercise live SQL Server dependency-aware ALTER COLUMN"); + + assert_eq!(prepared.diffs.len(), 1, "diffs={:?}", prepared.diffs); + let table_diff = &prepared.diffs[0]; + assert_eq!(table_diff.diff_type, "modified"); + assert_eq!(table_diff.columns.as_ref().map(Vec::len), Some(2)); + assert!(table_diff.indexes.is_none(), "unchanged indexes must be absent: {table_diff:?}"); + assert!(table_diff.foreign_keys.is_none(), "unchanged foreign keys must be absent: {table_diff:?}"); + assert!(!prepared.sync_sql.contains(&ordinary_index), "index must be discovered at runtime: {}", prepared.sync_sql); + assert!( + !prepared.sync_sql.contains(&outbound_foreign_key), + "foreign keys must be discovered at runtime: {}", + prepared.sync_sql + ); + assert!( + !prepared.sync_sql.contains(&inbound_foreign_key), + "inbound foreign keys are not represented by the table diff: {}", + prepared.sync_sql + ); + + assert_eq!(verification.rows.len(), 1, "verification={verification:?}"); + assert_eq!( + verification.rows[0], + vec![ + serde_json::json!(1), + serde_json::json!(1), + serde_json::json!(1), + serde_json::json!(1), + serde_json::json!(1), + ], + "columns={:?}, sync_sql={} ", + verification.columns, + prepared.sync_sql + ); +} + +#[tokio::test] +#[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL-family database"] +async fn real_postgres_round_trip_quotes_columns_and_leaves_expressions_bare() { + // PR #6312 review: exercise the full path end-to-end against a real PostgreSQL server + // instead of only asserting on generated text. Introspects a table whose unique index + // mixes a real column with an expression-hostile name ("order item", i.e. exactly the + // a.attname case the reviewer called out) and a genuine pg_get_indexdef expression key + // part, generates DDL with the same `create_index_sql` schema-diff sync uses, and + // executes that DDL back against the database to prove it's actually valid — not just + // plausible-looking text. + let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); + let pool = + dbx_core::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect postgres"); + let schema = format!("dbx_key_expr_{}", uuid::Uuid::new_v4().simple()); + dbx_core::db::postgres::execute_query(&pool, &format!("CREATE SCHEMA {schema}")).await.expect("create schema"); + + let exercise = async { + dbx_core::db::postgres::execute_query( + &pool, + &format!( + "CREATE TABLE {schema}.tankong_data (\ + \"order item\" integer, data_type text, data_time timestamp, height double precision)" + ), + ) + .await?; + dbx_core::db::postgres::execute_query( + &pool, + &format!( + "CREATE UNIQUE INDEX uq_tankong_sta_type_time ON {schema}.tankong_data \ + (\"order item\", data_type, data_time, \ + (COALESCE(height, '-1'::integer::double precision)))" + ), + ) + .await?; + + let indexes = dbx_core::db::postgres::list_indexes(&pool, &schema, "tankong_data").await?; + let index = indexes + .into_iter() + .find(|index| index.name == "uq_tankong_sta_type_time") + .ok_or_else(|| "introspection should return the created index".to_string())?; + + // Regenerate the index DDL through the exact same production function schema-diff + // sync calls, then execute it back against the real database to prove it's valid. + let ddl = create_index_sql("tankong_data", &index, DatabaseType::Highgo, Some(&schema)); + dbx_core::db::postgres::execute_query(&pool, &format!("DROP INDEX {schema}.uq_tankong_sta_type_time")).await?; + dbx_core::db::postgres::execute_query(&pool, &ddl).await?; + + Ok::<_, String>((index, ddl)) + } + .await; + + let cleanup = dbx_core::db::postgres::execute_query(&pool, &format!("DROP SCHEMA {schema} CASCADE")).await; + cleanup.expect("drop schema"); + let (index, recreate_ddl) = exercise.expect("exercise real postgres round trip"); + + assert_eq!( + index.columns, + vec!["order item", "data_type", "data_time", "COALESCE(height, '-1'::integer::double precision)"] + ); + assert_eq!(index.key_is_expression, vec![false, false, false, true]); + assert!(recreate_ddl.contains("\"order item\"")); + assert!(recreate_ddl.contains("COALESCE(height, '-1'::integer::double precision)")); + assert!(!recreate_ddl.contains("\"COALESCE")); +} diff --git a/crates/dbx-core/tests/public_api_compatibility.rs b/crates/dbx-core/tests/public_api_compatibility.rs new file mode 100644 index 0000000000..8ec0f004d0 --- /dev/null +++ b/crates/dbx-core/tests/public_api_compatibility.rs @@ -0,0 +1,33 @@ +use std::any::TypeId; + +fn assert_same_type() { + assert_eq!(TypeId::of::(), TypeId::of::()); +} + +#[test] +fn extracted_crates_preserve_legacy_type_identity() { + assert_same_type::( + ); + assert_same_type::(); + assert_same_type::(); + assert_same_type::(); + assert_same_type::( + ); + assert_same_type::(); + assert_same_type::< + dbx_core::table_structure_sql::TableStructureSqlOptions, + dbx_sql::table_structure_sql::TableStructureSqlOptions, + >(); + assert_same_type::(); + assert_same_type::(); + assert_same_type::(); + assert_same_type::(); + assert_same_type::(); +} + +#[test] +fn business_directories_preserve_legacy_namespaces() { + assert_same_type::(); + assert_same_type::( + ); +} diff --git a/crates/dbx-drivers/Cargo.toml b/crates/dbx-drivers/Cargo.toml new file mode 100644 index 0000000000..ad7f807af1 --- /dev/null +++ b/crates/dbx-drivers/Cargo.toml @@ -0,0 +1,78 @@ +[package] +name = "dbx-drivers" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +duckdb-sidecar = ["dbx-sql/duckdb-sidecar"] +dynamodb = ["dep:aws-sdk-dynamodb"] +mq-admin = ["dbx-types/mq-admin"] +sqlite-bundled = ["rusqlite/bundled"] +sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl"] +sqlite-multiple-ciphers = ["dep:libsqlite3-hotbundle"] +test-support = ["dbx-platform/test-support"] + +[dependencies] +dbx-types = { path = "../dbx-types", default-features = false } +dbx-sql = { path = "../dbx-sql", default-features = false } +dbx-platform = { path = "../dbx-platform", features = ["host-prompts", "downloads"] } +dbx-sqlite-worker = { path = "../dbx-sqlite-worker", default-features = false } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +json5 = "0.4" +regex = "1" +percent-encoding = "2" +log = "0.4" +tracing = "0.1" +tracing-subscriber = "0.3" +tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7", features = ["compat"] } +chrono = { version = "0.4", features = ["serde"] } +rust_decimal = { version = "1", features = ["serde"] } +uuid = { version = "1", features = ["v4", "serde"] } +tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-uuid-1", "with-serde_json-1"] } +deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } +tokio-postgres-rustls = "0.13" +postgres-openssl = "0.5" +openssl = { version = "0.10", features = ["vendored"] } +webpki-roots = "0.26" +rusqlite = { version = "0.32", default-features = false, features = ["load_extension", "backup", "functions", "column_decltype"] } +libsqlite3-hotbundle = { git = "https://github.com/zipg/libsqlite3-hotbundle.git", rev = "7f12f203b4f4694a273410bd9b02d77f82b6114e", features = ["cipher-sqlcipher"], optional = true } +mysql_async = { version = "0.37", default-features = false, features = ["default-rustls", "client_ed25519", "chrono", "rust_decimal"] } +sqlparser = { version = "0.62.0", features = ["visitor"] } +redis = { version = "0.32.2", features = ["tokio-comp", "tls-rustls", "tls-rustls-insecure", "tokio-rustls-comp", "sentinel", "cluster-async"] } +rustls = { version = "0.23", features = ["aws-lc-rs"] } +rustls-pemfile = "2.2" +tiberius = { version = "0.12.3", default-features = false, features = ["tds73", "chrono", "rust_decimal", "rustls", "sql-browser-tokio"] } +reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "stream", "rustls-tls", "socks"] } +futures = "0.3" +mongodb = { version = "3.2.5", features = ["socks5-proxy"] } +aws-sdk-dynamodb = { version = "=1.104.0", optional = true } +russh = "0.60" +pageant = "0.2" +portpicker = "0.1.1" +csv = "1" +base64 = "0.22" +sha2 = "0.10" +bytes = "1" +zip = { version = "4", default-features = false, features = ["deflate"] } +flate2 = "1" +crc = "3" +tar = "0.4" +zstd = "0.13" +tempfile = "3" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem", "Win32_Foundation", "Win32_Security"] } + +[dev-dependencies] +dbx-platform = { path = "../dbx-platform", features = ["test-support"] } +mysql_common = { git = "https://github.com/t8y2/rust_mysql_common.git", rev = "47740d85a8277167a5717afac785d37b46a36296", default-features = false } +tokio = { version = "1", features = ["test-util"] } +tokio-openssl = "0.6" + +[target.'cfg(all(unix, not(target_os = "redox")))'.dev-dependencies] +nix = { version = "0.31.3", features = ["user"] } diff --git a/crates/dbx-core/assets/agent-protocol-v2.json b/crates/dbx-drivers/assets/agent-protocol-v2.json similarity index 100% rename from crates/dbx-core/assets/agent-protocol-v2.json rename to crates/dbx-drivers/assets/agent-protocol-v2.json diff --git a/crates/dbx-core/src/agent_catalog.rs b/crates/dbx-drivers/src/agent_catalog.rs similarity index 100% rename from crates/dbx-core/src/agent_catalog.rs rename to crates/dbx-drivers/src/agent_catalog.rs diff --git a/crates/dbx-core/src/agent_connection.rs b/crates/dbx-drivers/src/agent_connection.rs similarity index 100% rename from crates/dbx-core/src/agent_connection.rs rename to crates/dbx-drivers/src/agent_connection.rs diff --git a/crates/dbx-core/src/agent_manager.rs b/crates/dbx-drivers/src/agent_manager.rs similarity index 98% rename from crates/dbx-core/src/agent_manager.rs rename to crates/dbx-drivers/src/agent_manager.rs index 56574cd3b7..52e0e58d65 100644 --- a/crates/dbx-core/src/agent_manager.rs +++ b/crates/dbx-drivers/src/agent_manager.rs @@ -19,16 +19,16 @@ pub const SQLITE_WORKER_NATIVE_PLATFORMS: &[&str] = &["linux-x64", "linux-aarch6 pub const DOWNLOAD_CACHE_DIR_NAME: &str = "download-cache"; pub const DOWNLOAD_CACHE_MAX_AGE_DAYS: u64 = 7; -pub(crate) type OperationLockTable = StdMutex>>>; +pub type OperationLockTable = StdMutex>>>; -pub(crate) struct OperationLockHandle<'a> { +pub struct OperationLockHandle<'a> { table: &'a OperationLockTable, key: String, lock: Arc>, } impl<'a> OperationLockHandle<'a> { - pub(crate) fn new(table: &'a OperationLockTable, key: &str, lock: Arc>) -> Self { + pub fn new(table: &'a OperationLockTable, key: &str, lock: Arc>) -> Self { Self { table, key: key.to_string(), lock } } } @@ -570,27 +570,26 @@ pub struct DriverStoreUsage { pub struct AgentManager { base_dir: PathBuf, app_version: String, - pub(crate) daemons: Mutex>, - pub(crate) connection_runtimes: Mutex< + pub daemons: Mutex>, + pub connection_runtimes: Mutex< std::collections::HashMap>>>, >, /// Serializes `load_state` → modify → `save_state` to prevent lost updates /// when multiple driver installs run concurrently. - pub(crate) state_lock: StdMutex<()>, + pub state_lock: StdMutex<()>, /// Per-JRE-key install locks so that concurrent driver installs sharing the /// same JRE download it only once (DCL pattern: lock → re-check installed → download). - pub(crate) jre_install_locks: OperationLockTable, + pub jre_install_locks: OperationLockTable, /// Per-driver locks serialize install, import, and uninstall operations /// targeting the same on-disk agent files. - pub(crate) driver_operation_locks: OperationLockTable, + pub driver_operation_locks: OperationLockTable, /// Driver operations may run concurrently, but JRE replacement/removal /// must exclude them until their dependent driver state is persisted. - pub(crate) installation_operation_lock: tokio::sync::RwLock<()>, + pub installation_operation_lock: tokio::sync::RwLock<()>, /// Per-operation cancellation tokens keyed by operation id (single installs /// use `install:`, batches `batch:` and `batch::`). Downloads /// observe the exact token threaded to them to abort promptly. - pub(crate) install_cancellations: - tokio::sync::Mutex>>, + pub install_cancellations: tokio::sync::Mutex>>, } impl Default for AgentManager { diff --git a/crates/dbx-core/src/agent_offline_export.rs b/crates/dbx-drivers/src/agent_offline_export.rs similarity index 100% rename from crates/dbx-core/src/agent_offline_export.rs rename to crates/dbx-drivers/src/agent_offline_export.rs diff --git a/crates/dbx-core/src/agent_recovery.rs b/crates/dbx-drivers/src/agent_recovery.rs similarity index 79% rename from crates/dbx-core/src/agent_recovery.rs rename to crates/dbx-drivers/src/agent_recovery.rs index 6562357d5c..bf28cfaf50 100644 --- a/crates/dbx-core/src/agent_recovery.rs +++ b/crates/dbx-drivers/src/agent_recovery.rs @@ -66,22 +66,3 @@ impl RecoveryPolicy { } } } - -#[cfg(test)] -mod tests { - #[test] - fn business_modules_do_not_consume_legacy_marker_fields() { - let business_sources = [ - ("query.rs", include_str!("query.rs")), - ("schema.rs", include_str!("schema.rs")), - ("connection.rs", include_str!("connection.rs")), - ]; - let retired_consumers = ["agent_rpc_error_category", "agent_rpc_error_session_id", "agent_session_disposition"]; - - for (name, source) in business_sources { - for consumer in retired_consumers { - assert!(!source.contains(consumer), "{name} still consumes {consumer}"); - } - } - } -} diff --git a/crates/dbx-core/src/agent_runtime.rs b/crates/dbx-drivers/src/agent_runtime.rs similarity index 99% rename from crates/dbx-core/src/agent_runtime.rs rename to crates/dbx-drivers/src/agent_runtime.rs index e931d86f94..5ee3700c8b 100644 --- a/crates/dbx-core/src/agent_runtime.rs +++ b/crates/dbx-drivers/src/agent_runtime.rs @@ -8,8 +8,8 @@ use crate::db::agent_driver::{AgentCallError, AgentDriverClient, AgentMethod, Ag use crate::models::connection::DatabaseType; pub struct SharedConnectionOpenError { - pub(crate) message: String, - pub(crate) runtime: Option>, + pub message: String, + pub runtime: Option>, } impl From for SharedConnectionOpenError { diff --git a/crates/dbx-core/src/agent_service.rs b/crates/dbx-drivers/src/agent_service.rs similarity index 99% rename from crates/dbx-core/src/agent_service.rs rename to crates/dbx-drivers/src/agent_service.rs index c84a64cced..3075a34869 100644 --- a/crates/dbx-core/src/agent_service.rs +++ b/crates/dbx-drivers/src/agent_service.rs @@ -2408,7 +2408,7 @@ pub fn ensure_driver_app_version( } pub fn is_app_version_compatible(min_app_version: &str, current_version: &str) -> bool { - !crate::update::is_newer_version(min_app_version, current_version) + !dbx_platform::version::is_newer_version(min_app_version, current_version) } pub fn download_temp_path(dest: &std::path::Path) -> std::path::PathBuf { @@ -2647,7 +2647,7 @@ async fn import_tar_zstd_driver_package( if let Some(installed) = am.load_state().installed_drivers.get(&info.db_type) { if installed.version != "0.1.0-local" && installed.version != "local" - && !crate::update::is_newer_version(&info.version, &installed.version) + && !dbx_platform::version::is_newer_version(&info.version, &installed.version) { result.drivers_skipped.push(info.db_type); return Ok(result); @@ -2899,7 +2899,7 @@ pub async fn import_offline_zip( if let Some(installed) = local_state.installed_drivers.get(db_type) { if installed.version != "0.1.0-local" && installed.version != "local" - && !crate::update::is_newer_version(&remote_driver.version, &installed.version) + && !dbx_platform::version::is_newer_version(&remote_driver.version, &installed.version) { result.drivers_skipped.push(db_type.clone()); continue; @@ -3464,11 +3464,11 @@ fn jre_dir_contains_java(path: &Path) -> bool { || path.join("Contents").join("Home").join("bin").join(java_name).is_file() } -pub(crate) fn validate_native_agent_binary(path: &Path) -> Result<(), String> { +pub fn validate_native_agent_binary(path: &Path) -> Result<(), String> { validate_native_agent_binary_for_platform(path, AgentManager::current_platform()) } -pub(crate) fn validate_native_agent_binary_for_platform(path: &Path, platform: &str) -> Result<(), String> { +pub fn validate_native_agent_binary_for_platform(path: &Path, platform: &str) -> Result<(), String> { let mut file = std::fs::File::open(path).map_err(|e| format!("Failed to read native agent: {e}"))?; let mut magic = [0_u8; 4]; file.read_exact(&mut magic).map_err(|e| format!("Failed to read native agent header: {e}"))?; diff --git a/crates/dbx-core/src/backend_error.rs b/crates/dbx-drivers/src/backend_error.rs similarity index 100% rename from crates/dbx-core/src/backend_error.rs rename to crates/dbx-drivers/src/backend_error.rs diff --git a/crates/dbx-core/src/database_capabilities.rs b/crates/dbx-drivers/src/database_capabilities.rs similarity index 100% rename from crates/dbx-core/src/database_capabilities.rs rename to crates/dbx-drivers/src/database_capabilities.rs diff --git a/crates/dbx-core/src/db/agent_driver.rs b/crates/dbx-drivers/src/db/agent_driver.rs similarity index 99% rename from crates/dbx-core/src/db/agent_driver.rs rename to crates/dbx-drivers/src/db/agent_driver.rs index c82d12e3ee..79d82403b8 100644 --- a/crates/dbx-core/src/db/agent_driver.rs +++ b/crates/dbx-drivers/src/db/agent_driver.rs @@ -488,7 +488,7 @@ pub enum AgentCallError { /// Business modules must use this adapter instead of inspecting the legacy /// `DBX_AGENT_ERROR_DATA` marker themselves. Callers must only use it after the /// owning pool has been identified as an Agent pool. -pub(crate) fn agent_error_from_legacy(error: &str, expected_session_id: Option<&str>) -> AgentCallError { +pub fn agent_error_from_legacy(error: &str, expected_session_id: Option<&str>) -> AgentCallError { if error == "Query canceled" { return AgentCallError::Canceled { stage: AgentErrorStage::Cancel, @@ -507,7 +507,7 @@ pub(crate) fn agent_error_from_legacy(error: &str, expected_session_id: Option<& /// Returns a typed error only when a mixed legacy path can prove the string /// originated from the Agent call corridor. -pub(crate) fn try_agent_error_from_legacy(error: &str) -> Option { +pub fn try_agent_error_from_legacy(error: &str) -> Option { let lower = error.to_ascii_lowercase(); let is_local_agent_failure = error == "Query canceled" || lower.starts_with("agent rpc call timed out") @@ -522,7 +522,7 @@ pub(crate) fn try_agent_error_from_legacy(error: &str) -> Option (is_agent_rpc_response_error(error) || is_local_agent_failure).then(|| agent_error_from_legacy(error, None)) } -pub(crate) fn agent_recovery_decision(error: &str, scope: RecoveryScope) -> RecoveryDecision { +pub fn agent_recovery_decision(error: &str, scope: RecoveryScope) -> RecoveryDecision { RecoveryPolicy::decide(&agent_error_from_legacy(error, None), scope) } @@ -731,7 +731,7 @@ fn legacy_agent_hints(data: &Value) -> LegacyAgentHints { } } -pub(crate) fn valid_agent_error_combination(context: &AgentErrorContext) -> bool { +pub fn valid_agent_error_combination(context: &AgentErrorContext) -> bool { let expected_outcome = match context.stage { AgentErrorStage::Request | AgentErrorStage::Checkout | AgentErrorStage::Connect | AgentErrorStage::Validate => { AgentOperationOutcome::NotStarted @@ -781,7 +781,7 @@ fn valid_ascii_identifier(value: &str) -> bool { !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_graphic()) } -pub(crate) fn category_name(category: AgentErrorCategory) -> &'static str { +pub fn category_name(category: AgentErrorCategory) -> &'static str { match category { AgentErrorCategory::Connection => "connection", AgentErrorCategory::Sql => "sql", @@ -792,7 +792,7 @@ pub(crate) fn category_name(category: AgentErrorCategory) -> &'static str { } } -pub(crate) fn stage_name(stage: AgentErrorStage) -> &'static str { +pub fn stage_name(stage: AgentErrorStage) -> &'static str { match stage { AgentErrorStage::Request => "request", AgentErrorStage::Checkout => "checkout", @@ -805,7 +805,7 @@ pub(crate) fn stage_name(stage: AgentErrorStage) -> &'static str { } } -pub(crate) fn outcome_name(outcome: AgentOperationOutcome) -> &'static str { +pub fn outcome_name(outcome: AgentOperationOutcome) -> &'static str { match outcome { AgentOperationOutcome::NotStarted => "not_started", AgentOperationOutcome::Unknown => "unknown", @@ -1048,7 +1048,7 @@ pub enum AgentCapability { StructuredErrorV1, } -pub(crate) fn legacy_agent_call_error(error: String, agent_session_id: Option<&str>) -> AgentCallError { +pub fn legacy_agent_call_error(error: String, agent_session_id: Option<&str>) -> AgentCallError { if !is_agent_rpc_response_error(&error) { return AgentCallError::Transport { message: error }; } @@ -1082,7 +1082,7 @@ fn parse_legacy_agent_error_data(data: &str) -> Option<(Value, &str)> { Some((value, &data[values.byte_offset()..])) } -pub(crate) fn append_legacy_error_context(error: &str, context: &str) -> String { +pub fn append_legacy_error_context(error: &str, context: &str) -> String { if error.contains(context) { return error.to_string(); } @@ -1984,7 +1984,7 @@ impl AgentDriverClient { self.call_method_with_timeout(AgentMethod::ConnectionInfo, serde_json::json!({}), timeout_duration).await } - pub(crate) fn set_identifier_quote(&mut self, identifier_quote: Option) { + pub fn set_identifier_quote(&mut self, identifier_quote: Option) { self.identifier_quote = identifier_quote.filter(|quote| !quote.trim().is_empty()); } @@ -3101,7 +3101,7 @@ fn agent_java_args_with_extra_args(jar_path: &str, extra_java_args: &[String]) - agent_java_args_with_extra_opts(jar_path, std::env::var(AGENT_JAVA_OPTS_ENV).ok().as_deref(), extra_java_args) } -pub(crate) fn validate_dameng_java_system_properties(options: &[String]) -> Result<(), String> { +pub fn validate_dameng_java_system_properties(options: &[String]) -> Result<(), String> { for (index, option) in options.iter().enumerate() { let option = option.trim(); let Some(property) = option.strip_prefix("-D") else { @@ -3470,8 +3470,8 @@ fn format_agent_startup_error(base: &str, child: &mut Child, stderr_tail: &Arc Self { + #[cfg(any(test, feature = "test-support"))] + pub fn test_stub() -> Self { Self { child: None, stdin: None, diff --git a/crates/dbx-core/src/db/clickhouse_driver.rs b/crates/dbx-drivers/src/db/clickhouse_driver.rs similarity index 99% rename from crates/dbx-core/src/db/clickhouse_driver.rs rename to crates/dbx-drivers/src/db/clickhouse_driver.rs index 87738809ef..a14c6f8488 100644 --- a/crates/dbx-core/src/db/clickhouse_driver.rs +++ b/crates/dbx-drivers/src/db/clickhouse_driver.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use super::{http_client_builder, with_connection_timeout}; -use crate::query::{canceled_error, MAX_ROWS}; +use crate::execution::{canceled_error, MAX_ROWS}; use crate::sql::starts_with_executable_sql_keyword; use crate::types::{ColumnInfo, DatabaseInfo, IndexInfo, ObjectStatistics, QueryResult, TableInfo}; @@ -930,7 +930,7 @@ mod tests { let url = build_query_url( "http://localhost:8123", Some("analytics"), - QueryResultLimit::Limited(crate::query::MAX_ROWS + 1), + QueryResultLimit::Limited(crate::execution::MAX_ROWS + 1), None, ); @@ -1124,14 +1124,14 @@ mod tests { fn limited_query_result_truncates_extra_probe_row() { let result = ChJsonResult { meta: vec![ChColumn { name: "id".to_string(), _type: "UInt64".to_string() }], - data: (0..=crate::query::MAX_ROWS).map(|value| vec![serde_json::Value::Number(value.into())]).collect(), - rows: crate::query::MAX_ROWS + 1, + data: (0..=crate::execution::MAX_ROWS).map(|value| vec![serde_json::Value::Number(value.into())]).collect(), + rows: crate::execution::MAX_ROWS + 1, }; let result = limited_query_result(result, 12, None); assert_eq!(result.columns, vec!["id"]); - assert_eq!(result.rows.len(), crate::query::MAX_ROWS); + assert_eq!(result.rows.len(), crate::execution::MAX_ROWS); assert_eq!(result.execution_time_ms, 12); assert!(result.truncated); } diff --git a/crates/dbx-core/src/db/cloudberry.rs b/crates/dbx-drivers/src/db/cloudberry.rs similarity index 100% rename from crates/dbx-core/src/db/cloudberry.rs rename to crates/dbx-drivers/src/db/cloudberry.rs diff --git a/crates/dbx-core/src/db/cloudflare_d1/mod.rs b/crates/dbx-drivers/src/db/cloudflare_d1/mod.rs similarity index 99% rename from crates/dbx-core/src/db/cloudflare_d1/mod.rs rename to crates/dbx-drivers/src/db/cloudflare_d1/mod.rs index a1be72b5d3..5fd4736958 100644 --- a/crates/dbx-core/src/db/cloudflare_d1/mod.rs +++ b/crates/dbx-drivers/src/db/cloudflare_d1/mod.rs @@ -9,15 +9,11 @@ use crate::types::{ TriggerInfo, }; -mod import; mod sql_guard; mod sql_lexer; -mod sql_limits; -#[cfg(test)] -mod transfer_tests; +pub mod sql_limits; mod trigger; -pub(crate) use import::{build_import_insert_batches, build_streaming_import_insert_batch}; pub use sql_limits::MAX_SQL_STATEMENT_BYTES; const CLOUDFLARE_API_BASE_URL: &str = "https://api.cloudflare.com/client/v4"; @@ -352,7 +348,7 @@ fn query_result( max_rows: Option, ) -> QueryResult { result.rows = result.rows.into_iter().map(|row| row.into_iter().map(json_value_for_js).collect()).collect(); - let row_limit = max_rows.unwrap_or(crate::query::MAX_ROWS).max(1); + let row_limit = max_rows.unwrap_or(crate::execution::MAX_ROWS).max(1); let truncated = result.rows.len() > row_limit; if truncated { result.rows.truncate(row_limit); diff --git a/crates/dbx-core/src/db/cloudflare_d1/sql_guard.rs b/crates/dbx-drivers/src/db/cloudflare_d1/sql_guard.rs similarity index 100% rename from crates/dbx-core/src/db/cloudflare_d1/sql_guard.rs rename to crates/dbx-drivers/src/db/cloudflare_d1/sql_guard.rs diff --git a/crates/dbx-core/src/db/cloudflare_d1/sql_lexer.rs b/crates/dbx-drivers/src/db/cloudflare_d1/sql_lexer.rs similarity index 100% rename from crates/dbx-core/src/db/cloudflare_d1/sql_lexer.rs rename to crates/dbx-drivers/src/db/cloudflare_d1/sql_lexer.rs diff --git a/crates/dbx-core/src/db/cloudflare_d1/sql_limits.rs b/crates/dbx-drivers/src/db/cloudflare_d1/sql_limits.rs similarity index 98% rename from crates/dbx-core/src/db/cloudflare_d1/sql_limits.rs rename to crates/dbx-drivers/src/db/cloudflare_d1/sql_limits.rs index c623e8b1e7..4373ee0e54 100644 --- a/crates/dbx-core/src/db/cloudflare_d1/sql_limits.rs +++ b/crates/dbx-drivers/src/db/cloudflare_d1/sql_limits.rs @@ -3,12 +3,12 @@ use std::ops::Range; pub const MAX_SQL_STATEMENT_BYTES: usize = 100_000; #[derive(Debug, PartialEq, Eq)] -pub(super) struct SqlBatch { +pub struct SqlBatch { pub sql: String, pub item_count: usize, } -pub(super) fn build_sql_batches( +pub fn build_sql_batches( item_count: usize, max_items: usize, item_label: &str, diff --git a/crates/dbx-core/src/db/cloudflare_d1/trigger.rs b/crates/dbx-drivers/src/db/cloudflare_d1/trigger.rs similarity index 100% rename from crates/dbx-core/src/db/cloudflare_d1/trigger.rs rename to crates/dbx-drivers/src/db/cloudflare_d1/trigger.rs diff --git a/crates/dbx-core/src/db/ddl_scan.rs b/crates/dbx-drivers/src/db/ddl_scan.rs similarity index 97% rename from crates/dbx-core/src/db/ddl_scan.rs rename to crates/dbx-drivers/src/db/ddl_scan.rs index 0d3f8634a1..46e36a4d8b 100644 --- a/crates/dbx-core/src/db/ddl_scan.rs +++ b/crates/dbx-drivers/src/db/ddl_scan.rs @@ -67,7 +67,7 @@ fn skip_dollar_quoted(sql: &str, pos: usize) -> Option { /// dollar-quoted content as opaque so parens or keywords inside a /// `DEFAULT`/`CHECK` expression can't confuse the count. Returns `None` if no /// top-level `(` is ever opened, or it's never closed. -pub(crate) fn find_top_level_paren_close(sql: &str, from: usize) -> Option { +pub fn find_top_level_paren_close(sql: &str, from: usize) -> Option { let mut i = from; let mut depth: usize = 0; let mut opened = false; @@ -134,7 +134,7 @@ fn find_top_level_semicolon(sql: &str, from: usize) -> Option { /// Splits `sql` into consecutive top-level statement ranges, each ending /// just past its own top-level `;`. The last statement may lack a trailing /// `;`, in which case its range runs to the end of the string. -pub(crate) fn top_level_statement_ranges(sql: &str) -> Vec> { +pub fn top_level_statement_ranges(sql: &str) -> Vec> { let mut ranges = Vec::new(); let mut start = 0; while start < sql.len() { @@ -155,7 +155,7 @@ pub(crate) fn top_level_statement_ranges(sql: &str) -> Vec> { /// If `stmt` (ignoring leading whitespace) begins with `CREATE TABLE /// "schema"."table"` or `CREATE FOREIGN TABLE "schema"."table"`, returns the /// unescaped `(schema, table)` pair. -pub(crate) fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> { +pub fn parse_create_table_relation(stmt: &str) -> Option<(String, String)> { let trimmed = stmt.trim_start(); let rest = trimmed.strip_prefix("CREATE FOREIGN TABLE ").or_else(|| trimmed.strip_prefix("CREATE TABLE "))?.trim_start(); diff --git a/crates/dbx-core/src/db/document_result.rs b/crates/dbx-drivers/src/db/document_result.rs similarity index 100% rename from crates/dbx-core/src/db/document_result.rs rename to crates/dbx-drivers/src/db/document_result.rs diff --git a/crates/dbx-core/src/db/dolt.rs b/crates/dbx-drivers/src/db/dolt.rs similarity index 100% rename from crates/dbx-core/src/db/dolt.rs rename to crates/dbx-drivers/src/db/dolt.rs diff --git a/crates/dbx-core/src/db/doris.rs b/crates/dbx-drivers/src/db/doris.rs similarity index 100% rename from crates/dbx-core/src/db/doris.rs rename to crates/dbx-drivers/src/db/doris.rs diff --git a/crates/dbx-core/src/db/duckdb_sql.rs b/crates/dbx-drivers/src/db/duckdb_sql.rs similarity index 100% rename from crates/dbx-core/src/db/duckdb_sql.rs rename to crates/dbx-drivers/src/db/duckdb_sql.rs diff --git a/crates/dbx-core/src/db/duckdb_worker_process.rs b/crates/dbx-drivers/src/db/duckdb_worker_process.rs similarity index 99% rename from crates/dbx-core/src/db/duckdb_worker_process.rs rename to crates/dbx-drivers/src/db/duckdb_worker_process.rs index d3b8775797..4330e56e67 100644 --- a/crates/dbx-core/src/db/duckdb_worker_process.rs +++ b/crates/dbx-drivers/src/db/duckdb_worker_process.rs @@ -19,7 +19,7 @@ use crate::db::duckdb_worker_protocol::{ DuckDbWorkerMethod, DuckDbWorkerObjectSourceParams, DuckDbWorkerRequest, DuckDbWorkerResponse, }; use crate::models::connection::AttachedDatabaseConfig; -use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT}; +use crate::runtime_config::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT}; /// Error code the worker reports when a query error left the DuckDB connection poisoned /// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the @@ -248,22 +248,22 @@ impl DuckDbWorkerClient { (Some(token), Some(duration)) => { tokio::select! { biased; - _ = token.cancelled() => self.cancel_or_kill(crate::query::canceled_error()).await, + _ = token.cancelled() => self.cancel_or_kill(crate::execution::canceled_error()).await, result = &mut future => result, - _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::query::timeout_error()).await, + _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::execution::timeout_error()).await, } } (Some(token), None) => { tokio::select! { biased; - _ = token.cancelled() => self.cancel_or_kill(crate::query::canceled_error()).await, + _ = token.cancelled() => self.cancel_or_kill(crate::execution::canceled_error()).await, result = &mut future => result, } } (None, Some(duration)) => { tokio::select! { result = &mut future => result, - _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::query::timeout_error()).await, + _ = tokio::time::sleep(duration) => self.cancel_or_kill(crate::execution::timeout_error()).await, } } (None, None) => future.await, diff --git a/crates/dbx-core/src/db/duckdb_worker_protocol.rs b/crates/dbx-drivers/src/db/duckdb_worker_protocol.rs similarity index 100% rename from crates/dbx-core/src/db/duckdb_worker_protocol.rs rename to crates/dbx-drivers/src/db/duckdb_worker_protocol.rs diff --git a/crates/dbx-core/src/db/dynamodb_driver.rs b/crates/dbx-drivers/src/db/dynamodb_driver.rs similarity index 100% rename from crates/dbx-core/src/db/dynamodb_driver.rs rename to crates/dbx-drivers/src/db/dynamodb_driver.rs diff --git a/crates/dbx-core/src/db/dynamodb_driver_disabled.rs b/crates/dbx-drivers/src/db/dynamodb_driver_disabled.rs similarity index 100% rename from crates/dbx-core/src/db/dynamodb_driver_disabled.rs rename to crates/dbx-drivers/src/db/dynamodb_driver_disabled.rs diff --git a/crates/dbx-core/src/db/easysearch_driver.rs b/crates/dbx-drivers/src/db/easysearch_driver.rs similarity index 100% rename from crates/dbx-core/src/db/easysearch_driver.rs rename to crates/dbx-drivers/src/db/easysearch_driver.rs diff --git a/crates/dbx-core/src/db/elasticsearch_driver.rs b/crates/dbx-drivers/src/db/elasticsearch_driver.rs similarity index 99% rename from crates/dbx-core/src/db/elasticsearch_driver.rs rename to crates/dbx-drivers/src/db/elasticsearch_driver.rs index 2f2ed2c33d..974ea798aa 100644 --- a/crates/dbx-core/src/db/elasticsearch_driver.rs +++ b/crates/dbx-drivers/src/db/elasticsearch_driver.rs @@ -2103,7 +2103,7 @@ fn validate_elasticsearch_ndjson(body: &str) -> Result { Ok(normalized) } -pub(crate) type SqlResponseParser = fn(&serde_json::Value, std::time::Instant) -> Option; +pub type SqlResponseParser = fn(&serde_json::Value, std::time::Instant) -> Option; pub async fn execute_rest_query(client: &EsClient, input: &str) -> Result { execute_rest_query_with_sql_parser(client, input, parse_sql_response, None).await @@ -2117,7 +2117,7 @@ pub async fn execute_rest_query_with_cursor( execute_rest_query_with_sql_parser(client, input, parse_sql_response, cursor).await } -pub(crate) async fn execute_rest_query_with_sql_parser( +pub async fn execute_rest_query_with_sql_parser( client: &EsClient, input: &str, sql_response_parser: SqlResponseParser, @@ -3057,7 +3057,7 @@ fn parse_sql_response(body: &serde_json::Value, start: std::time::Instant) -> Op parse_tabular_sql_response(body, start, "columns", "rows", None) } -pub(crate) fn parse_tabular_sql_response( +pub fn parse_tabular_sql_response( body: &serde_json::Value, start: std::time::Instant, columns_key: &str, diff --git a/crates/dbx-core/src/db/elasticsearch_sql.rs b/crates/dbx-drivers/src/db/elasticsearch_sql.rs similarity index 100% rename from crates/dbx-core/src/db/elasticsearch_sql.rs rename to crates/dbx-drivers/src/db/elasticsearch_sql.rs diff --git a/crates/dbx-core/src/db/file_validator.rs b/crates/dbx-drivers/src/db/file_validator.rs similarity index 100% rename from crates/dbx-core/src/db/file_validator.rs rename to crates/dbx-drivers/src/db/file_validator.rs diff --git a/crates/dbx-core/src/db/hbase_driver.rs b/crates/dbx-drivers/src/db/hbase_driver.rs similarity index 100% rename from crates/dbx-core/src/db/hbase_driver.rs rename to crates/dbx-drivers/src/db/hbase_driver.rs diff --git a/crates/dbx-core/src/db/http_tunnel.rs b/crates/dbx-drivers/src/db/http_tunnel.rs similarity index 100% rename from crates/dbx-core/src/db/http_tunnel.rs rename to crates/dbx-drivers/src/db/http_tunnel.rs diff --git a/crates/dbx-core/src/db/influxdb3_driver.rs b/crates/dbx-drivers/src/db/influxdb3_driver.rs similarity index 100% rename from crates/dbx-core/src/db/influxdb3_driver.rs rename to crates/dbx-drivers/src/db/influxdb3_driver.rs diff --git a/crates/dbx-core/src/db/influxdb_driver.rs b/crates/dbx-drivers/src/db/influxdb_driver.rs similarity index 100% rename from crates/dbx-core/src/db/influxdb_driver.rs rename to crates/dbx-drivers/src/db/influxdb_driver.rs diff --git a/crates/dbx-core/src/db/manticoresearch.rs b/crates/dbx-drivers/src/db/manticoresearch.rs similarity index 100% rename from crates/dbx-core/src/db/manticoresearch.rs rename to crates/dbx-drivers/src/db/manticoresearch.rs diff --git a/crates/dbx-core/src/db/meilisearch_driver.rs b/crates/dbx-drivers/src/db/meilisearch_driver.rs similarity index 100% rename from crates/dbx-core/src/db/meilisearch_driver.rs rename to crates/dbx-drivers/src/db/meilisearch_driver.rs diff --git a/crates/dbx-drivers/src/db/mod.rs b/crates/dbx-drivers/src/db/mod.rs new file mode 100644 index 0000000000..7bb5c604b4 --- /dev/null +++ b/crates/dbx-drivers/src/db/mod.rs @@ -0,0 +1,375 @@ +pub mod agent_driver; +pub mod clickhouse_driver; +pub mod cloudberry; +pub mod cloudflare_d1; +pub use cloudflare_d1 as cloudflare_d1_driver; +pub mod ddl_scan; +pub mod document_result; +pub mod dolt; +pub mod doris; +pub mod duckdb_sql; +#[cfg(feature = "duckdb-sidecar")] +pub mod duckdb_worker_process; +#[cfg(feature = "duckdb-sidecar")] +pub mod duckdb_worker_protocol; +#[cfg(feature = "dynamodb")] +#[path = "dynamodb_driver.rs"] +pub mod dynamodb_driver; +#[cfg(not(feature = "dynamodb"))] +#[path = "dynamodb_driver_disabled.rs"] +pub mod dynamodb_driver; +pub mod easysearch_driver; +pub mod elasticsearch_driver; +pub mod elasticsearch_sql; +pub mod file_validator; +pub mod hbase_driver; +pub mod http_tunnel; +pub mod influxdb3_driver; +pub mod influxdb_driver; +pub mod manticoresearch; +pub mod meilisearch_driver; +pub mod mongo_driver; +pub mod mysql; +pub mod mysql_compatible; +pub mod ob_oracle; +pub mod oceanbase_mysql; +pub mod opentenbase; +pub mod postgres; +pub mod proxy_tunnel; +pub mod questdb; +pub mod redis_driver; +pub mod rqlite_driver; +pub mod sqlite; +pub mod sqlite_worker; +pub mod sqlserver; +pub mod ssh_host_key; +pub use dbx_platform::ssh_prompt; +pub mod ssh_tunnel; +pub mod starrocks; +pub use dbx_sql::tdsql_mysql; +pub mod tidb; +pub mod transport_layer_tunnel; +pub mod turso_driver; +pub mod vector_driver; +pub mod victoriametrics_driver; +pub mod wkb; + +use crate::path_utils::expand_tilde; +use reqwest::ClientBuilder; +use reqwest::{Certificate, Identity}; +use std::fmt; +use std::future::Future; +use std::time::Duration; + +// Re-export types so that `db::QueryResult` etc. work within dbx-core +pub use crate::mysql_event_sql::MysqlEventInfo; +pub use crate::types::*; +pub use file_validator::validate_file_path; + +pub const CONNECTION_TIMEOUT_SECS: u64 = 5; +pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolCheckoutStage { + Wait, + Create, + Recycle, + Unknown, +} + +impl PoolCheckoutStage { + pub fn as_str(self) -> &'static str { + match self { + Self::Wait => "wait", + Self::Create => "create", + Self::Recycle => "recycle", + Self::Unknown => "unknown", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PoolCheckoutError { + Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration }, + Failed { database: &'static str, stage: PoolCheckoutStage, detail: String }, + Canceled, +} + +impl PoolCheckoutError { + pub fn stage(&self) -> Option { + match self { + Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage), + Self::Canceled => None, + } + } + + pub fn is_pool_saturation(&self) -> bool { + matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. }) + } +} + +impl fmt::Display for PoolCheckoutError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Timeout { database, stage, timeout } => write!( + formatter, + "{database} connection pool checkout timed out [stage={}, timeout_ms={}]", + stage.as_str(), + timeout.as_millis() + ), + Self::Failed { database, stage, detail } => { + write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str()) + } + Self::Canceled => formatter.write_str(crate::execution::QUERY_CANCELED), + } + } +} + +pub fn connection_timeout() -> Duration { + Duration::from_secs(CONNECTION_TIMEOUT_SECS) +} + +pub fn http_client_builder(timeout: Duration) -> ClientBuilder { + reqwest::Client::builder().connect_timeout(timeout).no_proxy() +} + +/// Augments an HTTP client builder with custom TLS material for drivers that +/// need to trust a non-system CA or present a client certificate (mTLS). +/// +/// * `ca_cert_path` adds an extra root certificate (PEM bundle) alongside the +/// system roots, so self-signed server certificates are trusted. +/// * `client_cert_path` + `client_key_path` (required together) are concatenated +/// into a single PEM identity, as expected by reqwest's rustls backend. +/// +/// Returns the augmented builder, or a descriptive error if a configured +/// certificate/key file cannot be read or parsed. Callers surface that error +/// instead of silently falling back to the system trust store, so a typo in a +/// cert path fails the connection loudly rather than being ignored. +pub fn apply_tls_certificates( + mut builder: ClientBuilder, + ca_cert_path: Option<&str>, + client_cert_path: Option<&str>, + client_key_path: Option<&str>, +) -> Result { + // The desktop build unifies reqwest with both `default-tls` and `rustls-tls`, + // and `TlsBackend` defaults to native-tls there, which rejects the PEM identity + // below with `incompatible TLS identity type`. Pin rustls like the Consul and + // etcd-metrics clients so custom CA roots and PEM client identities work in + // every target. + builder = builder.use_rustls_tls(); + if let Some(ca) = ca_cert_path.filter(|path| !path.is_empty()) { + let ca = expand_tilde(ca); + let contents = std::fs::read(&ca) + .map_err(|error| format!("Failed to read Elasticsearch CA certificate at {ca}: {error}"))?; + let certificates = Certificate::from_pem_bundle(&contents) + .map_err(|error| format!("Failed to parse Elasticsearch CA certificate at {ca}: {error}"))?; + for certificate in certificates { + builder = builder.add_root_certificate(certificate); + } + } + if let (Some(cert_path), Some(key_path)) = + (client_cert_path.filter(|path| !path.is_empty()), client_key_path.filter(|path| !path.is_empty())) + { + let cert_path = expand_tilde(cert_path); + let key_path = expand_tilde(key_path); + let mut pem = std::fs::read(&cert_path) + .map_err(|error| format!("Failed to read Elasticsearch client certificate at {cert_path}: {error}"))?; + // reqwest's rustls backend concatenates the cert and key as two PEM + // blocks; ensure they are newline-separated so parsing succeeds even + // when the cert file does not end with a trailing newline. + if !pem.ends_with(b"\n") { + pem.push(b'\n'); + } + pem.extend( + std::fs::read(&key_path) + .map_err(|error| format!("Failed to read Elasticsearch client key at {key_path}: {error}"))?, + ); + let identity = Identity::from_pem(&pem) + .map_err(|error| format!("Failed to parse Elasticsearch client identity: {error}"))?; + builder = builder.identity(identity); + } + Ok(builder) +} + +pub use dbx_types::json::JS_MAX_SAFE_INTEGER; +pub use dbx_types::json::{json_value_for_js, safe_i64_to_json, safe_u64_to_json}; + +pub fn hex_encode(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out +} + +pub fn binary_value_to_json(bytes: &[u8]) -> serde_json::Value { + serde_json::Value::String(format!("0x{}", hex_encode(bytes))) +} + +pub fn tcp_probe_timeout() -> Duration { + Duration::from_secs(TCP_PROBE_TIMEOUT_SECS) +} + +pub fn parse_connect_timeout(url: &str) -> Duration { + parse_connect_timeout_with_fallback(url, connection_timeout()) +} + +pub fn parse_connect_timeout_with_fallback(url: &str, fallback: Duration) -> Duration { + let Some(query) = url.split('?').nth(1) else { + return fallback; + }; + for param in query.split('&') { + let trimmed = param.trim(); + if trimmed.is_empty() { + continue; + } + let (key, value) = match trimmed.split_once('=') { + Some(pair) => pair, + None => continue, + }; + if key.eq_ignore_ascii_case("connect_timeout") + || key.eq_ignore_ascii_case("connectTimeout") + || key.eq_ignore_ascii_case("connection_timeout") + || key.eq_ignore_ascii_case("connectionTimeout") + { + if let Ok(v) = value.parse::() { + if (1..=300).contains(&v) { + return Duration::from_secs(v); + } + } + } + } + fallback +} + +pub async fn with_connection_timeout(label: &str, timeout: Duration, future: F) -> Result +where + F: Future>, +{ + tokio::time::timeout(timeout, future) + .await + .map_err(|_| format!("{label} connection timed out ({}s)", timeout.as_secs()))? +} + +pub async fn probe_tcp_endpoint(label: &str, host: &str, port: u16, timeout: Duration) -> Result<(), String> { + tokio::time::timeout(timeout, tokio::net::TcpStream::connect((host, port))) + .await + .map_err(|_| format!("{label} TCP connection timed out ({}s)", timeout.as_secs()))? + .map(|_| ()) + .map_err(|e| format!("{label} TCP connection failed: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn binary_values_are_displayed_as_prefixed_hex() { + assert_eq!(binary_value_to_json(&[0x00, 0x01, 0xab, 0xff]), serde_json::json!("0x0001abff")); + } +} + +#[cfg(test)] +mod tls_tests { + use super::{apply_tls_certificates, http_client_builder}; + use std::path::Path; + use std::process::Command; + use std::time::Duration; + + /// Generates a throwaway CA + client certificate/key (PEM) via openssl and + /// returns their paths. If openssl is unavailable the files stay empty, which + /// makes `apply_tls_certificates` surface a read error instead of degrading. + fn gen_certs(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) { + let ca_key = dir.join("ca.key"); + let ca_pem = dir.join("ca.pem"); + let csr = dir.join("client.csr"); + let client_key = dir.join("client.key"); + let client_crt = dir.join("client.crt"); + // Extensions force an X.509 v3 client cert; some toolchains (e.g. + // OpenSSL 3.0 on CI) emit v1 without one, which rustls rejects. + let client_ext = dir.join("client.ext"); + let _ = std::fs::write(&client_ext, "basicConstraints=critical,CA:FALSE\n"); + let _ = Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ca_key.to_str().unwrap(), + "-out", + ca_pem.to_str().unwrap(), + "-days", + "1", + "-subj", + "/CN=dbx-test-ca", + ]) + .output(); + let _ = Command::new("openssl") + .args([ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + client_key.to_str().unwrap(), + "-out", + csr.to_str().unwrap(), + "-subj", + "/CN=dbx-test-client", + ]) + .output(); + let _ = Command::new("openssl") + .args([ + "x509", + "-req", + "-in", + csr.to_str().unwrap(), + "-CA", + ca_pem.to_str().unwrap(), + "-CAkey", + ca_key.to_str().unwrap(), + "-CAcreateserial", + "-out", + client_crt.to_str().unwrap(), + "-days", + "1", + "-extfile", + client_ext.to_str().unwrap(), + ]) + .output(); + (ca_pem, client_crt, client_key) + } + + #[test] + fn apply_tls_certificates_builds_with_ca_and_client_cert() { + let dir = std::env::temp_dir().join(format!("dbx-tls-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (ca, client_crt, client_key) = gen_certs(&dir); + + let built = apply_tls_certificates( + http_client_builder(Duration::from_secs(5)), + ca.to_str(), + client_crt.to_str(), + client_key.to_str(), + ) + .expect("client should build with CA + client cert") + .build(); + assert!(built.is_ok(), "built client should be usable with CA + client cert: {:?}", built.err()); + + // A misconfigured CA path must now surface a descriptive error instead of + // silently degrading to the system trust store. + let missing = apply_tls_certificates( + http_client_builder(Duration::from_secs(5)), + Some("/nonexistent/ca.pem"), + None, + None, + ); + assert!(missing.is_err(), "missing CA path must surface a read error"); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/dbx-core/src/db/mongo_driver.rs b/crates/dbx-drivers/src/db/mongo_driver.rs similarity index 99% rename from crates/dbx-core/src/db/mongo_driver.rs rename to crates/dbx-drivers/src/db/mongo_driver.rs index 1e5dd94293..88416ace59 100644 --- a/crates/dbx-core/src/db/mongo_driver.rs +++ b/crates/dbx-drivers/src/db/mongo_driver.rs @@ -9,9 +9,9 @@ use mongodb::{ use serde::{Deserialize, Serialize}; use super::with_connection_timeout; -use crate::document_ops::{MongoGridFsBucketInfo, MongoGridFsFileInfo}; use crate::models::connection::DatabaseConnectionInfo; use crate::types::IndexInfo; +use dbx_types::document::{MongoGridFsBucketInfo, MongoGridFsFileInfo}; use futures::{io::AsyncReadExt, io::AsyncWriteExt, TryStreamExt}; use percent_encoding::percent_decode_str; use std::{collections::HashSet, time::Duration}; @@ -852,11 +852,7 @@ fn is_automatic_id_index(index: &IndexModel) -> bool { || matches!(index.keys.get("_id"), Some(Bson::Double(value)) if *value == 1.0)) } -pub(crate) fn validate_clone_collection_names( - database: &str, - source_name: &str, - target_name: &str, -) -> Result<(), String> { +pub fn validate_clone_collection_names(database: &str, source_name: &str, target_name: &str) -> Result<(), String> { validate_mongo_namespace_name(database, "Database")?; validate_mongo_namespace_name(source_name, "Source collection")?; validate_mongo_namespace_name(target_name, "Target collection")?; diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-drivers/src/db/mysql.rs similarity index 99% rename from crates/dbx-core/src/db/mysql.rs rename to crates/dbx-drivers/src/db/mysql.rs index b065890ba5..1b9d087422 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-drivers/src/db/mysql.rs @@ -22,7 +22,6 @@ use crate::models::connection::{ is_mysql_jdbc_tls_param, mysql_jdbc_tls_mode, ConnectionConfig, DatabaseConnectionInfo, DatabaseType, MysqlJdbcTlsMode, }; -use crate::schema::{table_name_filter_matches, TableNameFilter}; use crate::sql::{starts_with_executable_sql_keyword, starts_with_executable_sql_keyword_for_database}; use crate::types::{ ColumnInfo, CompletionAssistantCandidate, CompletionAssistantCandidateKind, CompletionAssistantMatchMode, @@ -30,6 +29,7 @@ use crate::types::{ ForeignKeyInfo, IndexInfo, LargeValueCell, ObjectInfo, ObjectStatistics, QueryMessage, QueryResult, SpatialColumnBuilder, TableInfo, TriggerInfo, }; +use dbx_types::metadata_filter::{table_name_filter_matches, TableNameFilter}; use super::file_validator::validate_file_path; use crate::mysql_event_sql::MysqlEventInfo; @@ -44,7 +44,7 @@ pub struct MySqlPool { } impl MySqlPool { - pub(crate) fn new(opts: O, max_connections: usize) -> Self + pub fn new(opts: O, max_connections: usize) -> Self where mysql_async::Opts: TryFrom, >::Error: std::error::Error, @@ -60,8 +60,8 @@ impl MySqlPool { /// generation. Pool options are not an identity: a reconnect creates a new /// pool with identical options, and a late health probe for the old pool /// must not be allowed to remove that replacement from routing. - #[cfg(test)] - pub(crate) fn is_same_pool(&self, other: &Self) -> bool { + #[cfg(any(test, feature = "test-support"))] + pub fn is_same_pool(&self, other: &Self) -> bool { std::sync::Arc::ptr_eq(&self.inner.metrics(), &other.inner.metrics()) } } @@ -141,15 +141,12 @@ const MYSQL_TCP_KEEPALIVE_MS: u32 = 30_000; const MYSQL_SQL_PACKET_MARGIN_MAX_BYTES: usize = 64 * 1024; #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum MySqlCatalogDialect { +pub enum MySqlCatalogDialect { Doris, StarRocks, } -pub(crate) fn mysql_catalog_dialect( - db_type: DatabaseType, - driver_profile: Option<&str>, -) -> Option { +pub fn mysql_catalog_dialect(db_type: DatabaseType, driver_profile: Option<&str>) -> Option { if super::doris::is_profile(&db_type, driver_profile) { Some(MySqlCatalogDialect::Doris) } else if super::starrocks::is_profile(&db_type, driver_profile) { @@ -469,7 +466,7 @@ fn mysql_bytes_to_json(bytes: Vec, column: &mysql_async::Column) -> serde_js /// families. Binary charset (63) means BLOB; any other charset means TEXT. Value /// decoding already follows that rule — the header type must match, or TEXT /// columns flash as `blob` until table metadata arrives. -pub(crate) fn mysql_column_type_name(column: &mysql_async::Column) -> String { +pub fn mysql_column_type_name(column: &mysql_async::Column) -> String { use mysql_async::consts::ColumnType::*; let ty = column.column_type(); let flags = column.flags(); @@ -547,7 +544,7 @@ pub(crate) fn mysql_column_type_name(column: &mysql_async::Column) -> String { .to_string() } -pub(crate) fn mysql_value_to_json(row: &mysql_async::Row, idx: usize) -> serde_json::Value { +pub fn mysql_value_to_json(row: &mysql_async::Row, idx: usize) -> serde_json::Value { let Some(column) = row.columns_ref().get(idx) else { return serde_json::Value::Null; }; @@ -1527,11 +1524,11 @@ fn catalog_switch_query(dialect: MySqlCatalogDialect, catalog: &str) -> String { } } -pub(crate) fn catalog_setup_query_for_url(dialect: MySqlCatalogDialect, url: &str) -> Option { +pub fn catalog_setup_query_for_url(dialect: MySqlCatalogDialect, url: &str) -> Option { mysql_connection_catalog(url).map(|catalog| catalog_switch_query(dialect, &catalog)) } -pub(crate) fn catalog_database_context_queries( +pub fn catalog_database_context_queries( dialect: Option, catalog: Option<&str>, database: &str, @@ -1548,7 +1545,7 @@ pub(crate) fn catalog_database_context_queries( Ok(queries) } -pub(crate) async fn apply_catalog_database_context( +pub async fn apply_catalog_database_context( conn: &mut mysql_async::Conn, dialect: Option, catalog: Option<&str>, @@ -3810,7 +3807,7 @@ fn apply_mysql_generated_column_expressions(columns: &mut [ColumnInfo], expressi /// Example: "主键" → UTF-8 bytes [E4 B8 BB E9 94 AE] /// → each byte → CP1252 char → UTF-8 re-encoded → garbled text /// → reversal: map each char back to its CP1252 byte, decode as UTF-8 -pub(crate) fn fix_potential_double_encoding(s: &str) -> String { +pub fn fix_potential_double_encoding(s: &str) -> String { // Map each character to its CP1252 byte value let mut bytes = Vec::with_capacity(s.len()); for c in s.chars() { @@ -4052,7 +4049,7 @@ fn show_columns_sql(database: &str, table: &str, full: bool) -> String { } fn query_result_row_limit(max_rows: Option) -> usize { - max_rows.unwrap_or(crate::query::MAX_ROWS).max(1) + max_rows.unwrap_or(crate::execution::MAX_ROWS).max(1) } fn should_collect_text_result_set(sql: &str, row_limit: usize, max_rows: Option) -> bool { @@ -4203,7 +4200,7 @@ where ); Ok(conn) } - Err(err) if err == crate::query::QUERY_CANCELED => { + Err(err) if err == crate::execution::QUERY_CANCELED => { let _ = tokio::time::timeout(cleanup_timeout, conn.disconnect()).await; Err(err) } @@ -4217,7 +4214,7 @@ where let _ = tokio::time::timeout(cleanup_timeout, conn.disconnect()).await; let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?; if let Err(err) = ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await { - if err == crate::query::QUERY_CANCELED { + if err == crate::execution::QUERY_CANCELED { let _ = tokio::time::timeout(cleanup_timeout, conn.disconnect()).await; } return Err(err); @@ -4233,15 +4230,15 @@ where } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct MysqlCheckoutSnapshot { - pub(crate) connection_count: usize, - pub(crate) connections_in_pool: usize, - pub(crate) max_connections: usize, +pub struct MysqlCheckoutSnapshot { + pub connection_count: usize, + pub connections_in_pool: usize, + pub max_connections: usize, } -pub(crate) type MysqlCheckoutStage = super::PoolCheckoutStage; +pub type MysqlCheckoutStage = super::PoolCheckoutStage; -pub(crate) fn classify_mysql_checkout_stage(snapshot: MysqlCheckoutSnapshot) -> MysqlCheckoutStage { +pub fn classify_mysql_checkout_stage(snapshot: MysqlCheckoutSnapshot) -> MysqlCheckoutStage { if snapshot.connections_in_pool > 0 { MysqlCheckoutStage::Recycle } else if snapshot.connection_count < snapshot.max_connections { @@ -4264,10 +4261,7 @@ where } } -pub(crate) async fn checkout_mysql_conn

( - pool: &P, - timeout: Duration, -) -> Result +pub async fn checkout_mysql_conn

(pool: &P, timeout: Duration) -> Result where P: MySqlPoolAccess + ?Sized, { @@ -4298,7 +4292,7 @@ where Some(token) => { tokio::select! { biased; - _ = token.cancelled() => Err(crate::query::canceled_error()), + _ = token.cancelled() => Err(crate::execution::canceled_error()), result = get_future => result, } } @@ -4337,7 +4331,7 @@ async fn ping_conn_with_timeout_and_cancel( Some(token) => { tokio::select! { biased; - _ = token.cancelled() => Err(crate::query::canceled_error()), + _ = token.cancelled() => Err(crate::execution::canceled_error()), result = ping_future => result, } } @@ -4428,7 +4422,7 @@ async fn execute_result_set_with_text_protocol_on_conn( result_key_columns: &[String], diagnostic_trace_id: Option<&str>, start: Instant, - progress_clock: Option<&crate::query::StreamProgressClock>, + progress_clock: Option<&crate::execution::StreamProgressClock>, ) -> Result { let diagnostics_enabled = diagnostic_trace_id.is_some() && log::log_enabled!(log::Level::Info); let dispatch_started_at = diagnostics_enabled.then(Instant::now); @@ -4821,7 +4815,7 @@ async fn execute_result_set_with_prepared_protocol_on_conn( result_key_columns: &[String], diagnostic_trace_id: Option<&str>, start: Instant, - progress_clock: Option<&crate::query::StreamProgressClock>, + progress_clock: Option<&crate::execution::StreamProgressClock>, ) -> Result { let diagnostics_enabled = diagnostic_trace_id.is_some() && log::log_enabled!(log::Level::Info); let dispatch_started_at = diagnostics_enabled.then(Instant::now); @@ -4937,14 +4931,14 @@ pub async fn max_allowed_packet(pool: &MySqlPool) -> Result { max_allowed_packet_on_conn(&mut conn).await } -pub(crate) async fn max_allowed_packet_on_conn(conn: &mut mysql_async::Conn) -> Result { +pub async fn max_allowed_packet_on_conn(conn: &mut mysql_async::Conn) -> Result { query_first_column::(conn, "SELECT @@max_allowed_packet") .await .map_err(|error| error.to_string())? .ok_or_else(|| "MySQL did not return @@max_allowed_packet".to_string()) } -pub(crate) fn mysql_sql_statement_hard_limit(max_allowed_packet: u64) -> Option { +pub fn mysql_sql_statement_hard_limit(max_allowed_packet: u64) -> Option { let packet_bytes = usize::try_from(max_allowed_packet).ok()?; if packet_bytes == 0 { return None; @@ -4976,13 +4970,13 @@ where /// Unlike the Postgres/SQL Server variants there is no returns-rows gate here: /// a write delivers no rows, so no progress mark ever resets the clock and the /// budget degrades to the same plain wall-clock timeout anyway. -pub(crate) async fn execute_query_with_max_rows_progress

( +pub async fn execute_query_with_max_rows_progress

( pool: &P, sql: &str, bare: bool, max_rows: Option, dialect: MySqlQueryDialect, - progress_clock: std::sync::Arc, + progress_clock: std::sync::Arc, timeout: Option, ) -> Result where @@ -4991,7 +4985,7 @@ where let mut conn = get_conn_with_health_check(pool).await?; let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs())); let clock_for_query = progress_clock.clone(); - crate::query::await_stream_with_progress_timeout( + crate::execution::await_stream_with_progress_timeout( async move { execute_query_on_conn_with_limits_progress( &mut conn, @@ -5084,7 +5078,7 @@ async fn stream_query_result_text( while let Some(row) = stream.next().await { if cancelled.load(std::sync::atomic::Ordering::SeqCst) { - return Err(crate::query::canceled_error()); + return Err(crate::execution::canceled_error()); } if rows_exported as usize >= row_limit { break; @@ -5125,7 +5119,7 @@ async fn stream_query_result_prepared( while let Some(row) = stream.next().await { if cancelled.load(std::sync::atomic::Ordering::SeqCst) { - return Err(crate::query::canceled_error()); + return Err(crate::execution::canceled_error()); } if rows_exported as usize >= row_limit { break; @@ -5238,10 +5232,10 @@ pub async fn execute_query_on_conn_with_limits( /// [`execute_query_on_conn_with_limits`] with an optional progress clock. /// /// The clock is marked for every row the server delivers, so a caller wrapping -/// this in an inactivity budget (see [`crate::query::await_stream_with_progress_timeout`]) +/// this in an inactivity budget (see [`crate::execution::await_stream_with_progress_timeout`]) /// only times out on a genuine stall, not on a long-but-steady stream — which is /// what lets a transfer of a large table survive the configured timeout. -pub(crate) async fn execute_query_on_conn_with_limits_progress( +pub async fn execute_query_on_conn_with_limits_progress( conn: &mut mysql_async::Conn, sql: &str, bare: bool, @@ -5250,7 +5244,7 @@ pub(crate) async fn execute_query_on_conn_with_limits_progress( result_key_columns: &[String], dialect: MySqlQueryDialect, diagnostic_trace_id: Option<&str>, - progress_clock: Option<&crate::query::StreamProgressClock>, + progress_clock: Option<&crate::execution::StreamProgressClock>, ) -> Result { let start = Instant::now(); let row_limit = query_result_row_limit(max_rows); @@ -5418,7 +5412,7 @@ pub async fn execute_query_results_on_conn_with_limits( } #[derive(Debug)] -pub(crate) struct MySqlNonResultBatchOutcome { +pub struct MySqlNonResultBatchOutcome { pub results: Vec, pub error: Option, } @@ -5428,7 +5422,7 @@ pub(crate) struct MySqlNonResultBatchOutcome { /// one OK packet per statement, so affected-row counts and failure positions /// remain attributable to the original statements without one network round /// trip per statement. -pub(crate) async fn execute_non_result_batch_on_conn( +pub async fn execute_non_result_batch_on_conn( conn: &mut mysql_async::Conn, sql: &str, expected_results: usize, @@ -5535,7 +5529,7 @@ fn prefers_text_protocol_query(sql: &str, dialect: MySqlQueryDialect) -> bool { is_result_set_query(sql, dialect) || requires_text_protocol_query(sql, dialect) } -pub(crate) fn is_result_set_query(sql: &str, dialect: MySqlQueryDialect) -> bool { +pub fn is_result_set_query(sql: &str, dialect: MySqlQueryDialect) -> bool { // MySQL 的表维护语句虽然不是 SELECT，但服务器会返回包含表名和执行结果的表格。 // 如果把它们当成普通写入语句，后续 drop_result 会直接丢弃这些返回行。 starts_with_executable_sql_keyword_for_database( @@ -5550,7 +5544,7 @@ pub(crate) fn is_result_set_query(sql: &str, dialect: MySqlQueryDialect) -> bool || dialect.supports_admin_show_results && is_admin_show_query(sql) } -pub(crate) fn is_batchable_non_result_query(sql: &str, dialect: MySqlQueryDialect) -> bool { +pub fn is_batchable_non_result_query(sql: &str, dialect: MySqlQueryDialect) -> bool { !is_result_set_query(sql, dialect) && !is_mysql_add_column_if_not_exists(sql) && starts_with_executable_sql_keyword_for_database( @@ -5599,7 +5593,7 @@ fn mysql_add_column_if_not_exists(sql: &str) -> Option bool { +pub fn is_mysql_add_column_if_not_exists(sql: &str) -> bool { mysql_add_column_if_not_exists(sql).is_some() } @@ -6033,7 +6027,7 @@ mod tests { let sql = "WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 20) \ SELECT REPEAT('x', 20000) AS payload, n FROM seq WHERE SLEEP(0.05) = 0"; - let progress_clock = std::sync::Arc::new(crate::query::StreamProgressClock::new()); + let progress_clock = std::sync::Arc::new(crate::execution::StreamProgressClock::new()); let result = execute_query_with_max_rows_progress( &pool, sql, @@ -6049,10 +6043,10 @@ mod tests { // Contrast: the same statement under a never-reset wall-clock budget must // time out, proving this test actually exercises the difference. - let wall_clock = crate::query::await_stream_with_progress_timeout( + let wall_clock = crate::execution::await_stream_with_progress_timeout( execute_query_with_max_rows(&pool, sql, false, None, Default::default()), Some(Duration::from_millis(200)), - std::sync::Arc::new(crate::query::StreamProgressClock::new()), + std::sync::Arc::new(crate::execution::StreamProgressClock::new()), None, "Query timed out after 0 seconds".to_string(), ) diff --git a/crates/dbx-core/src/db/mysql_compatible.rs b/crates/dbx-drivers/src/db/mysql_compatible.rs similarity index 100% rename from crates/dbx-core/src/db/mysql_compatible.rs rename to crates/dbx-drivers/src/db/mysql_compatible.rs diff --git a/crates/dbx-core/src/db/ob_oracle.rs b/crates/dbx-drivers/src/db/ob_oracle.rs similarity index 100% rename from crates/dbx-core/src/db/ob_oracle.rs rename to crates/dbx-drivers/src/db/ob_oracle.rs diff --git a/crates/dbx-core/src/db/oceanbase_mysql.rs b/crates/dbx-drivers/src/db/oceanbase_mysql.rs similarity index 100% rename from crates/dbx-core/src/db/oceanbase_mysql.rs rename to crates/dbx-drivers/src/db/oceanbase_mysql.rs diff --git a/crates/dbx-core/src/db/opentenbase.rs b/crates/dbx-drivers/src/db/opentenbase.rs similarity index 100% rename from crates/dbx-core/src/db/opentenbase.rs rename to crates/dbx-drivers/src/db/opentenbase.rs diff --git a/crates/dbx-core/src/db/postgres.rs b/crates/dbx-drivers/src/db/postgres.rs similarity index 94% rename from crates/dbx-core/src/db/postgres.rs rename to crates/dbx-drivers/src/db/postgres.rs index 8a936a9291..fc822c55c3 100644 --- a/crates/dbx-core/src/db/postgres.rs +++ b/crates/dbx-drivers/src/db/postgres.rs @@ -29,8 +29,8 @@ use tokio_postgres::{AsyncMessage, NoTls, Row, SimpleQueryMessage, Socket}; use tokio_util::sync::CancellationToken; use super::file_validator::validate_file_path; +use crate::execution::{await_stream_with_progress_timeout, DbOperationBudget, StreamProgressClock}; use crate::models::connection::DatabaseType; -use crate::query::{await_stream_with_progress_timeout, DbOperationBudget, StreamProgressClock}; use crate::sql::starts_with_executable_sql_keyword; use crate::types::{ ColumnInfo, CompletionAssistantCandidate, CompletionAssistantCandidateKind, CompletionAssistantMatchMode, @@ -41,7 +41,7 @@ use crate::types::{ SpatialColumnBuilder, TableInfo, TriggerInfo, }; -pub(crate) const GAUSSDB_COMPATIBILITY_SQL: &str = +pub const GAUSSDB_COMPATIBILITY_SQL: &str = "SELECT datcompatibility FROM pg_catalog.pg_database WHERE datname = current_database()"; pub async fn gaussdb_identifier_quote(pool: &Pool) -> Option { @@ -52,7 +52,7 @@ pub async fn gaussdb_identifier_quote(pool: &Pool) -> Option { gaussdb_identifier_quote_for_compatibility_mode(&compatibility_mode).map(str::to_string) } -pub(crate) fn gaussdb_identifier_quote_for_compatibility_mode(compatibility_mode: &str) -> Option<&'static str> { +pub fn gaussdb_identifier_quote_for_compatibility_mode(compatibility_mode: &str) -> Option<&'static str> { match compatibility_mode.trim().to_ascii_uppercase().as_str() { "M" | "B" | "MYSQL" => Some("`"), "A" | "PG" | "ORA" | "POSTGRESQL" => Some("\""), @@ -884,7 +884,7 @@ fn format_pg_timestamptz(value: DateTime) -> String { /// 时间类型解码失败后的回退目标，与原 if 链中时间分支之后的匹配顺序一一对应。 #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum PgTemporalFallback { +pub enum PgTemporalFallback { /// 数组类型名（下划线开头）落到通用数组解码。 GenericArray, /// `VECTOR(...)` 形式的类型名落到 pgvector 解码。 @@ -895,7 +895,7 @@ pub(crate) enum PgTemporalFallback { /// 每列一次的类型分类结果，避免在逐单元格路径上重复 `to_uppercase` 与字符串比较链。 #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(crate) enum PgColType { +pub enum PgColType { Bytea, Json, Bool, @@ -984,7 +984,7 @@ fn pg_type_requires_text_protocol(pg_type: &Type, col_type: PgColType) -> bool { } } -pub(crate) fn classify_pg_type(type_name: &str) -> PgColType { +pub fn classify_pg_type(type_name: &str) -> PgColType { let upper = type_name.to_uppercase(); let temporal_type_name = upper.strip_prefix('_').unwrap_or(&upper); @@ -1064,11 +1064,11 @@ pub(crate) fn classify_pg_type(type_name: &str) -> PgColType { PgColType::Other } -pub(crate) fn classify_pg_column_types(column_types: &[String]) -> Vec { +pub fn classify_pg_column_types(column_types: &[String]) -> Vec { column_types.iter().map(|type_name| classify_pg_type(type_name)).collect() } -pub(crate) fn pg_value_to_json_classified(row: &Row, idx: usize, col_type: PgColType) -> serde_json::Value { +pub fn pg_value_to_json_classified(row: &Row, idx: usize, col_type: PgColType) -> serde_json::Value { match col_type { PgColType::Bytea => row .try_get::<_, Vec>(idx) @@ -2078,7 +2078,7 @@ async fn finish_prepared_select( } } -pub(crate) async fn execute_select_query( +pub async fn execute_select_query( client: &deadpool_postgres::Client, sql: &str, start: Instant, @@ -2087,7 +2087,7 @@ pub(crate) async fn execute_select_query( execute_select_query_with_progress(client, sql, start, row_limit, None, false).await } -pub(crate) async fn execute_select_query_unnamed( +pub async fn execute_select_query_unnamed( client: &deadpool_postgres::Client, sql: &str, start: Instant, @@ -2241,7 +2241,7 @@ async fn stream_select_query_text( } #[cfg(test)] -pub(crate) async fn stream_select_query_inner_unnamed( +pub async fn stream_select_query_inner_unnamed( client: &deadpool_postgres::Client, sql: &str, row_limit: Option, @@ -2250,7 +2250,7 @@ pub(crate) async fn stream_select_query_inner_unnamed( stream_select_query_inner_with_mode(client, sql, row_limit, on_item, true).await } -pub(crate) async fn stream_select_query_inner_unnamed_with_cancel( +pub async fn stream_select_query_inner_unnamed_with_cancel( client: &deadpool_postgres::Client, sql: &str, row_limit: Option, @@ -2260,7 +2260,7 @@ pub(crate) async fn stream_select_query_inner_unnamed_with_cancel( cancel_context: Option<&PostgresCancelContext>, ) -> Result { if cancel_token.is_some_and(CancellationToken::is_cancelled) { - return Err(crate::query::canceled_error()); + return Err(crate::execution::canceled_error()); } let pg_cancel_token = client.cancel_token(); let progress_clock = Arc::new(StreamProgressClock::new()); @@ -2286,7 +2286,7 @@ pub(crate) async fn stream_select_query_inner_unnamed_with_cancel( ) .await; - if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) { + if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::execution::QUERY_CANCELED) { let original_error = result.as_ref().expect_err("timeout or cancellation result").clone(); cancel_postgres_query(pg_cancel_token, cancel_context, budget.cancel_timeout).await; if tokio::time::timeout(budget.cleanup_timeout, stream.as_mut()).await.is_err() { @@ -2390,7 +2390,7 @@ async fn stream_query_rows_on_client( while let Some(row_result) = stream.next().await { if cancelled.load(std::sync::atomic::Ordering::SeqCst) { - return Err(crate::query::canceled_error()); + return Err(crate::execution::canceled_error()); } if rows_exported as usize >= row_limit { break; @@ -2421,7 +2421,7 @@ async fn stream_query_rows_text_on_client( while let Some(message) = stream.next().await { if cancelled.load(std::sync::atomic::Ordering::SeqCst) { - return Err(crate::query::canceled_error()); + return Err(crate::execution::canceled_error()); } if rows_exported as usize >= row_limit { break; @@ -4486,7 +4486,7 @@ pub async fn list_foreign_keys_for_relations( .await } -async fn list_foreign_keys_for_relations_with_sql( +pub async fn list_foreign_keys_for_relations_with_sql( client: &deadpool_postgres::Client, sql: &str, schemas: &[&str], @@ -4513,7 +4513,7 @@ fn postgres_foreign_keys_for_relations_query_tiers() -> [&'static str; 2] { [postgres_foreign_keys_for_relations_sql(), postgres_foreign_keys_for_relations_compat_sql()] } -fn postgres_foreign_keys_for_relations_sql() -> &'static str { +pub fn postgres_foreign_keys_for_relations_sql() -> &'static str { "SELECT rel.rel_schema, rel.rel_table, \ con.conname AS constraint_name, \ a.attname AS column_name, \ @@ -4546,7 +4546,7 @@ fn postgres_foreign_keys_for_relations_sql() -> &'static str { // reference an earlier FROM item (implicit LATERAL), so bound the series by // INDEX_MAX_KEYS (32, stable across PostgreSQL 9–18) and cap the ordinal with // a join-condition guard instead. -fn postgres_foreign_keys_for_relations_compat_sql() -> &'static str { +pub fn postgres_foreign_keys_for_relations_compat_sql() -> &'static str { "SELECT rel.rel_schema, rel.rel_table, \ con.conname AS constraint_name, \ a.attname AS column_name, \ @@ -4673,7 +4673,7 @@ pub async fn get_table_partition_local_objects_for_relations( .await } -async fn get_table_partition_local_objects_for_relations_with_sql( +pub async fn get_table_partition_local_objects_for_relations_with_sql( client: &deadpool_postgres::Client, sql: &str, oids: &[i64], @@ -4697,7 +4697,7 @@ fn postgres_table_partition_local_objects_for_relations_query_tiers() -> [&'stat ] } -fn postgres_table_partition_local_objects_for_relations_sql() -> &'static str { +pub fn postgres_table_partition_local_objects_for_relations_sql() -> &'static str { "SELECT con.conrelid::bigint AS relid, 'constraint'::text AS object_kind, con.conname AS object_name, con.contype::text AS object_type \ FROM pg_catalog.pg_constraint con \ WHERE con.conrelid = ANY($1::bigint[]) AND con.contype IN ('p','u','f') \ @@ -4736,7 +4736,7 @@ fn postgres_table_partition_local_objects_for_relations_sql() -> &'static str { // column. Pre-11 servers do not have parent-linked constraint clones, so the // compatibility tier can omit only that filter and keep the same batched // result shape. -fn postgres_table_partition_local_objects_for_relations_compat_sql() -> &'static str { +pub fn postgres_table_partition_local_objects_for_relations_compat_sql() -> &'static str { "SELECT con.conrelid::bigint AS relid, 'constraint'::text AS object_kind, con.conname AS object_name, con.contype::text AS object_type \ FROM pg_catalog.pg_constraint con \ WHERE con.conrelid = ANY($1::bigint[]) AND con.contype IN ('p','u','f') \ @@ -7170,7 +7170,7 @@ pub async fn get_columns(pool: &Pool, schema: &str, table: &str) -> Result String { +pub fn pg_quote_literal(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } @@ -7234,7 +7234,7 @@ pub async fn get_redshift_columns(pool: &Pool, schema: &str, table: &str) -> Res &client, &redshift_columns_sql(schema, table), Instant::now(), - crate::query::MAX_ROWS, + crate::execution::MAX_ROWS, None, None, ) @@ -7242,7 +7242,7 @@ pub async fn get_redshift_columns(pool: &Pool, schema: &str, table: &str) -> Res Ok(redshift_columns_from_query_result(result)) } -pub(crate) fn pg_quote_ident(ident: &str) -> String { +pub fn pg_quote_ident(ident: &str) -> String { format!("\"{}\"", ident.replace('"', "\"\"")) } @@ -7258,7 +7258,7 @@ fn pg_quote_catalog_identifier(ident: &str) -> String { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum PostgresSearchPathContext { +pub enum PostgresSearchPathContext { Query, Transaction, LocalTransaction, @@ -7275,7 +7275,7 @@ struct PostgresSearchPathBaseline { type PostgresSearchPathBaselineEntry = (Weak, PostgresSearchPathBaseline); type PostgresSearchPathBaselineCache = Mutex>; -pub(crate) fn postgres_set_search_path_sql(schema: &str, context: PostgresSearchPathContext) -> String { +pub fn postgres_set_search_path_sql(schema: &str, context: PostgresSearchPathContext) -> String { let (scope, suffix) = match context { // Ordinary queries and exports historically fall back to public for // extensions and helper functions after checking the selected schema. @@ -7459,7 +7459,7 @@ async fn postgres_search_path_baseline( Ok(baseline) } -pub(crate) async fn set_postgres_search_path( +pub async fn set_postgres_search_path( client: &deadpool_postgres::Client, schema: &str, context: PostgresSearchPathContext, @@ -7503,7 +7503,7 @@ pub(crate) async fn set_postgres_search_path( } fn query_result_row_limit(max_rows: Option) -> usize { - max_rows.unwrap_or(crate::query::MAX_ROWS).max(1) + max_rows.unwrap_or(crate::execution::MAX_ROWS).max(1) } /// Returns whether a parsed DML statement produces a result set, or `None` when @@ -7533,7 +7533,7 @@ fn postgres_query_body_dml(body: &SetExpr) -> Option<&Statement> { /// Returns whether PostgreSQL should execute this statement through the row /// retrieval path. DML without `RETURNING` needs `execute` for its command /// tag/affected-row count, while DML with `RETURNING` produces a result set. -pub(crate) fn postgres_statement_returns_rows(sql: &str) -> bool { +pub fn postgres_statement_returns_rows(sql: &str) -> bool { if starts_with_executable_sql_keyword(sql, &["SELECT", "SHOW", "EXPLAIN", "TABLE"]) { return true; } @@ -7626,7 +7626,7 @@ pub async fn execute_query_with_max_rows( /// a genuine stall (no row for the whole timeout) is reported as a timeout. /// Statements that return no rows have no incremental progress to report, so /// they keep the plain wall-clock path. -pub(crate) async fn execute_query_with_max_rows_progress( +pub async fn execute_query_with_max_rows_progress( pool: &Pool, sql: &str, max_rows: Option, @@ -7636,7 +7636,8 @@ pub(crate) async fn execute_query_with_max_rows_progress( if !postgres_statement_returns_rows(sql) { // DDL/DML expose no incremental progress, so keep the original wall-clock // budget: a hung write must still be bounded by the configured timeout. - return crate::query::wait_for_query_opt(None, timeout, execute_query_with_max_rows(pool, sql, max_rows)).await; + return crate::execution::wait_for_query_opt(None, timeout, execute_query_with_max_rows(pool, sql, max_rows)) + .await; } let start = Instant::now(); @@ -7841,7 +7842,8 @@ pub async fn stream_select_query_with_cancel( timeout_error.clone(), ) .await; - if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) { + if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::execution::QUERY_CANCELED) + { cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), budget.cancel_timeout).await; } result @@ -8004,7 +8006,7 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel( /// post-query schema cleanup must re-issue `SET search_path TO DEFAULT` for /// those engines. Every other backend — and an unknown (`None`) type — keeps /// the historical `RESET search_path` behavior. -pub(crate) fn reset_search_path_sql(db_type: Option) -> &'static str { +pub fn reset_search_path_sql(db_type: Option) -> &'static str { match db_type { Some(DatabaseType::Gaussdb | DatabaseType::OpenGauss) => "SET search_path TO DEFAULT", _ => "RESET search_path", @@ -8057,7 +8059,7 @@ fn postgres_schema_reset_cleanup_error(err: String) -> String { format!("PostgreSQL schema.reset cleanup failed: {err}") } -pub(crate) async fn execute_postgres_infra_statement( +pub async fn execute_postgres_infra_statement( client: &deadpool_postgres::Client, sql: &str, timeout_duration: Duration, @@ -8069,7 +8071,7 @@ pub(crate) async fn execute_postgres_infra_statement( .map_err(pg_error_to_string_plain) } -pub(crate) async fn wait_postgres_operation( +pub async fn wait_postgres_operation( pg_cancel_token: tokio_postgres::CancelToken, cancel_context: Option, timeout_duration: Option, @@ -8099,7 +8101,7 @@ where biased; _ = token.cancelled() => { cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await; - Err(crate::query::canceled_error()) + Err(crate::execution::canceled_error()) } result = tokio::time::timeout(duration, future) => match result { Ok(result) => result, @@ -8122,7 +8124,7 @@ where biased; _ = token.cancelled() => { cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await; - Err(crate::query::canceled_error()) + Err(crate::execution::canceled_error()) } result = future => result, } @@ -8202,7 +8204,7 @@ async fn execute_postgres_user_query_with_mode( ) .await; - if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) { + if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::execution::QUERY_CANCELED) { cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await; } @@ -8211,9 +8213,9 @@ async fn execute_postgres_user_query_with_mode( /// PostgreSQL pool checkout with timeout and cancel token support. /// When the checkout phase is stuck, the cancel token can terminate the wait early. -pub(crate) type PostgresCheckoutStage = super::PoolCheckoutStage; +pub type PostgresCheckoutStage = super::PoolCheckoutStage; -pub(crate) fn classify_postgres_checkout_stage(status: deadpool_postgres::Status) -> PostgresCheckoutStage { +pub fn classify_postgres_checkout_stage(status: deadpool_postgres::Status) -> PostgresCheckoutStage { if status.waiting > 0 || (status.available == 0 && status.size >= status.max_size) { PostgresCheckoutStage::Wait } else if status.available > 0 { @@ -8235,7 +8237,7 @@ fn postgres_pool_error_stage(error: &PoolError) -> PostgresCheckoutStage { } } -pub(crate) async fn checkout_postgres_client_classified( +pub async fn checkout_postgres_client_classified( pool: &Pool, cancel_token: Option<&CancellationToken>, checkout_timeout: Duration, @@ -8558,7 +8560,7 @@ const POSTGRES_COLUMN_ACL_PRIVILEGES_SQL: &str = WHERE n.nspname = $1 AND c.relname = $2 AND c.relkind IN ('r', 'p') \ ORDER BY 5, 1, 2, 3, 4"; -pub(crate) fn postgres_owner_object_type(relkind: &str) -> &str { +pub fn postgres_owner_object_type(relkind: &str) -> &str { match relkind { "r" => "TABLE", "v" => "VIEW", @@ -8576,11 +8578,7 @@ pub(crate) fn postgres_owner_object_type(relkind: &str) -> &str { /// give a specific diagnostic ("it's a view, not a table") instead of a bare /// "not found" when a `(schema, table)` request turns out not to be a /// table/partition/foreign table. -pub(crate) async fn postgres_relation_relkind( - pool: &Pool, - schema: &str, - table: &str, -) -> Result, String> { +pub async fn postgres_relation_relkind(pool: &Pool, schema: &str, table: &str) -> Result, String> { let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?; let row = client .query_opt( @@ -10269,7 +10267,7 @@ mod tests { assert_eq!(result, Ok(7)); assert_eq!(rollback_calls.get(), 1); - for operation_error in ["query failed", crate::query::QUERY_CANCELED, "Query timed out after 30 seconds"] { + for operation_error in ["query failed", crate::execution::QUERY_CANCELED, "Query timed out after 30 seconds"] { let rollback_calls = Cell::new(0); let result = run_postgres_operation_with_rollback( || async { Err::<(), _>(operation_error.to_string()) }, @@ -12049,7 +12047,7 @@ mod tests { #[test] fn row_limit_falls_back_to_default() { - let default = crate::query::MAX_ROWS; + let default = crate::execution::MAX_ROWS; assert_eq!(query_result_row_limit(None), default); } @@ -12511,7 +12509,7 @@ mod tests { .await .expect_err("cancellation must remain active when query timeout is disabled"); - assert_eq!(error, crate::query::QUERY_CANCELED); + assert_eq!(error, crate::execution::QUERY_CANCELED); } #[test] @@ -13100,176 +13098,6 @@ mod tests { assert!(info.is_primary_key); } - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] - async fn postgres_owned_serial_table_ddl_round_trips_without_external_sequences() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let suffix = uuid::Uuid::new_v4().simple(); - let schema = format!("dbx 6405 \"{suffix}"); - let schema_ident = pg_quote_ident(&schema); - let table_name = "order\"items"; - let table_ident = pg_quote_ident(table_name); - let table = format!("{schema_ident}.{table_ident}"); - let custom_sequence_name = "custom\"source"; - let custom_sequence = format!("{schema_ident}.{}", pg_quote_ident(custom_sequence_name)); - let custom_table_name = "custom defaults"; - let custom_table = format!("{schema_ident}.{}", pg_quote_ident(custom_table_name)); - let owned_custom_sequence_name = "owned custom source"; - let owned_custom_sequence = format!("{schema_ident}.{}", pg_quote_ident(owned_custom_sequence_name)); - let owned_custom_table_name = "owned custom defaults"; - let owned_custom_table = format!("{schema_ident}.{}", pg_quote_ident(owned_custom_table_name)); - let dropped_default_table_name = "dropped default"; - let dropped_default_table = format!("{schema_ident}.{}", pg_quote_ident(dropped_default_table_name)); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE SCHEMA {schema_ident}; \ - CREATE TABLE {table} (\ - \"small\"\"id\" smallserial, \ - \"regular id\" serial, \ - \"large\"\"id\" bigserial, \ - identity_id bigint GENERATED BY DEFAULT AS IDENTITY, \ - base bigint DEFAULT 2, \ - generated bigint GENERATED ALWAYS AS (base * 2) STORED\ - ); \ - CREATE SEQUENCE {custom_sequence}; \ - CREATE TABLE {custom_table} (id bigint DEFAULT nextval({}::regclass)); \ - CREATE TABLE {owned_custom_table} (id bigint NOT NULL); \ - CREATE SEQUENCE {owned_custom_sequence} OWNED BY {owned_custom_table}.id; \ - ALTER TABLE {owned_custom_table} ALTER COLUMN id \ - SET DEFAULT nextval({}::regclass) + 7; \ - CREATE TABLE {dropped_default_table} (id serial); \ - ALTER TABLE {dropped_default_table} ALTER COLUMN id DROP DEFAULT", - pg_quote_literal(&custom_sequence), - pg_quote_literal(&owned_custom_sequence), - )) - .await - .expect("create serial probe objects"); - - let ddl = crate::schema::pg_ddl(&pool, &schema, table_name).await.expect("read serial table ddl"); - let custom_ddl = - crate::schema::pg_ddl(&pool, &schema, custom_table_name).await.expect("read custom-default table ddl"); - let owned_custom_ddl = crate::schema::pg_ddl(&pool, &schema, owned_custom_table_name) - .await - .expect("read owned custom-default table ddl"); - let dropped_default_ddl = crate::schema::pg_ddl(&pool, &schema, dropped_default_table_name) - .await - .expect("read dropped-default table ddl"); - - assert!(ddl.contains("\"small\"\"id\" smallserial NOT NULL"), "ddl: {ddl}"); - assert!(ddl.contains("\"regular id\" serial NOT NULL"), "ddl: {ddl}"); - assert!(ddl.contains("\"large\"\"id\" bigserial NOT NULL"), "ddl: {ddl}"); - assert!(ddl.contains("\"identity_id\" bigint generated by default as identity"), "ddl: {ddl}"); - assert!(ddl.contains("\"generated\" bigint generated always as ((base * 2)) stored"), "ddl: {ddl}"); - assert!(!ddl.contains("nextval("), "ddl: {ddl}"); - assert!(custom_ddl.contains("bigint DEFAULT nextval("), "ddl: {custom_ddl}"); - assert!(!custom_ddl.contains("bigserial"), "ddl: {custom_ddl}"); - assert!(owned_custom_ddl.contains("bigint NOT NULL DEFAULT (nextval("), "ddl: {owned_custom_ddl}"); - assert!(owned_custom_ddl.contains(" + 7)"), "ddl: {owned_custom_ddl}"); - assert!(!owned_custom_ddl.contains("bigserial"), "ddl: {owned_custom_ddl}"); - assert!(dropped_default_ddl.contains("\"id\" integer NOT NULL"), "ddl: {dropped_default_ddl}"); - assert!(!dropped_default_ddl.contains("serial"), "ddl: {dropped_default_ddl}"); - assert!(!dropped_default_ddl.contains("DEFAULT"), "ddl: {dropped_default_ddl}"); - - client.batch_execute(&format!("DROP TABLE {table} CASCADE; {ddl}")).await.expect("replay copied table ddl"); - let inserted = client - .query_one( - &format!( - "INSERT INTO {table} DEFAULT VALUES RETURNING \"small\"\"id\", \"regular id\", \"large\"\"id\", identity_id, generated" - ), - &[], - ) - .await - .expect("insert with replayed sequence defaults"); - assert_eq!(inserted.get::<_, i16>(0), 1); - assert_eq!(inserted.get::<_, i32>(1), 1); - assert_eq!(inserted.get::<_, i64>(2), 1); - assert_eq!(inserted.get::<_, i64>(3), 1); - assert_eq!(inserted.get::<_, i64>(4), 4); - - client.batch_execute(&format!("DROP SCHEMA {schema_ident} CASCADE")).await.expect("drop serial probe schema"); - } - - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] - async fn postgres_column_metadata_dependency_joins_return_each_column_once() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let suffix = uuid::Uuid::new_v4().simple(); - let schema = format!("dbx 6547 \"{suffix}"); - let schema_ident = pg_quote_ident(&schema); - let table_name = "issue6547_repro"; - let table_ident = pg_quote_ident(table_name); - let table = format!("{schema_ident}.{table_ident}"); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE SCHEMA {schema_ident}; \ - CREATE TABLE {table} (\ - id serial PRIMARY KEY, \ - value text, \ - small_id smallserial, \ - big_id bigserial, \ - ident_id bigint GENERATED BY DEFAULT AS IDENTITY, \ - plain_int integer, \ - vc varchar(32), \ - num numeric(10, 2), \ - ts timestamp\ - ); \ - CREATE SEQUENCE {schema_ident}.serial_decoy OWNED BY {table}.id; \ - CREATE SEQUENCE {schema_ident}.identity_decoy OWNED BY {table}.ident_id; \ - CREATE INDEX idx_issue6547_repro_id ON {table}(id); \ - CREATE INDEX idx_issue6547_repro_big ON {table}(big_id); \ - CREATE UNIQUE INDEX uniq_issue6547_repro_vc ON {table}(vc);" - )) - .await - .expect("create repro table"); - - let columns = get_columns(&pool, &schema, table_name).await.expect("get columns"); - // One pg_attribute row must produce exactly one ColumnInfo even when - // the column has multiple sequence dependencies and several indexes. - // Only the sequence used by the default, or the internal identity - // sequence, is semantic metadata for the column. - let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); - for column in &columns { - assert!(seen.insert(column.name.as_str()), "duplicate column {:?} in {columns:?}", column.name); - } - assert_eq!(columns.len(), 9, "expected 9 columns, got {columns:?}"); - - for (name, expected_extra) in [("id", "serial"), ("small_id", "smallserial"), ("big_id", "bigserial")] { - let column = columns.iter().find(|c| c.name == name).unwrap_or_else(|| panic!("missing {name}")); - assert_eq!(column.extra.as_deref(), Some(expected_extra), "{name} extra mismatch"); - } - let ident = columns.iter().find(|c| c.name == "ident_id").expect("ident_id column"); - assert!( - ident.extra.as_deref().is_some_and(|extra| extra.starts_with("generated by default as identity")), - "ident_id extra: {:?}", - ident.extra - ); - // Ordinary columns keep their plain metadata. - let vc = columns.iter().find(|c| c.name == "vc").expect("vc column"); - assert_eq!(vc.data_type, "character varying(32)"); - let num = columns.iter().find(|c| c.name == "num").expect("num column"); - assert_eq!(num.data_type, "numeric(10,2)"); - assert_eq!(num.numeric_precision, Some(10)); - assert_eq!(num.numeric_scale, Some(2)); - let ts = columns.iter().find(|c| c.name == "ts").expect("ts column"); - assert_eq!(ts.data_type, "timestamp without time zone"); - - // The rendered column section of the DDL must contain each column - // definition exactly once. Before the dependency-join fix these - // queries could return "id" (serial + PRIMARY KEY + secondary index - // = several pg_depend rows) any number of times. - let ddl = crate::schema::pg_ddl(&pool, &schema, table_name).await.expect("read repro table ddl"); - for needle in [r#" "id" serial NOT NULL"#, r#" "value" text"#, r#" "vc" character varying(32)"#] { - assert_eq!(ddl.matches(needle).count(), 1, "needle {needle:?} not unique in ddl:\n{ddl}"); - } - - drop(client); - execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")).await.expect("drop repro schema"); - } - #[tokio::test] #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] async fn postgres_column_metadata_compat_query_returns_each_column_once() { @@ -13374,492 +13202,6 @@ mod tests { execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")).await.expect("drop batch repro schema"); } - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL 9.x database"] - async fn postgres_legacy_table_ddl_uses_compatible_partition_probe() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let version = execute_query(&pool, "SHOW server_version_num").await.expect("query PostgreSQL version"); - let version_num = version.rows[0][0] - .as_str() - .expect("server_version_num should be text") - .parse::() - .expect("server_version_num should be numeric"); - assert!((90000..100000).contains(&version_num), "expected PostgreSQL 9.x, got {version_num}"); - - let schema = format!("dbx_legacy_ddl_{}", uuid::Uuid::new_v4().simple()); - let schema_ident = pg_quote_ident(&schema); - let parent = format!("{schema_ident}.parent"); - let child = format!("{schema_ident}.child"); - let function = format!("{schema_ident}.fill_child_name"); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE SCHEMA {schema_ident}; \ - CREATE TABLE {parent} (id integer PRIMARY KEY); \ - CREATE TABLE {child} (parent_id integer REFERENCES {parent}(id), name text) INHERITS ({parent}); \ - ALTER TABLE {child} ADD PRIMARY KEY (id); \ - CREATE INDEX child_name_idx ON {child}(name); \ - CREATE FUNCTION {function}() RETURNS trigger LANGUAGE plpgsql AS $$BEGIN NEW.name := COALESCE(NEW.name, chr(120)); RETURN NEW; END$$; \ - CREATE TRIGGER child_bi BEFORE INSERT ON {child} FOR EACH ROW EXECUTE PROCEDURE {function}()" - )) - .await - .expect("create PostgreSQL 9.x DDL fixtures"); - drop(client); - - let partition_info = - get_table_partition_info(&pool, &schema, "child").await.expect("classify PostgreSQL 9.x inherited table"); - let ddl = crate::schema::pg_ddl(&pool, &schema, "child").await; - // 查看表 DDL（DISPLAY 路径）在 9.x 上不能因 relispartition 报错， - // INHERITS 子表也不能被当成声明式分区渲染。 - // The view-DDL path must not fail on the missing relispartition - // column pre-10, and INHERITS children must render as plain tables. - let ddl_with_partitions = crate::schema::pg_ddl_with_partitions(&pool, &schema, "child").await; - let parent_ddl = crate::schema::pg_ddl_with_partitions(&pool, &schema, "parent").await; - execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")) - .await - .expect("drop PostgreSQL 9.x DDL fixtures"); - let ddl = ddl.expect("generate PostgreSQL 9.x table DDL"); - let ddl_with_partitions = ddl_with_partitions.expect("generate PostgreSQL 9.x partition-tree DDL"); - let parent_ddl = parent_ddl.expect("generate PostgreSQL 9.x partition-tree DDL for parent"); - - assert!(ddl.contains("CREATE TABLE"), "ddl: {ddl}"); - assert!(ddl.contains("PRIMARY KEY"), "ddl: {ddl}"); - assert!(ddl.contains("FOREIGN KEY"), "ddl: {ddl}"); - assert!(ddl.contains("CREATE INDEX \"child_name_idx\""), "ddl: {ddl}"); - assert!(ddl.contains("CREATE TRIGGER child_bi"), "ddl: {ddl}"); - assert_eq!(partition_info, PostgresTablePartitionInfo::default()); - assert!(!ddl.contains("PARTITION OF"), "ddl: {ddl}"); - assert!(ddl_with_partitions.contains("CREATE TABLE"), "ddl_with_partitions: {ddl_with_partitions}"); - assert!(!ddl_with_partitions.contains("PARTITION OF"), "ddl_with_partitions: {ddl_with_partitions}"); - // 9.x 没有声明式分区：父表的分区树只含父表自身（子表按普通表）。 - assert_eq!(parent_ddl.matches("CREATE TABLE").count(), 1, "parent_ddl: {parent_ddl}"); - assert!(!parent_ddl.contains("PARTITION OF"), "parent_ddl: {parent_ddl}"); - } - - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] - async fn postgres_legacy_batched_ddl_metadata_preserves_relation_pairs_across_compat_tiers() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let schema = format!("dbx_batch_fkeys_{}", uuid::Uuid::new_v4().simple()); - let schema_ident = pg_quote_ident(&schema); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE SCHEMA {schema_ident}; \ - CREATE TABLE {schema_ident}.parent_a (id integer PRIMARY KEY); \ - CREATE TABLE {schema_ident}.parent_b (id integer PRIMARY KEY); \ - CREATE TABLE {schema_ident}.child_a ( \ - id integer PRIMARY KEY, parent_id integer, \ - CONSTRAINT child_a_parent_fk FOREIGN KEY (parent_id) REFERENCES {schema_ident}.parent_a(id) \ - ); \ - CREATE TABLE {schema_ident}.child_b ( \ - id integer PRIMARY KEY, parent_id integer, \ - CONSTRAINT child_b_parent_fk FOREIGN KEY (parent_id) REFERENCES {schema_ident}.parent_b(id) \ - ); \ - CREATE TABLE {schema_ident}.without_fk (id integer PRIMARY KEY)" - )) - .await - .expect("create batched foreign-key fixtures"); - - let version_num = client - .query_one("SHOW server_version_num", &[]) - .await - .expect("query PostgreSQL version") - .get::<_, String>(0) - .parse::() - .expect("server_version_num should be numeric"); - let has_conparentid = client - .query_one( - "SELECT EXISTS ( \ - SELECT 1 FROM pg_catalog.pg_attribute \ - WHERE attrelid = 'pg_catalog.pg_constraint'::pg_catalog.regclass \ - AND attname = 'conparentid' AND NOT attisdropped \ - )", - &[], - ) - .await - .expect("probe pg_constraint.conparentid") - .get::<_, bool>(0); - if version_num < 110000 { - assert!(!has_conparentid, "pre-11 PostgreSQL must not have parent-linked constraint clones"); - } - - let empty = list_foreign_keys_for_relations(&pool, &[]).await.expect("empty relation batch"); - assert!(empty.is_empty()); - - let single_relations = vec![(schema.clone(), "child_a".to_string())]; - let single = list_foreign_keys_for_relations(&pool, &single_relations).await.expect("single relation batch"); - let single_fk = &single[&(schema.clone(), "child_a".to_string())][0]; - assert_eq!(single_fk.name, "child_a_parent_fk"); - assert_eq!(single_fk.ref_table, "parent_a"); - - // Keep the pairs deliberately out of name order. A cross product or - // independently unnested array would duplicate or misattribute keys. - let relations = vec![ - (schema.clone(), "child_b".to_string()), - (schema.clone(), "without_fk".to_string()), - (schema.clone(), "child_a".to_string()), - ]; - let batched = list_foreign_keys_for_relations(&pool, &relations).await.expect("multi-relation batch"); - assert_eq!(batched.len(), 2, "batched keys: {batched:?}"); - assert!(!batched.contains_key(&(schema.clone(), "without_fk".to_string()))); - assert_eq!(batched[&(schema.clone(), "child_a".to_string())][0].ref_table, "parent_a"); - assert_eq!(batched[&(schema.clone(), "child_b".to_string())][0].ref_table, "parent_b"); - - let schemas: Vec<&str> = relations.iter().map(|(schema, _)| schema.as_str()).collect(); - let tables: Vec<&str> = relations.iter().map(|(_, table)| table.as_str()).collect(); - let compat = list_foreign_keys_for_relations_with_sql( - &client, - postgres_foreign_keys_for_relations_compat_sql(), - &schemas, - &tables, - ) - .await - .expect("subscript-paired compatibility query"); - assert_eq!(compat[&(schema.clone(), "child_a".to_string())][0].ref_table, "parent_a"); - assert_eq!(compat[&(schema.clone(), "child_b".to_string())][0].ref_table, "parent_b"); - - let modern = list_foreign_keys_for_relations_with_sql( - &client, - postgres_foreign_keys_for_relations_sql(), - &schemas, - &tables, - ) - .await; - if version_num >= 90400 { - let modern = modern.expect("multi-array unnest primary query"); - assert_eq!(modern[&(schema.clone(), "child_a".to_string())][0].ref_table, "parent_a"); - assert_eq!(modern[&(schema.clone(), "child_b".to_string())][0].ref_table, "parent_b"); - } else { - modern.expect_err("pre-9.4 server must reject multi-array unnest"); - } - - let relation_oids: Vec = client - .query( - "SELECT c.oid::bigint \ - FROM pg_catalog.pg_class c \ - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ - WHERE n.nspname = $1 ORDER BY c.relname", - &[&schema], - ) - .await - .expect("query fixture relation oids") - .into_iter() - .map(|row| row.get(0)) - .collect(); - get_table_partition_local_objects_for_relations_with_sql( - &client, - postgres_table_partition_local_objects_for_relations_compat_sql(), - &relation_oids, - ) - .await - .expect("pre-parent-clone local-object query"); - let modern_local_objects = get_table_partition_local_objects_for_relations_with_sql( - &client, - postgres_table_partition_local_objects_for_relations_sql(), - &relation_oids, - ) - .await; - if version_num >= 90300 { - modern_local_objects.expect("JSON-extraction local-object query"); - } else { - modern_local_objects.expect_err("PostgreSQL 9.2 must reject the JSON ->> operator"); - } - drop(client); - - let child_ddl = crate::schema::pg_ddl_with_partitions(&pool, &schema, "child_a") - .await - .expect("display DDL with foreign key"); - let without_fk_ddl = crate::schema::pg_ddl_with_partitions(&pool, &schema, "without_fk") - .await - .expect("display DDL without foreign key"); - execute_query(&pool, &format!("DROP SCHEMA {schema_ident} CASCADE")) - .await - .expect("drop batched foreign-key fixtures"); - - assert!(child_ddl.contains("FOREIGN KEY"), "child ddl: {child_ddl}"); - assert!(!without_fk_ddl.contains("FOREIGN KEY"), "without-fk ddl: {without_fk_ddl}"); - } - - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] - async fn postgres_ddl_emits_not_valid_check_constraints_as_separate_replayable_alter() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let pid = std::process::id(); - let schema = format!("dbx_not_valid_check_{pid}"); - let schema_ident = pg_quote_ident(&schema); - let table = format!("{schema_ident}.events"); - - execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema"); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE TABLE {table} (id integer, amount integer); \ - ALTER TABLE {table} ADD CONSTRAINT c_pos CHECK (amount > 0) NOT VALID" - )) - .await - .expect("create table with unvalidated check constraint"); - - let ddl = crate::schema::pg_ddl(&pool, &schema, "events").await.expect("events ddl"); - - // A `NOT VALID` CHECK constraint is illegal inline in a `CREATE - // TABLE` column list — Postgres only accepts it after `ALTER TABLE - // ADD CONSTRAINT`. Replaying the generated DDL onto a fresh schema - // exercises that this is real, executable syntax, not just a string - // match. - let replay_schema = format!("{schema}_replay"); - let replay_schema_ident = pg_quote_ident(&replay_schema); - execute_query(&pool, &format!("CREATE SCHEMA {replay_schema_ident}")).await.expect("create replay schema"); - client - .batch_execute(&ddl.replace(&schema, &replay_schema)) - .await - .unwrap_or_else(|error| panic!("replay not-valid-check ddl failed: {error:?}; ddl: {ddl}")); - - client - .batch_execute(&format!("DROP SCHEMA {schema_ident} CASCADE; DROP SCHEMA {replay_schema_ident} CASCADE")) - .await - .expect("drop schemas"); - - assert!(!ddl.contains("NOT VALID)"), "NOT VALID must not appear inline in the column list: {ddl}"); - assert!(ddl.contains("CREATE TABLE"), "ddl: {ddl}"); - assert!(!ddl.contains("\"amount\" integer,"), "check constraint must not be inlined into columns: {ddl}"); - assert!(ddl.contains("ALTER TABLE"), "ddl: {ddl}"); - assert!( - ddl.contains("ADD CONSTRAINT \"c_pos\" CHECK (amount > 0) NOT VALID;"), - "expected a separate ALTER TABLE ADD CONSTRAINT ... NOT VALID statement: {ddl}" - ); - } - - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] - async fn postgres_partition_metadata_renders_replayable_children_and_subpartitions() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let pid = std::process::id(); - let schema = format!("dbx_partition_meta_{pid}"); - let schema_ident = pg_quote_ident(&schema); - let replay_schema = format!("{schema}_replay"); - let replay_schema_ident = pg_quote_ident(&replay_schema); - let parent = format!("{schema_ident}.parent"); - let child = format!("{schema_ident}.child"); - let default_child = format!("{schema_ident}.child_default"); - let subpartition = format!("{schema_ident}.subpartition"); - let inherited_parent = format!("{schema_ident}.inherited_parent"); - let inherited_child = format!("{schema_ident}.inherited_child"); - // Declared in "public" (not the per-test schema) so the same - // qualified name keeps resolving after `schema_ident` substrings are - // rewritten to `replay_schema_ident` for the replay pass below. - let trigger_fn = format!("public.dbx_partition_meta_trigger_fn_{pid}"); - - execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema"); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE OR REPLACE FUNCTION {trigger_fn}() RETURNS trigger LANGUAGE plpgsql AS \ - $$ BEGIN RETURN NEW; END $$; \ - CREATE TABLE {parent} (id integer, payload text, status text DEFAULT 'pending') PARTITION BY RANGE (id); \ - CREATE TABLE {child} PARTITION OF {parent} ( \ - PRIMARY KEY (id), \ - CONSTRAINT child_payload_not_blank CHECK (payload <> ''), \ - status WITH OPTIONS DEFAULT 'child-status' \ - ) FOR VALUES FROM (1) TO (10); \ - CREATE INDEX child_payload_idx ON {child} (payload); \ - CREATE TABLE {default_child} PARTITION OF {parent} DEFAULT; \ - CREATE TABLE {subpartition} PARTITION OF {parent} FOR VALUES FROM (10) TO (20) PARTITION BY HASH (payload); \ - ALTER TABLE {parent} ADD CONSTRAINT parent_id_positive CHECK (id > 0); \ - CREATE TRIGGER parent_biu BEFORE INSERT ON {parent} FOR EACH ROW EXECUTE FUNCTION {trigger_fn}(); \ - CREATE TABLE {inherited_parent} (id integer, bucket integer, PRIMARY KEY (id, bucket)) PARTITION BY RANGE (bucket); \ - CREATE TABLE {inherited_child} PARTITION OF {inherited_parent} FOR VALUES FROM (1) TO (10)" - )) - .await - .expect("create partitioned tables"); - - let parent_info = get_table_partition_info(&pool, &schema, "parent").await.expect("parent metadata"); - let child_info = get_table_partition_info(&pool, &schema, "child").await.expect("child metadata"); - let default_info = get_table_partition_info(&pool, &schema, "child_default").await.expect("default metadata"); - let subpartition_info = - get_table_partition_info(&pool, &schema, "subpartition").await.expect("subpartition metadata"); - let child_local_objects = - get_table_partition_local_objects(&pool, &schema, "child").await.expect("child local objects"); - let inherited_child_local_objects = get_table_partition_local_objects(&pool, &schema, "inherited_child") - .await - .expect("inherited child local objects"); - let parent_ddl = crate::schema::pg_ddl_with_partitions(&pool, &schema, "parent").await.expect("parent ddl"); - let child_ddl = crate::schema::pg_ddl(&pool, &schema, "child").await.expect("child ddl"); - let default_ddl = crate::schema::pg_ddl(&pool, &schema, "child_default").await.expect("default ddl"); - let subpartition_ddl = crate::schema::pg_ddl(&pool, &schema, "subpartition").await.expect("subpartition ddl"); - let inherited_parent_ddl = crate::schema::pg_ddl_with_partitions(&pool, &schema, "inherited_parent") - .await - .expect("inherited parent ddl"); - let inherited_child_ddl = - crate::schema::pg_ddl(&pool, &schema, "inherited_child").await.expect("inherited child ddl"); - // The bare (non-recursive) parent DDL, exactly what database export - // and table transfer produce per-relation — used below to give the - // single-relation partition ddls (child_ddl, default_ddl, - // subpartition_ddl, inherited_child_ddl) a real parent shell to - // replay against, so their syntax is actually exercised by - // PostgreSQL rather than only checked with string assertions. - let bare_parent_ddl = crate::schema::pg_ddl(&pool, &schema, "parent").await.expect("bare parent ddl"); - let bare_inherited_parent_ddl = - crate::schema::pg_ddl(&pool, &schema, "inherited_parent").await.expect("bare inherited parent ddl"); - - // `parent_ddl`/`inherited_parent_ddl` already embed their full - // partition subtree (that's the fix under test), so replaying the - // per-partition ddl strings on top would recreate the same relations - // twice. Only the two top-level requests need replaying; the - // individually-fetched partition ddls above are replayed separately - // below, on top of a bare (childless) copy of their parent. - execute_query(&pool, &format!("CREATE SCHEMA {replay_schema_ident}")).await.expect("create replay schema"); - for ddl in [&parent_ddl, &inherited_parent_ddl] { - client - .batch_execute(&ddl.replace(&schema, &replay_schema)) - .await - .unwrap_or_else(|error| panic!("replay partition ddl failed: {error:?}; ddl: {ddl}")); - } - - // Single-relation partition ddls are now a real production code - // path (database export/table transfer fetch each partition this - // way), so replay them for real too: a fresh schema gets just the - // bare parents, then each single-relation partition ddl on top. - let bare_schema = format!("{schema}_bare"); - let bare_schema_ident = pg_quote_ident(&bare_schema); - execute_query(&pool, &format!("CREATE SCHEMA {bare_schema_ident}")).await.expect("create bare schema"); - for ddl in [&bare_parent_ddl, &bare_inherited_parent_ddl] { - client - .batch_execute(&ddl.replace(&schema, &bare_schema)) - .await - .unwrap_or_else(|error| panic!("replay bare parent ddl failed: {error:?}; ddl: {ddl}")); - } - for ddl in [&child_ddl, &default_ddl, &subpartition_ddl, &inherited_child_ddl] { - client - .batch_execute(&ddl.replace(&schema, &bare_schema)) - .await - .unwrap_or_else(|error| panic!("replay single-relation partition ddl failed: {error:?}; ddl: {ddl}")); - } - - client - .batch_execute(&format!( - "DROP SCHEMA {schema_ident} CASCADE; DROP SCHEMA {replay_schema_ident} CASCADE; \ - DROP SCHEMA {bare_schema_ident} CASCADE; DROP FUNCTION IF EXISTS {trigger_fn}()" - )) - .await - .expect("drop schemas and trigger function"); - - assert_eq!(parent_info.key.as_deref(), Some("RANGE (id)")); - assert!(!parent_info.is_partition); - assert_eq!(child_info.parent_schema.as_deref(), Some(schema.as_str())); - assert_eq!(child_info.parent_table.as_deref(), Some("parent")); - assert_eq!(child_info.bound.as_deref(), Some("FOR VALUES FROM (1) TO (10)")); - assert!(child_local_objects.has_primary_key); - assert!(child_local_objects.indexes.contains("child_payload_idx")); - assert!(child_local_objects.check_constraints.contains("child_payload_not_blank")); - assert!(!child_local_objects.check_constraints.contains("parent_id_positive")); - assert_eq!(child_local_objects.column_defaults.get("status"), Some(&PostgresColumnDefaultState::Overridden)); - assert!(!inherited_child_local_objects.has_primary_key); - assert!(inherited_child_local_objects.indexes.is_empty()); - assert_eq!(default_info.bound.as_deref(), Some("DEFAULT")); - assert_eq!(subpartition_info.key.as_deref(), Some("HASH (payload)")); - assert!(parent_ddl.contains(") PARTITION BY RANGE (id);"), "ddl: {parent_ddl}"); - assert!(parent_ddl.contains("CONSTRAINT \"parent_id_positive\" CHECK (id > 0)"), "ddl: {parent_ddl}"); - // The parent's trigger is automatically cloned onto every partition - // by PostgreSQL itself when the partition is created/attached, so it - // must appear exactly once in the combined DDL, not once per relation. - assert_eq!(parent_ddl.matches("CREATE TRIGGER").count(), 1, "ddl: {parent_ddl}"); - assert!(child_ddl.contains("CREATE TABLE"), "ddl: {child_ddl}"); - assert!(child_ddl.contains("PARTITION OF"), "ddl: {child_ddl}"); - assert!(child_ddl.contains("PRIMARY KEY (\"id\")"), "ddl: {child_ddl}"); - assert!(child_ddl.contains("FOR VALUES FROM (1) TO (10);"), "ddl: {child_ddl}"); - assert!(child_ddl.contains("CREATE INDEX \"child_payload_idx\""), "ddl: {child_ddl}"); - assert!(child_ddl.contains("CONSTRAINT \"child_payload_not_blank\" CHECK (payload"), "ddl: {child_ddl}"); - assert!(!child_ddl.contains("parent_id_positive"), "ddl: {child_ddl}"); - assert!(child_ddl.contains("\"status\" WITH OPTIONS DEFAULT 'child-status'"), "ddl: {child_ddl}"); - assert!( - !child_ddl.contains("CREATE TRIGGER"), - "child must not redeclare the parent's cloned trigger: {child_ddl}" - ); - assert!(default_ddl.contains(" DEFAULT;"), "ddl: {default_ddl}"); - assert!( - subpartition_ddl.contains("FOR VALUES FROM (10) TO (20) PARTITION BY HASH (payload);"), - "ddl: {subpartition_ddl}" - ); - assert!(!inherited_child_ddl.contains("PRIMARY KEY"), "ddl: {inherited_child_ddl}"); - assert!(!inherited_child_ddl.contains("CREATE INDEX"), "ddl: {inherited_child_ddl}"); - } - - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database with postgres_fdw available"] - async fn postgres_partition_ddl_uses_foreign_table_syntax_for_foreign_partitions() { - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = connect(&url, Duration::from_secs(5)).await.expect("connect postgres"); - let pid = std::process::id(); - let schema = format!("dbx_partition_fdw_{pid}"); - let schema_ident = pg_quote_ident(&schema); - let server = format!("dbx_partition_fdw_server_{pid}"); - let server_ident = pg_quote_ident(&server); - let replay_schema = format!("{schema}_replay"); - let replay_schema_ident = pg_quote_ident(&replay_schema); - let parent = format!("{schema_ident}.parent"); - let local_child = format!("{schema_ident}.local_child"); - let foreign_child = format!("{schema_ident}.foreign_child"); - - execute_query(&pool, &format!("CREATE SCHEMA {schema_ident}")).await.expect("create schema"); - let client = pool.get().await.expect("get postgres client"); - client - .batch_execute(&format!( - "CREATE EXTENSION IF NOT EXISTS postgres_fdw; \ - CREATE SERVER {server_ident} FOREIGN DATA WRAPPER postgres_fdw \ - OPTIONS (host 'localhost', dbname 'postgres', port '5432'); \ - CREATE USER MAPPING FOR CURRENT_USER SERVER {server_ident} \ - OPTIONS (user 'postgres', password 'postgres'); \ - CREATE TABLE {parent} (id integer, payload text) PARTITION BY RANGE (id); \ - CREATE TABLE {local_child} PARTITION OF {parent} FOR VALUES FROM (1) TO (10); \ - CREATE FOREIGN TABLE {foreign_child} PARTITION OF {parent} \ - FOR VALUES FROM (10) TO (20) \ - SERVER {server_ident} OPTIONS (schema_name 'public', table_name 'foreign_child_data')" - )) - .await - .expect("create partitioned tables with a foreign partition"); - - let foreign_info = get_table_partition_info(&pool, &schema, "foreign_child").await.expect("foreign child info"); - let local_info = get_table_partition_info(&pool, &schema, "local_child").await.expect("local child info"); - let parent_ddl = crate::schema::pg_ddl_with_partitions(&pool, &schema, "parent").await.expect("parent ddl"); - let foreign_ddl = crate::schema::pg_ddl(&pool, &schema, "foreign_child").await.expect("foreign child ddl"); - - execute_query(&pool, &format!("CREATE SCHEMA {replay_schema_ident}")).await.expect("create replay schema"); - client - .batch_execute(&parent_ddl.replace(&schema, &replay_schema)) - .await - .unwrap_or_else(|error| panic!("replay foreign partition ddl failed: {error:?}; ddl: {parent_ddl}")); - - client - .batch_execute(&format!( - "DROP SCHEMA {schema_ident} CASCADE; DROP SCHEMA {replay_schema_ident} CASCADE; \ - DROP SERVER {server_ident} CASCADE" - )) - .await - .expect("drop schemas and foreign server"); - - assert!(!local_info.is_foreign); - assert!(foreign_info.is_foreign); - assert_eq!(foreign_info.foreign_server.as_deref(), Some(server.as_str())); - assert!( - foreign_info.foreign_options.contains(&("schema_name".to_string(), "public".to_string())), - "options: {:?}", - foreign_info.foreign_options - ); - assert!(parent_ddl.contains("CREATE FOREIGN TABLE"), "ddl: {parent_ddl}"); - assert!(foreign_ddl.starts_with("CREATE FOREIGN TABLE"), "ddl: {foreign_ddl}"); - assert!(foreign_ddl.contains("PARTITION OF"), "ddl: {foreign_ddl}"); - assert!(foreign_ddl.contains(&format!("SERVER {}", pg_quote_ident(&server))), "ddl: {foreign_ddl}"); - assert!(foreign_ddl.contains("\"schema_name\" 'public'"), "ddl: {foreign_ddl}"); - assert!(foreign_ddl.contains("\"table_name\" 'foreign_child_data'"), "ddl: {foreign_ddl}"); - assert!(!foreign_ddl.contains("PRIMARY KEY"), "ddl: {foreign_ddl}"); - assert!(!foreign_ddl.contains("CREATE INDEX"), "ddl: {foreign_ddl}"); - } - #[tokio::test] #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL database"] async fn postgres_schema_context_prioritizes_selected_schema_and_cleans_up() { diff --git a/crates/dbx-core/src/db/proxy_tunnel.rs b/crates/dbx-drivers/src/db/proxy_tunnel.rs similarity index 99% rename from crates/dbx-core/src/db/proxy_tunnel.rs rename to crates/dbx-drivers/src/db/proxy_tunnel.rs index cc4e27019f..9a7991a425 100644 --- a/crates/dbx-core/src/db/proxy_tunnel.rs +++ b/crates/dbx-drivers/src/db/proxy_tunnel.rs @@ -137,7 +137,7 @@ async fn connect_via_proxy(proxy: &ProxyEndpoint, remote: &RemoteEndpoint) -> Re } #[cfg(feature = "mq-admin")] -pub(crate) async fn connect_via_socks5_proxy( +pub async fn connect_via_socks5_proxy( proxy_host: &str, proxy_port: u16, proxy_username: &str, diff --git a/crates/dbx-core/src/db/questdb.rs b/crates/dbx-drivers/src/db/questdb.rs similarity index 100% rename from crates/dbx-core/src/db/questdb.rs rename to crates/dbx-drivers/src/db/questdb.rs diff --git a/crates/dbx-core/src/db/redis_driver.rs b/crates/dbx-drivers/src/db/redis_driver.rs similarity index 99% rename from crates/dbx-core/src/db/redis_driver.rs rename to crates/dbx-drivers/src/db/redis_driver.rs index 5a3d3460f9..ee8c383ca3 100644 --- a/crates/dbx-core/src/db/redis_driver.rs +++ b/crates/dbx-drivers/src/db/redis_driver.rs @@ -347,8 +347,8 @@ pub enum RedisConnection { Cluster(RedisClusterPool), } -#[cfg(test)] -pub(crate) fn redis_connection_test_stub() -> RedisConnection { +#[cfg(any(test, feature = "test-support"))] +pub fn redis_connection_test_stub() -> RedisConnection { RedisConnection::Cluster(RedisClusterPool { connection: None, seed_nodes: Vec::new(), diff --git a/crates/dbx-core/src/db/rqlite_driver.rs b/crates/dbx-drivers/src/db/rqlite_driver.rs similarity index 99% rename from crates/dbx-core/src/db/rqlite_driver.rs rename to crates/dbx-drivers/src/db/rqlite_driver.rs index 56119f1ace..7538eade78 100644 --- a/crates/dbx-core/src/db/rqlite_driver.rs +++ b/crates/dbx-drivers/src/db/rqlite_driver.rs @@ -353,7 +353,7 @@ fn query_result_from_rqlite_result( execution_time_ms: u128, max_rows: Option, ) -> QueryResult { - let row_limit = max_rows.unwrap_or(crate::query::MAX_ROWS).max(1); + let row_limit = max_rows.unwrap_or(crate::execution::MAX_ROWS).max(1); let truncated = result.values.len() > row_limit; if truncated { result.values.truncate(row_limit); diff --git a/crates/dbx-core/src/db/sqlite.rs b/crates/dbx-drivers/src/db/sqlite.rs similarity index 99% rename from crates/dbx-core/src/db/sqlite.rs rename to crates/dbx-drivers/src/db/sqlite.rs index 38fadcd0a9..adf409bd38 100644 --- a/crates/dbx-core/src/db/sqlite.rs +++ b/crates/dbx-drivers/src/db/sqlite.rs @@ -685,7 +685,7 @@ mod tests { execute_query(&pool, &format!("INSERT INTO progress_probe (id) VALUES ({id})")).await.expect("insert row"); } - let progress_clock = std::sync::Arc::new(crate::query::StreamProgressClock::new()); + let progress_clock = std::sync::Arc::new(crate::execution::StreamProgressClock::new()); assert!(!progress_clock.marked(), "a fresh clock must not report progress"); let result = execute_query_with_max_rows_progress( @@ -1433,7 +1433,7 @@ mod tests { .expect("list attached triggers") .iter() .any(|trigger| trigger.name == "child_touch")); - assert!(crate::schema::sqlite_ddl(&pool, "analytics.db", "child") + assert!(crate::metadata::sqlite_ddl(&pool, "analytics.db", "child") .await .expect("read attached table DDL") .contains("CREATE TABLE child")); @@ -1863,15 +1863,15 @@ fn sqlite_completion_like_pattern(request: &CompletionAssistantRequest) -> Strin } } -pub(crate) fn sqlite_quote_ident(value: &str) -> String { +pub fn sqlite_quote_ident(value: &str) -> String { format!("\"{}\"", value.replace('"', "\"\"")) } -pub(crate) fn sqlite_quote_schema_ident(value: &str) -> String { +pub fn sqlite_quote_schema_ident(value: &str) -> String { sqlite_quote_ident(sqlite_schema_name(value)) } -pub(crate) fn sqlite_quote_schema_ident_for_connection(conn: &Connection, value: &str) -> Result { +pub fn sqlite_quote_schema_ident_for_connection(conn: &Connection, value: &str) -> Result { Ok(sqlite_quote_ident(&sqlite_schema_name_for_connection(conn, value)?)) } @@ -2735,7 +2735,7 @@ pub async fn execute_query(pool: &SqliteHandle, sql: &str) -> Result) -> usize { - max_rows.unwrap_or(crate::query::MAX_ROWS).max(1) + max_rows.unwrap_or(crate::execution::MAX_ROWS).max(1) } /// DML with `RETURNING` produces a result set in SQLite and must use the @@ -2868,23 +2868,23 @@ pub async fn execute_query_with_max_rows( /// this in an inactivity budget only times out on a genuine stall, not on a long /// but steady read. The SQLite worker (sidecar) path keeps the plain call: its RPC /// returns the whole result at once and exposes no incremental progress. -pub(crate) async fn execute_query_with_max_rows_progress( +pub async fn execute_query_with_max_rows_progress( pool: &SqliteHandle, sql: &str, max_rows: Option, - progress_clock: std::sync::Arc, + progress_clock: std::sync::Arc, timeout: Option, ) -> Result { let sql = normalize_sqlite_sql(sql); if let Some(worker) = pool.worker() { // The sidecar worker returns the whole result in one RPC and exposes no // incremental progress, so keep the original wall-clock budget. - return crate::query::wait_for_query_opt(None, timeout, worker.query(&sql, max_rows)).await; + return crate::execution::wait_for_query_opt(None, timeout, worker.query(&sql, max_rows)).await; } let pool = pool.clone(); let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs())); let clock_for_query = progress_clock.clone(); - crate::query::await_stream_with_progress_timeout( + crate::execution::await_stream_with_progress_timeout( async move { tokio::task::spawn_blocking(move || execute_query_blocking(&pool, &sql, max_rows, Some(&clock_for_query))) .await @@ -2902,7 +2902,7 @@ fn execute_query_blocking( pool: &SqliteHandle, sql: &str, max_rows: Option, - progress_clock: Option<&crate::query::StreamProgressClock>, + progress_clock: Option<&crate::execution::StreamProgressClock>, ) -> Result { let start = Instant::now(); let row_limit = query_result_row_limit(max_rows); diff --git a/crates/dbx-core/src/db/sqlite_worker.rs b/crates/dbx-drivers/src/db/sqlite_worker.rs similarity index 100% rename from crates/dbx-core/src/db/sqlite_worker.rs rename to crates/dbx-drivers/src/db/sqlite_worker.rs diff --git a/crates/dbx-core/src/db/sqlserver.rs b/crates/dbx-drivers/src/db/sqlserver.rs similarity index 99% rename from crates/dbx-core/src/db/sqlserver.rs rename to crates/dbx-drivers/src/db/sqlserver.rs index 7162648547..f67d5a6ea6 100644 --- a/crates/dbx-core/src/db/sqlserver.rs +++ b/crates/dbx-drivers/src/db/sqlserver.rs @@ -1,4 +1,4 @@ -use crate::query::MAX_ROWS; +use crate::execution::MAX_ROWS; use crate::sql::starts_with_executable_sql_keyword; use crate::types::{ ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, LinkedServerInfo, ObjectStatistics, QueryMessage, QueryResult, @@ -536,7 +536,7 @@ const TIBERIUS_DONE_PROC_TOKEN_EVENT_LINE: u32 = 159; const TIBERIUS_DONE_IN_PROC_TOKEN_EVENT_LINE: u32 = 165; #[derive(Debug, Clone)] -pub(crate) struct SqlServerBatchResult { +pub struct SqlServerBatchResult { pub result: QueryResult, pub server_message: bool, } @@ -712,7 +712,7 @@ async fn collect_first_result_limited( result_offset: usize, sql: &str, query: &SqlServerUnsafeTypeQuery, - progress_clock: Option<&crate::query::StreamProgressClock>, + progress_clock: Option<&crate::execution::StreamProgressClock>, ) -> Result { let row_limit = query_result_row_limit(max_rows); let mut remaining_offset = result_offset; @@ -1851,13 +1851,13 @@ pub async fn stream_first_result_set( loop { if cancel_token.as_ref().is_some_and(|token| token.is_cancelled()) { - return Err(crate::query::canceled_error()); + return Err(crate::execution::canceled_error()); } let item = match cancel_token.as_ref() { Some(token) => { tokio::select! { biased; - _ = token.cancelled() => return Err(crate::query::canceled_error()), + _ = token.cancelled() => return Err(crate::execution::canceled_error()), item = stream.try_next() => item.map_err(|e| e.to_string())?, } } @@ -3176,11 +3176,11 @@ pub async fn execute_query_with_max_rows( /// every streamed item, so a large table that keeps arriving is never cancelled /// merely for exceeding the timeout in total — only a genuine stall is. Statements /// without a result stream expose no incremental progress and keep the plain path. -pub(crate) async fn execute_query_with_max_rows_progress( +pub async fn execute_query_with_max_rows_progress( client: &mut SqlServerClient, sql: &str, max_rows: Option, - progress_clock: std::sync::Arc, + progress_clock: std::sync::Arc, timeout: Option, ) -> Result { let returns_rows = starts_with_executable_sql_keyword(sql, &["SELECT", "EXEC", "WITH", "TABLE"]) @@ -3188,7 +3188,7 @@ pub(crate) async fn execute_query_with_max_rows_progress( if !returns_rows { // DDL/DML expose no incremental progress, so keep the original wall-clock // budget: a hung write must still be bounded by the configured timeout. - return crate::query::wait_for_query_opt( + return crate::execution::wait_for_query_opt( None, timeout, execute_query_with_max_rows_inner(client, sql, max_rows, None), @@ -3197,7 +3197,7 @@ pub(crate) async fn execute_query_with_max_rows_progress( } let timeout_error = format!("Query timed out after {} seconds", timeout.map_or(0, |timeout| timeout.as_secs())); let clock_for_query = progress_clock.clone(); - crate::query::await_stream_with_progress_timeout( + crate::execution::await_stream_with_progress_timeout( execute_query_with_max_rows_inner(client, sql, max_rows, Some(&clock_for_query)), timeout, progress_clock, @@ -3211,7 +3211,7 @@ async fn execute_query_with_max_rows_inner( client: &mut SqlServerClient, sql: &str, max_rows: Option, - progress_clock: Option<&crate::query::StreamProgressClock>, + progress_clock: Option<&crate::execution::StreamProgressClock>, ) -> Result { let start = Instant::now(); let result_offset = crate::query_result_sql::sqlserver_result_offset(sql); @@ -3282,7 +3282,7 @@ pub async fn execute_batch_with_max_rows( .map(|results| results.into_iter().map(|result| result.result).collect()) } -pub(crate) async fn execute_batch_with_max_rows_metadata( +pub async fn execute_batch_with_max_rows_metadata( client: &mut SqlServerClient, sql: &str, max_rows: Option, @@ -3357,7 +3357,7 @@ pub async fn execute_simple_batch_with_max_rows( .map(|results| results.into_iter().map(|result| result.result).collect()) } -pub(crate) async fn execute_simple_batch_with_max_rows_metadata( +pub async fn execute_simple_batch_with_max_rows_metadata( client: &mut SqlServerClient, sql: &str, max_rows: Option, @@ -4370,8 +4370,8 @@ mod tests { #[test] fn sqlserver_dml_output_uses_the_result_set_execution_path() { let source = include_str!("sqlserver.rs"); - let execute_query = source.split("pub async fn execute_query_with_max_rows").nth(1).unwrap(); - let execute_query = execute_query.split("pub async fn execute_batch").next().unwrap(); + let execute_query = source.split("pub async fn execute_query_with_max_rows(").nth(1).unwrap(); + let execute_query = execute_query.split("pub async fn execute_batch(").next().unwrap(); assert!(execute_query.contains("sqlserver_dml_output_returns_rows(sql)")); assert!(execute_query.contains("client.query(query.sql.as_str(), &[])")); @@ -4380,8 +4380,8 @@ mod tests { #[test] fn sqlserver_single_result_entrypoint_keeps_result_returning_batches() { let source = include_str!("sqlserver.rs"); - let execute_query = source.split("pub async fn execute_query_with_max_rows").nth(1).unwrap(); - let execute_query = execute_query.split("pub async fn execute_batch").next().unwrap(); + let execute_query = source.split("pub async fn execute_query_with_max_rows(").nth(1).unwrap(); + let execute_query = execute_query.split("pub async fn execute_batch(").next().unwrap(); assert!(execute_query.contains("!sqlserver_batch_can_use_execute(sql)")); assert!(execute_query.contains("execute_simple_batch_first_result_with_max_rows(client, sql, max_rows)")); @@ -4397,11 +4397,11 @@ mod tests { #[test] fn sqlserver_user_query_paths_do_not_collect_full_results_before_limiting() { let source = include_str!("sqlserver.rs"); - let execute_query = source.split("pub async fn execute_query").nth(1).unwrap(); - let execute_query = execute_query.split("pub async fn execute_batch").next().unwrap(); + let execute_query = source.split("pub async fn execute_query(").nth(1).unwrap(); + let execute_query = execute_query.split("pub async fn execute_batch(").next().unwrap(); assert!(!execute_query.contains("into_first_result")); - let execute_batch = source.split("pub async fn execute_batch").nth(1).unwrap(); + let execute_batch = source.split("pub async fn execute_batch(").nth(1).unwrap(); let execute_batch = execute_batch.split("#[cfg(test)]").next().unwrap(); assert!(!execute_batch.contains("into_results")); } @@ -4409,7 +4409,7 @@ mod tests { #[test] fn sqlserver_explicit_simple_batch_bypasses_result_type_probing() { let source = include_str!("sqlserver.rs"); - let simple_batch = source.split("pub async fn execute_simple_batch_with_max_rows").nth(1).unwrap(); + let simple_batch = source.split("pub async fn execute_simple_batch_with_max_rows(").nth(1).unwrap(); let simple_batch = simple_batch.split("fn strip_dbx_sqlserver_row_number_column").next().unwrap(); assert!(simple_batch.contains("client.simple_query(sql)")); diff --git a/crates/dbx-core/src/db/ssh_host_key.rs b/crates/dbx-drivers/src/db/ssh_host_key.rs similarity index 100% rename from crates/dbx-core/src/db/ssh_host_key.rs rename to crates/dbx-drivers/src/db/ssh_host_key.rs diff --git a/crates/dbx-core/src/db/ssh_tunnel.rs b/crates/dbx-drivers/src/db/ssh_tunnel.rs similarity index 99% rename from crates/dbx-core/src/db/ssh_tunnel.rs rename to crates/dbx-drivers/src/db/ssh_tunnel.rs index 67cb4d40c8..dde34f4482 100644 --- a/crates/dbx-core/src/db/ssh_tunnel.rs +++ b/crates/dbx-drivers/src/db/ssh_tunnel.rs @@ -48,7 +48,7 @@ const KEYBOARD_INTERACTIVE_PROMPT_TIMEOUT: Duration = Duration::from_secs(300); /// SSH client handler. Holds a host-key verifier so that /// [`client::Handler::check_server_key`] can reject untrusted/changed server /// keys *before* any credential is sent. -pub(crate) struct SshClient { +pub struct SshClient { host_key_verifier: Arc, host: String, port: u16, @@ -361,7 +361,7 @@ async fn authenticate_keyboard_interactive( } #[allow(clippy::too_many_arguments)] -pub(crate) async fn connect_and_authenticate( +pub async fn connect_and_authenticate( connect_host: &str, connect_port: u16, host_key_host: &str, @@ -1381,7 +1381,7 @@ impl TunnelManager { } #[allow(clippy::too_many_arguments)] - pub(crate) async fn start_tunnel_on_local_port( + pub async fn start_tunnel_on_local_port( &self, connection_id: &str, connect_host: &str, @@ -1458,7 +1458,7 @@ impl TunnelManager { } #[allow(clippy::too_many_arguments)] - pub(crate) async fn start_socks5_proxy( + pub async fn start_socks5_proxy( &self, connection_id: &str, connect_host: &str, @@ -1869,7 +1869,7 @@ async fn bind_tunnel_listener( Ok((listener, local_port)) } -pub(crate) fn effective_hop_timeout(hop: &SshTunnelConfig) -> u64 { +pub fn effective_hop_timeout(hop: &SshTunnelConfig) -> u64 { if hop.connect_timeout_secs == 0 { crate::models::connection::default_ssh_connect_timeout_secs() } else { diff --git a/crates/dbx-core/src/db/starrocks.rs b/crates/dbx-drivers/src/db/starrocks.rs similarity index 99% rename from crates/dbx-core/src/db/starrocks.rs rename to crates/dbx-drivers/src/db/starrocks.rs index dd104c5e74..23dab966e7 100644 --- a/crates/dbx-core/src/db/starrocks.rs +++ b/crates/dbx-drivers/src/db/starrocks.rs @@ -58,7 +58,7 @@ fn materialized_views_sql(database: &str) -> String { ) } -pub(crate) fn materialized_view_definition_sql(database: &str, name: &str) -> String { +pub fn materialized_view_definition_sql(database: &str, name: &str) -> String { format!( "SELECT MATERIALIZED_VIEW_DEFINITION \ FROM information_schema.materialized_views \ diff --git a/crates/dbx-core/src/db/tidb.rs b/crates/dbx-drivers/src/db/tidb.rs similarity index 100% rename from crates/dbx-core/src/db/tidb.rs rename to crates/dbx-drivers/src/db/tidb.rs diff --git a/crates/dbx-core/src/db/transport_layer_tunnel.rs b/crates/dbx-drivers/src/db/transport_layer_tunnel.rs similarity index 99% rename from crates/dbx-core/src/db/transport_layer_tunnel.rs rename to crates/dbx-drivers/src/db/transport_layer_tunnel.rs index c0ff386288..f06d30f4ea 100644 --- a/crates/dbx-core/src/db/transport_layer_tunnel.rs +++ b/crates/dbx-drivers/src/db/transport_layer_tunnel.rs @@ -66,7 +66,7 @@ pub async fn start_transport_layers( .await } -pub(crate) async fn start_transport_layers_with_final_ssh_local_port( +pub async fn start_transport_layers_with_final_ssh_local_port( connection_id: &str, layers: &[TransportLayerConfig], remote_host: &str, @@ -91,7 +91,7 @@ pub(crate) async fn start_transport_layers_with_final_ssh_local_port( } #[cfg(feature = "mq-admin")] -pub(crate) async fn start_transport_layers_with_final_ssh_socks5( +pub async fn start_transport_layers_with_final_ssh_socks5( connection_id: &str, layers: &[TransportLayerConfig], ssh_tunnels: &TunnelManager, diff --git a/crates/dbx-core/src/db/turso_driver.rs b/crates/dbx-drivers/src/db/turso_driver.rs similarity index 99% rename from crates/dbx-core/src/db/turso_driver.rs rename to crates/dbx-drivers/src/db/turso_driver.rs index 0d7f0d1ae9..f2c229641b 100644 --- a/crates/dbx-core/src/db/turso_driver.rs +++ b/crates/dbx-drivers/src/db/turso_driver.rs @@ -571,7 +571,7 @@ fn query_result_from_turso_result( execution_time_ms: u128, max_rows: Option, ) -> QueryResult { - let row_limit = max_rows.unwrap_or(crate::query::MAX_ROWS).max(1); + let row_limit = max_rows.unwrap_or(crate::execution::MAX_ROWS).max(1); let truncated = result.values.len() > row_limit; if truncated { result.values.truncate(row_limit); diff --git a/crates/dbx-core/src/db/vector_driver.rs b/crates/dbx-drivers/src/db/vector_driver.rs similarity index 99% rename from crates/dbx-core/src/db/vector_driver.rs rename to crates/dbx-drivers/src/db/vector_driver.rs index 66549e8197..724045675c 100644 --- a/crates/dbx-core/src/db/vector_driver.rs +++ b/crates/dbx-drivers/src/db/vector_driver.rs @@ -220,10 +220,7 @@ pub async fn list_collections(client: &VectorClient) -> Result Result, String> { +pub async fn list_collections_with_db(client: &VectorClient, database: &str) -> Result, String> { match client.kind { VectorDbKind::Qdrant => list_qdrant_collections(client).await, VectorDbKind::Milvus => list_milvus_collections(client, database).await, @@ -843,11 +840,11 @@ fn starts_with_http_method(input: &str) -> bool { ["GET ", "POST ", "PUT ", "DELETE "].iter().any(|prefix| input.to_ascii_uppercase().starts_with(prefix)) } -pub(crate) fn path_segment(value: &str) -> String { +pub fn path_segment(value: &str) -> String { utf8_percent_encode(value, PATH_SEGMENT_ENCODE_SET).to_string() } -pub(crate) fn chroma_collections_path(tenant: &str, database: &str) -> String { +pub fn chroma_collections_path(tenant: &str, database: &str) -> String { let tenant = tenant.trim(); let database = database.trim(); format!( @@ -857,12 +854,7 @@ pub(crate) fn chroma_collections_path(tenant: &str, database: &str) -> String { ) } -pub(crate) fn chroma_collection_path( - tenant: &str, - database: &str, - collection: &str, - operation: Option<&str>, -) -> String { +pub fn chroma_collection_path(tenant: &str, database: &str, collection: &str, operation: Option<&str>) -> String { let mut path = format!("{}/{}", chroma_collections_path(tenant, database), path_segment(collection)); if let Some(operation) = operation { path.push('/'); @@ -871,7 +863,7 @@ pub(crate) fn chroma_collection_path( path } -pub(crate) fn query_value(value: &str) -> String { +pub fn query_value(value: &str) -> String { utf8_percent_encode(value, QUERY_VALUE_ENCODE_SET).to_string() } diff --git a/crates/dbx-core/src/db/victoriametrics_driver.rs b/crates/dbx-drivers/src/db/victoriametrics_driver.rs similarity index 100% rename from crates/dbx-core/src/db/victoriametrics_driver.rs rename to crates/dbx-drivers/src/db/victoriametrics_driver.rs diff --git a/crates/dbx-core/src/db/wkb.rs b/crates/dbx-drivers/src/db/wkb.rs similarity index 98% rename from crates/dbx-core/src/db/wkb.rs rename to crates/dbx-drivers/src/db/wkb.rs index 7936d05dc3..e79e5ac2d5 100644 --- a/crates/dbx-core/src/db/wkb.rs +++ b/crates/dbx-drivers/src/db/wkb.rs @@ -38,7 +38,7 @@ enum WkbGeometry { } #[derive(Debug, Clone, PartialEq)] -pub(crate) struct DecodedGeometry { +pub struct DecodedGeometry { pub wkt: String, pub srid: Option, } @@ -301,7 +301,7 @@ fn parse_wkb_geometry(reader: &mut WkbReader<'_>) -> Option<(WkbGeometry, Option Some((geometry, srid)) } -pub(crate) fn decode_wkb_geometry(raw: &[u8]) -> Option { +pub fn decode_wkb_geometry(raw: &[u8]) -> Option { let mut reader = WkbReader::new(raw); let (geometry, srid) = parse_wkb_geometry(&mut reader)?; if reader.pos != raw.len() { @@ -314,7 +314,7 @@ pub(crate) fn decode_wkb_geometry(raw: &[u8]) -> Option { /// /// Supports both standard WKB (MySQL) and Extended WKB (PostgreSQL/PostGIS). /// Returns `None` if the bytes cannot be parsed as valid WKB. -pub(crate) fn wkb_to_wkt(raw: &[u8]) -> Option { +pub fn wkb_to_wkt(raw: &[u8]) -> Option { decode_wkb_geometry(raw).map(|geometry| geometry.wkt) } diff --git a/crates/dbx-drivers/src/execution.rs b/crates/dbx-drivers/src/execution.rs new file mode 100644 index 0000000000..ed453596a3 --- /dev/null +++ b/crates/dbx-drivers/src/execution.rs @@ -0,0 +1,340 @@ +use std::future::Future; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; +use std::time::Duration; +use tokio::time::timeout; +use tokio_util::sync::CancellationToken; + +use crate::db; +use crate::models::connection::ConnectionConfig; + +pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30); +pub const MAX_ROWS: usize = 10000; +pub const QUERY_CANCELED: &str = "Query canceled"; + +/// Unified database operation execution budget. +/// query_timeout = None only means SQL execution has no upper limit; +/// checkout/connect/recycle/cancel/cleanup always have hard upper limits and cannot be disabled. +#[derive(Debug, Clone)] +pub struct DbOperationBudget { + pub checkout_timeout: Duration, + pub connect_timeout: Duration, + pub recycle_timeout: Duration, + pub query_timeout: Option, + pub cancel_timeout: Duration, + pub cleanup_timeout: Duration, +} + +impl DbOperationBudget { + /// Build an execution budget from connection config. + /// checkout/connect/recycle use connect_timeout_secs (clamped to 1s minimum, 300s maximum). + /// query_timeout follows resolve_query_timeout semantics (Some(0) -> None). + /// cancel/cleanup are fixed values and cannot be disabled. + pub fn from_config(connect_timeout_secs: u64, query_timeout_secs: Option) -> Self { + let infra_timeout = Duration::from_secs(connect_timeout_secs.clamp(1, 300)); + Self { + checkout_timeout: infra_timeout, + connect_timeout: infra_timeout, + recycle_timeout: infra_timeout, + query_timeout: resolve_query_timeout(query_timeout_secs), + cancel_timeout: Duration::from_secs(5), + cleanup_timeout: Duration::from_secs(3), + } + } + + pub fn from_connection_config(config: &ConnectionConfig) -> Self { + // effective_query_timeout_secs, not the raw field: it preserves 0 ("no limit") + // and applies the per-database-type floor Cloud Spanner needs for DDL. + Self::from_config(config.effective_connect_timeout_secs(), Some(config.effective_query_timeout_secs())) + } + + /// Use global default values (when no connection config is available). + pub fn with_defaults() -> Self { + let default_infra = db::connection_timeout(); + Self { + checkout_timeout: default_infra, + connect_timeout: default_infra, + recycle_timeout: default_infra, + query_timeout: Some(QUERY_TIMEOUT), + cancel_timeout: Duration::from_secs(5), + cleanup_timeout: Duration::from_secs(3), + } + } +} + +pub struct StreamProgressClock { + started_at: tokio::time::Instant, + last_progress_ms: AtomicU64, + #[cfg(any(test, feature = "test-support"))] + marked: std::sync::atomic::AtomicBool, +} + +impl Default for StreamProgressClock { + fn default() -> Self { + Self::new() + } +} + +impl StreamProgressClock { + pub fn new() -> Self { + Self { + started_at: tokio::time::Instant::now(), + last_progress_ms: AtomicU64::new(0), + #[cfg(any(test, feature = "test-support"))] + marked: std::sync::atomic::AtomicBool::new(false), + } + } + + pub fn mark(&self) { + self.last_progress_ms.store(self.started_at.elapsed().as_millis() as u64, Ordering::Relaxed); + #[cfg(any(test, feature = "test-support"))] + self.marked.store(true, Ordering::Relaxed); + } + + /// Whether any progress has been recorded yet. Test-only: production code + /// only needs the derived inactivity window. A row read within the first + /// millisecond records a zero timestamp, so this cannot be derived from + /// `last_progress_ms`. + #[cfg(any(test, feature = "test-support"))] + pub fn marked(&self) -> bool { + self.marked.load(Ordering::Relaxed) + } + + fn elapsed_since_progress(&self) -> Duration { + let last_progress_ms = self.last_progress_ms.load(Ordering::Relaxed); + let elapsed_ms = self.started_at.elapsed().as_millis() as u64; + Duration::from_millis(elapsed_ms.saturating_sub(last_progress_ms)) + } +} + +pub fn timeout_error() -> String { + timeout_error_for(QUERY_TIMEOUT) +} + +pub fn timeout_error_for(timeout_duration: Duration) -> String { + let seconds = timeout_duration.as_secs().max(1); + format!("Query timed out after {seconds} seconds") +} + +pub fn canceled_error() -> String { + QUERY_CANCELED.to_string() +} + +pub async fn await_stream_with_progress_timeout( + stream_future: F, + timeout: Option, + progress_clock: Arc, + cancel_token: Option<&CancellationToken>, + timeout_message: String, +) -> Result +where + F: Future>, +{ + let Some(timeout) = timeout else { + return match cancel_token { + Some(token) => { + tokio::select! { + biased; + _ = token.cancelled() => Err(canceled_error()), + result = stream_future => result, + } + } + None => stream_future.await, + }; + }; + + tokio::pin!(stream_future); + loop { + // Query timeout is an inactivity budget, not a cap on total stream duration. + let remaining = timeout.saturating_sub(progress_clock.elapsed_since_progress()); + if remaining.is_zero() { + return Err(timeout_message.clone()); + } + let sleep = tokio::time::sleep(remaining); + tokio::pin!(sleep); + + match cancel_token { + Some(token) => { + tokio::select! { + biased; + _ = token.cancelled() => return Err(canceled_error()), + result = &mut stream_future => return result, + _ = &mut sleep => {}, + } + } + None => { + tokio::select! { + biased; + result = &mut stream_future => return result, + _ = &mut sleep => {}, + } + } + } + + if progress_clock.elapsed_since_progress() >= timeout { + return Err(timeout_message); + } + } +} + +pub fn is_canceled(cancel_token: &Option) -> bool { + cancel_token.as_ref().map(|token| token.is_cancelled()).unwrap_or(false) +} + +pub async fn wait_for_query(cancel_token: Option, future: F) -> Result +where + F: Future>, +{ + wait_for_query_with_timeout(cancel_token, QUERY_TIMEOUT, future).await +} + +pub async fn wait_for_query_with_timeout( + cancel_token: Option, + timeout_duration: Duration, + future: F, +) -> Result +where + F: Future>, +{ + wait_for_result_with_timeout(cancel_token, timeout_duration, future).await +} + +pub async fn wait_for_result_with_timeout( + cancel_token: Option, + timeout_duration: Duration, + future: F, +) -> Result +where + F: Future>, +{ + if let Some(token) = cancel_token { + tokio::select! { + biased; + _ = token.cancelled() => Err(canceled_error()), + result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration))?, + } + } else { + timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration))? + } +} + +/// Like `wait_for_query_with_timeout` but with an optional timeout. +/// `None` means no timeout (only cancellation can stop the query). +pub async fn wait_for_query_opt( + cancel_token: Option, + timeout_duration: Option, + future: F, +) -> Result +where + F: Future>, +{ + wait_for_result_opt(cancel_token, timeout_duration, future).await +} + +pub async fn wait_for_result_opt( + cancel_token: Option, + timeout_duration: Option, + future: F, +) -> Result +where + F: Future>, +{ + match timeout_duration { + Some(d) => wait_for_result_with_timeout(cancel_token, d, future).await, + None => match cancel_token { + Some(token) => { + tokio::select! { + biased; + _ = token.cancelled() => Err(canceled_error()), + result = future => result, + } + } + None => future.await, + }, + } +} + +pub async fn wait_for_value_opt( + cancel_token: Option, + timeout_duration: Option, + future: F, +) -> Result +where + F: Future, +{ + match timeout_duration { + Some(timeout_duration) => { + if let Some(token) = cancel_token { + tokio::select! { + biased; + _ = token.cancelled() => Err(canceled_error()), + result = timeout(timeout_duration, future) => result.map_err(|_| timeout_error_for(timeout_duration)), + } + } else { + timeout(timeout_duration, future).await.map_err(|_| timeout_error_for(timeout_duration)) + } + } + None => match cancel_token { + Some(token) => { + tokio::select! { + biased; + _ = token.cancelled() => Err(canceled_error()), + result = future => Ok(result), + } + } + None => Ok(future.await), + }, + } +} + +/// Locks a mutex-guarded shared connection (e.g. SQL Server's single connection +/// per pool key) and reports how long the caller waited for the lock alongside +/// the guard. Callers should fold the returned wait time into any execution-time +/// metric they report, since a driver-level timer that only starts once the lock +/// is held cannot see time spent queued behind another operation on the same +/// connection. +pub async fn lock_shared_client_with_wait<'a, T>( + client: &'a Arc>, + cancel_token: Option, + timeout_duration: Option, +) -> Result<(tokio::sync::MutexGuard<'a, T>, u128), String> { + let started_at = std::time::Instant::now(); + let guard = wait_for_value_opt(cancel_token, timeout_duration, client.lock()).await?; + Ok((guard, started_at.elapsed().as_millis())) +} + +pub fn query_timeout_duration(timeout_secs: Option) -> Option { + match timeout_secs { + Some(0) => None, + Some(n) => Some(Duration::from_secs(n)), + None => Some(QUERY_TIMEOUT), + } +} + +pub fn resolve_query_timeout(timeout_secs: Option) -> Option { + query_timeout_duration(timeout_secs) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn default_progress_clock_tracks_inactivity_from_creation_and_marks() { + let clock = StreamProgressClock::default(); + assert_eq!(clock.elapsed_since_progress(), Duration::ZERO); + assert!(!clock.marked()); + + tokio::time::advance(Duration::from_secs(2)).await; + assert_eq!(clock.elapsed_since_progress(), Duration::from_secs(2)); + + clock.mark(); + assert!(clock.marked()); + assert_eq!(clock.elapsed_since_progress(), Duration::ZERO); + + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(clock.elapsed_since_progress(), Duration::from_secs(1)); + } +} diff --git a/crates/dbx-drivers/src/lib.rs b/crates/dbx-drivers/src/lib.rs new file mode 100644 index 0000000000..f6b4f602c8 --- /dev/null +++ b/crates/dbx-drivers/src/lib.rs @@ -0,0 +1,25 @@ +#![recursion_limit = "256"] + +pub use dbx_sql::query_result_sql; + +pub use dbx_platform::download::DownloadSource; +pub use dbx_platform::{path_utils, process}; +pub use dbx_sql::{mysql_ddl_normalize, mysql_event_sql, sql, sql_dialect, sql_error_position, sqlserver_temporal}; +pub use dbx_types::{database_manifest, models, types}; + +pub mod agent_catalog; +pub mod agent_connection; +pub mod agent_manager; +pub mod agent_offline_export; +pub mod agent_recovery; +pub mod agent_runtime; +pub mod agent_service; +pub mod backend_error; +pub mod database_capabilities; +pub mod db; +pub mod execution; +pub mod metadata; +pub mod mongo_oidc; +pub mod mongo_shell; +pub mod runtime_config; +pub mod ssh_config; diff --git a/crates/dbx-drivers/src/metadata.rs b/crates/dbx-drivers/src/metadata.rs new file mode 100644 index 0000000000..7d31ec9222 --- /dev/null +++ b/crates/dbx-drivers/src/metadata.rs @@ -0,0 +1,16 @@ +use crate::db; + +pub async fn sqlite_ddl(pool: &db::sqlite::SqliteHandle, schema: &str, table: &str) -> Result { + let pool = pool.clone(); + let schema = schema.to_string(); + let table = table.to_string(); + tokio::task::spawn_blocking(move || { + pool.with_connection(|conn| { + let schema = db::sqlite::sqlite_quote_schema_ident_for_connection(conn, &schema)?; + let sql = format!("SELECT sql FROM {}.sqlite_master WHERE type='table' AND name=?1", schema); + conn.query_row(&sql, [table], |row| row.get::<_, String>(0)).map_err(|e| e.to_string()) + }) + }) + .await + .map_err(|e| e.to_string())? +} diff --git a/crates/dbx-core/src/mongo_oidc.rs b/crates/dbx-drivers/src/mongo_oidc.rs similarity index 100% rename from crates/dbx-core/src/mongo_oidc.rs rename to crates/dbx-drivers/src/mongo_oidc.rs diff --git a/crates/dbx-core/src/mongo_shell.rs b/crates/dbx-drivers/src/mongo_shell.rs similarity index 100% rename from crates/dbx-core/src/mongo_shell.rs rename to crates/dbx-drivers/src/mongo_shell.rs diff --git a/crates/dbx-drivers/src/runtime_config.rs b/crates/dbx-drivers/src/runtime_config.rs new file mode 100644 index 0000000000..54aa040b98 --- /dev/null +++ b/crates/dbx-drivers/src/runtime_config.rs @@ -0,0 +1,7 @@ +pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1; +pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16; +pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4; + +pub fn normalize_duckdb_worker_max_processes(value: usize) -> usize { + value.clamp(DUCKDB_WORKER_MAX_PROCESSES_MIN, DUCKDB_WORKER_MAX_PROCESSES_MAX) +} diff --git a/crates/dbx-core/src/ssh_config.rs b/crates/dbx-drivers/src/ssh_config.rs similarity index 100% rename from crates/dbx-core/src/ssh_config.rs rename to crates/dbx-drivers/src/ssh_config.rs diff --git a/crates/dbx-formats/Cargo.toml b/crates/dbx-formats/Cargo.toml new file mode 100644 index 0000000000..9cb9445cc0 --- /dev/null +++ b/crates/dbx-formats/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dbx-formats" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +chrono = { version = "0.4", features = ["serde"] } +zip = { version = "4", default-features = false, features = ["deflate"] } + +[dev-dependencies] +uuid = { version = "1", features = ["v4"] } +calamine = { version = "0.30.1", features = ["dates"] } +tempfile = "3" diff --git a/crates/dbx-core/src/csv_export.rs b/crates/dbx-formats/src/csv_export.rs similarity index 76% rename from crates/dbx-core/src/csv_export.rs rename to crates/dbx-formats/src/csv_export.rs index 36e06658c8..f21f52db2c 100644 --- a/crates/dbx-core/src/csv_export.rs +++ b/crates/dbx-formats/src/csv_export.rs @@ -1,15 +1,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::fmt; -use std::fs::File; -use std::io::{BufWriter, Write}; - -use crate::connection::AppState; -use crate::models::connection::DatabaseType; -use crate::query::{execute_sql_statement_with_options, QueryExecutionOptions}; -use crate::sql_dialect::{build_table_data_select_sql, TableDataSelectSqlOptions}; - -const TABLE_DATA_EXPORT_PAGE_SIZE: usize = 10_000; +use std::io::Write; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -19,25 +11,6 @@ pub enum CsvQuoteMode { Necessary, } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TableCsvExportOptions { - pub file_path: String, - pub connection_id: String, - pub database: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub schema: Option, - pub table_name: String, - #[serde(default)] - pub columns: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub page_size: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_secs: Option, - #[serde(default)] - pub csv_quote_mode: CsvQuoteMode, -} - /// CSV 转义直写目标 buffer：包引号 + 内部 `"` 翻倍。值不含 `"` 时整段拷贝， /// 不做 replace 分配（逐批流式导出对每个单元格调用，是导出热路径）。 fn push_csv_escaped_content(out: &mut String, value: &str) { @@ -50,7 +23,7 @@ fn push_csv_escaped_content(out: &mut String, value: &str) { out.push_str(rest); } -pub(crate) fn push_csv_escaped(out: &mut String, value: &str) { +pub fn push_csv_escaped(out: &mut String, value: &str) { out.push('"'); push_csv_escaped_content(out, value); out.push('"'); @@ -60,7 +33,7 @@ fn csv_field_needs_quotes(value: &str) -> bool { value.bytes().any(|byte| matches!(byte, b',' | b'"' | b'\n' | b'\r')) } -pub(crate) fn push_csv_field(out: &mut String, value: &str, quote_mode: CsvQuoteMode) { +pub fn push_csv_field(out: &mut String, value: &str, quote_mode: CsvQuoteMode) { if quote_mode == CsvQuoteMode::All || csv_field_needs_quotes(value) { push_csv_escaped(out, value); } else { @@ -78,7 +51,7 @@ impl fmt::Write for CsvEscapedWriter<'_> { } /// 将表导出 CSV 值直接写入已有 buffer；包括 NULL 在内的值均保留分页导出的带引号旧语义。 -pub(crate) fn push_csv_text_value(out: &mut String, value: &Value) { +pub fn push_csv_text_value(out: &mut String, value: &Value) { out.push('"'); match value { Value::Null => {} @@ -142,7 +115,7 @@ fn push_tsv_value(out: &mut String, value: &Value) { } } -pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) { +pub fn push_tsv_row(out: &mut String, row: &[Value]) { for (cell_index, cell) in row.iter().enumerate() { if cell_index > 0 { out.push('\t'); @@ -155,13 +128,13 @@ pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) { /// 并设上限——估算只是性能提示，绝不能因病态输入放大成巨额分配 const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024; -pub(crate) fn estimated_rows_capacity(rows: &[Vec]) -> usize { +pub fn estimated_rows_capacity(rows: &[Vec]) -> usize { let cells: usize = rows.iter().map(Vec::len).fold(0usize, usize::saturating_add); cells.saturating_mul(12).min(ROWS_CAPACITY_ESTIMATE_MAX) } #[cfg_attr(not(test), allow(dead_code))] -pub(crate) fn escape_csv(value: &str) -> String { +pub fn escape_csv(value: &str) -> String { let mut out = String::with_capacity(value.len() + 2); push_csv_escaped(&mut out, value); out @@ -180,7 +153,7 @@ fn format_csv_value(value: &Value) -> String { } #[cfg_attr(not(test), allow(dead_code))] -pub(crate) fn escape_tsv(value: &str) -> String { +pub fn escape_tsv(value: &str) -> String { let mut out = String::with_capacity(value.len()); push_tsv_escaped(&mut out, value); out @@ -196,13 +169,13 @@ fn push_tsv_rows(out: &mut String, rows: &[Vec]) { } #[cfg_attr(not(test), allow(dead_code))] -pub(crate) fn format_tsv_rows(rows: &[Vec]) -> String { +pub fn format_tsv_rows(rows: &[Vec]) -> String { let mut out = String::with_capacity(estimated_rows_capacity(rows)); push_tsv_rows(&mut out, rows); out } -pub(crate) fn format_tsv(columns: &[String], rows: &[Vec]) -> String { +pub fn format_tsv(columns: &[String], rows: &[Vec]) -> String { let mut out = String::with_capacity( estimated_rows_capacity(rows).saturating_add(columns.len().saturating_mul(12)).min(ROWS_CAPACITY_ESTIMATE_MAX), ); @@ -220,11 +193,11 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec]) -> String { /// Format query-result rows as CSV text without a header row. Database NULLs /// use the same empty-cell representation as table-data exports. Used by the /// streaming query-result export for batches after the first. -pub(crate) fn push_query_result_csv_row(out: &mut String, row: &[Value]) { +pub fn push_query_result_csv_row(out: &mut String, row: &[Value]) { push_query_result_csv_row_with_quote_mode(out, row, CsvQuoteMode::All); } -pub(crate) fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) { +pub fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) { for (cell_index, cell) in row.iter().enumerate() { if cell_index > 0 { out.push(','); @@ -233,11 +206,11 @@ pub(crate) fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: & } } -pub(crate) fn push_table_csv_row(out: &mut String, row: &[Value]) { +pub fn push_table_csv_row(out: &mut String, row: &[Value]) { push_table_csv_row_with_quote_mode(out, row, CsvQuoteMode::All); } -pub(crate) fn push_table_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) { +pub fn push_table_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) { for (cell_index, cell) in row.iter().enumerate() { if cell_index > 0 { out.push(','); @@ -312,7 +285,7 @@ pub fn format_query_result_csv_with_quote_mode( format_csv_with_quote_mode(columns, rows, quote_mode) } -fn write_csv_text_row( +pub fn write_csv_text_row( writer: &mut impl Write, values: impl IntoIterator, quote_mode: CsvQuoteMode, @@ -328,7 +301,7 @@ fn write_csv_text_row( Ok(()) } -fn write_csv_value_row( +pub fn write_csv_value_row( writer: &mut impl Write, values: impl IntoIterator, quote_mode: CsvQuoteMode, @@ -351,86 +324,6 @@ fn write_csv_value_row( Ok(()) } -async fn connection_database_type(state: &AppState, connection_id: &str) -> Result { - state - .configs - .read() - .await - .get(connection_id) - .map(|config| config.db_type) - .ok_or_else(|| format!("Connection config not found: {connection_id}")) -} - -pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExportOptions) -> Result { - let database_type = connection_database_type(state, &options.connection_id).await?; - let page_size = options.page_size.unwrap_or(TABLE_DATA_EXPORT_PAGE_SIZE).max(1); - let mut writer = - BufWriter::new(File::create(&options.file_path).map_err(|err| format!("Failed to write CSV file: {err}"))?); - writer.write_all("\u{FEFF}".as_bytes()).map_err(|err| err.to_string())?; - - let mut offset = 0usize; - let mut rows_exported = 0u64; - let mut wrote_header = false; - - loop { - let sql = build_table_data_select_sql(TableDataSelectSqlOptions { - database_type: Some(database_type), - schema: options.schema.clone(), - table_name: options.table_name.clone(), - table_type: None, - primary_keys: Vec::new(), - columns: options.columns.clone(), - fallback_order_columns: Vec::new(), - order_by: None, - limit: Some(page_size), - offset: Some(offset), - where_input: None, - include_row_id: false, - ..Default::default() - }); - let result = execute_sql_statement_with_options( - state, - &options.connection_id, - &options.database, - &sql, - options.schema.as_deref(), - None, - QueryExecutionOptions { - max_rows: Some(page_size), - timeout_secs: options.timeout_secs, - ..Default::default() - }, - ) - .await?; - - if !wrote_header { - write_csv_text_row(&mut writer, result.columns, options.csv_quote_mode)?; - wrote_header = true; - } - - let fetched = result.rows.len(); - if fetched == 0 { - break; - } - for row in result.rows { - writer.write_all(b"\n").map_err(|err| err.to_string())?; - write_csv_value_row(&mut writer, row, options.csv_quote_mode)?; - } - - rows_exported += fetched as u64; - if fetched < page_size { - break; - } - offset += fetched; - } - - if rows_exported == 0 { - writer.write_all(b"\n").map_err(|err| err.to_string())?; - } - writer.flush().map_err(|err| err.to_string())?; - Ok(rows_exported) -} - #[cfg(test)] mod tests { use super::{ diff --git a/crates/dbx-core/src/export_split_zip.rs b/crates/dbx-formats/src/export_split_zip.rs similarity index 100% rename from crates/dbx-core/src/export_split_zip.rs rename to crates/dbx-formats/src/export_split_zip.rs diff --git a/crates/dbx-formats/src/lib.rs b/crates/dbx-formats/src/lib.rs new file mode 100644 index 0000000000..76d6858357 --- /dev/null +++ b/crates/dbx-formats/src/lib.rs @@ -0,0 +1,6 @@ +pub mod csv_export; +pub mod export_split_zip; +pub mod sql_file_zip_package; +pub mod temporal_format; +pub mod text_export; +pub mod xlsx_export; diff --git a/crates/dbx-core/src/sql_file_zip_package.rs b/crates/dbx-formats/src/sql_file_zip_package.rs similarity index 100% rename from crates/dbx-core/src/sql_file_zip_package.rs rename to crates/dbx-formats/src/sql_file_zip_package.rs diff --git a/crates/dbx-core/src/temporal_format.rs b/crates/dbx-formats/src/temporal_format.rs similarity index 99% rename from crates/dbx-core/src/temporal_format.rs rename to crates/dbx-formats/src/temporal_format.rs index e7bafbe928..00d889dd7d 100644 --- a/crates/dbx-core/src/temporal_format.rs +++ b/crates/dbx-formats/src/temporal_format.rs @@ -19,7 +19,7 @@ enum ParsedTemporal { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ExcelTemporalKind { +pub enum ExcelTemporalKind { Date, DateTime, } @@ -155,7 +155,7 @@ fn parse_temporal(value: &str, preferred_pattern: Option<&str>) -> Option, preferred_pattern: Option<&str>, @@ -225,7 +225,7 @@ fn active_temporal_pattern(pattern: Option<&str>) -> Option<&str> { pattern.filter(|pattern| !pattern.trim().is_empty()) } -pub(crate) fn strip_csv_force_text_wrapper(value: &str) -> &str { +pub fn strip_csv_force_text_wrapper(value: &str) -> &str { value.strip_prefix("=\"").and_then(|value| value.strip_suffix('"')).unwrap_or(value) } @@ -333,7 +333,7 @@ pub fn format_temporal_export_rows_with_string_types( format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern).into_owned() } -pub(crate) fn normalize_temporal_import_value_cow<'a>( +pub fn normalize_temporal_import_value_cow<'a>( value: &'a Value, data_type: Option<&str>, pattern: Option<&str>, diff --git a/crates/dbx-core/src/text_export.rs b/crates/dbx-formats/src/text_export.rs similarity index 100% rename from crates/dbx-core/src/text_export.rs rename to crates/dbx-formats/src/text_export.rs diff --git a/crates/dbx-core/src/xlsx_export.rs b/crates/dbx-formats/src/xlsx_export.rs similarity index 99% rename from crates/dbx-core/src/xlsx_export.rs rename to crates/dbx-formats/src/xlsx_export.rs index b7c429603d..2e7e1c837d 100644 --- a/crates/dbx-core/src/xlsx_export.rs +++ b/crates/dbx-formats/src/xlsx_export.rs @@ -5,7 +5,7 @@ use std::io::{Cursor, Seek, Write}; use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind}; -pub(crate) const XLSX_MAX_DATA_ROWS: usize = 1_048_575; +pub const XLSX_MAX_DATA_ROWS: usize = 1_048_575; const XLSX_DATE_STYLE: usize = 2; const XLSX_DATETIME_STYLE: usize = 3; const NUMERIC_RIGHT_ALIGN_STYLE: usize = 4; @@ -194,7 +194,7 @@ fn effective_header<'a>(column: &'a str, comment: Option<&'a str>) -> &'a str { } /// Build a single `` XML fragment for the header row (row 1). -pub(crate) fn header_row_xml(columns: &[String], column_comments: &[Option]) -> String { +pub fn header_row_xml(columns: &[String], column_comments: &[Option]) -> String { format!( "{}", columns @@ -254,7 +254,7 @@ fn write_zip_entry(zip: &mut zip::ZipWriter, path: &str, con /// immediately. Callers then feed data rows via [`StreamingXlsxWriter::write_row`] /// and finalize with [`StreamingXlsxWriter::finish`]. #[cfg(test)] -pub(crate) fn start_streaming_xlsx_workbook( +pub fn start_streaming_xlsx_workbook( writer: W, sheet_name: Option<&str>, columns: &[String], @@ -264,7 +264,7 @@ pub(crate) fn start_streaming_xlsx_workbook( } #[cfg(test)] -pub(crate) fn start_streaming_xlsx_workbook_with_trailing_sheets( +pub fn start_streaming_xlsx_workbook_with_trailing_sheets( writer: W, sheet_name: Option<&str>, columns: &[String], @@ -344,7 +344,7 @@ fn start_xlsx_writer_inner( }) } -pub(crate) fn start_streaming_xlsx_workbook_with_options( +pub fn start_streaming_xlsx_workbook_with_options( writer: W, sheet_name: Option<&str>, columns: &[String], @@ -370,7 +370,7 @@ pub(crate) fn start_streaming_xlsx_workbook_with_options( } #[cfg(test)] -pub(crate) fn start_streaming_xlsx_workbook_with_max_rows( +pub fn start_streaming_xlsx_workbook_with_max_rows( writer: W, sheet_name: Option<&str>, columns: &[String], @@ -505,7 +505,7 @@ impl StreamingXlsxWriter { } /// Convenience wrapper that finalizes a streaming workbook. -pub(crate) fn finish_streaming_xlsx_workbook(writer: StreamingXlsxWriter) -> Result { +pub fn finish_streaming_xlsx_workbook(writer: StreamingXlsxWriter) -> Result { writer.finish() } @@ -1099,7 +1099,7 @@ fn split_sheets_for_max_rows<'a>( max_data_rows_per_sheet: usize, ) -> Vec> { // `slice::chunks` panics on a zero chunk size. No caller passes 0 today - // (production uses XLSX_MAX_DATA_ROWS), but this is pub(crate)-reachable + // (production uses XLSX_MAX_DATA_ROWS), but this is pub-reachable // through build_xlsx_workbook_multi_with_max_rows, so guard defensively. let max_data_rows_per_sheet = max_data_rows_per_sheet.max(1); let mut expanded = Vec::with_capacity(sheets.len()); @@ -1173,7 +1173,7 @@ pub fn build_xlsx_workbook_multi_with_auto_filter( /// Any worksheet whose data rows exceed `max_data_rows_per_sheet` is split into /// multiple worksheets (each with a header row, column widths, frozen pane, /// and autoFilter). -pub(crate) fn build_xlsx_workbook_multi_with_max_rows( +pub fn build_xlsx_workbook_multi_with_max_rows( sheets: &[XlsxWorksheetData], max_data_rows_per_sheet: usize, date_time_format: Option<&str>, diff --git a/crates/dbx-platform/Cargo.toml b/crates/dbx-platform/Cargo.toml new file mode 100644 index 0000000000..1bfc22ea8a --- /dev/null +++ b/crates/dbx-platform/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "dbx-platform" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +host-prompts = ["dep:serde", "dep:uuid", "tokio/sync"] +test-support = ["host-prompts"] +downloads = ["dep:reqwest", "dep:futures", "dep:serde"] + +[dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"], optional = true } +futures = { version = "0.3", optional = true } +tokio = { version = "1", features = ["process"] } +serde = { version = "1.0", features = ["derive"], optional = true } +uuid = { version = "1", features = ["v4"], optional = true } + +[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] +nix = { version = "0.31.3", features = ["user"] } diff --git a/crates/dbx-platform/src/download.rs b/crates/dbx-platform/src/download.rs new file mode 100644 index 0000000000..ad5a7e880d --- /dev/null +++ b/crates/dbx-platform/src/download.rs @@ -0,0 +1,118 @@ +pub const R2_CDN_BASE: &str = "https://dl.dbxio.com/"; +pub const GITHUB_RELEASE_DOWNLOAD_PREFIX: &str = "https://github.com/t8y2/dbx/releases/download/"; +pub const CNB_RELEASE_DOWNLOAD_PREFIX: &str = "https://cnb.cool/dbxio.com/dbx/-/releases/download/"; + +#[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "lowercase")] +pub enum DownloadSource { + #[default] + Official, + Cnb, +} + +impl DownloadSource { + pub fn download_candidate_urls(self, github_url: &str, r2_path: &str) -> Result, String> { + match self { + Self::Official => Ok(download_candidate_urls(github_url, r2_path)), + Self::Cnb => Ok(mirror_download_candidate_urls( + r2_path, + rewrite_github_release_url(github_url, CNB_RELEASE_DOWNLOAD_PREFIX)?, + )), + } + } +} + +fn mirror_download_candidate_urls(r2_path: &str, mirror_url: String) -> Vec { + let r2_url = format!("{R2_CDN_BASE}{r2_path}"); + vec![mirror_url, r2_url] +} + +fn rewrite_github_release_url(url: &str, target_prefix: &str) -> Result { + if url.starts_with(target_prefix) { + return Ok(url.to_string()); + } + url.strip_prefix(GITHUB_RELEASE_DOWNLOAD_PREFIX) + .map(|path| format!("{target_prefix}{path}")) + .ok_or_else(|| format!("Unsupported DBX release download URL: {url}")) +} + +pub fn download_candidate_urls(github_url: &str, r2_path: &str) -> Vec { + vec![format!("{R2_CDN_BASE}{r2_path}"), github_url.to_string()] +} + +use std::pin::Pin; + +type ResponseFuture = Pin> + Send>>; + +pub async fn race_download( + client: &reqwest::Client, + github_url: &str, + r2_path: &str, + user_agent: &str, +) -> Result { + race_download_urls(client, &download_candidate_urls(github_url, r2_path), user_agent).await +} + +pub async fn race_download_urls( + client: &reqwest::Client, + urls: &[String], + user_agent: &str, +) -> Result { + use futures::future::select_ok; + + let mut futs: Vec = Vec::with_capacity(urls.len()); + + for url in urls { + let client = client.clone(); + let url = url.clone(); + let ua = user_agent.to_string(); + futs.push(Box::pin(async move { + client + .get(url) + .header(reqwest::header::USER_AGENT, ua) + .header(reqwest::header::ACCEPT_ENCODING, "identity") + .send() + .await + .and_then(|r| r.error_for_status()) + .map_err(|e| format!("{e}")) + }) as ResponseFuture); + } + + match select_ok(futs).await { + Ok((resp, _)) => Ok(resp), + Err(last_err) => Err(last_err), + } +} + +#[cfg(test)] +mod tests { + use super::{download_candidate_urls, DownloadSource}; + + #[test] + fn download_candidates_exclude_third_party_github_proxy() { + let urls = download_candidate_urls( + "https://github.com/t8y2/dbx/releases/latest/download/latest.json", + "releases/latest/latest.json", + ); + + assert_eq!( + urls, + vec![ + "https://dl.dbxio.com/releases/latest/latest.json", + "https://github.com/t8y2/dbx/releases/latest/download/latest.json", + ] + ); + } + + #[test] + fn mirror_download_candidates_prefer_selected_source() { + let github_url = "https://github.com/t8y2/dbx/releases/download/agents-latest/agent-registry.json"; + assert_eq!( + DownloadSource::Cnb.download_candidate_urls(github_url, "agents/agent-registry.json").unwrap(), + vec![ + "https://cnb.cool/dbxio.com/dbx/-/releases/download/agents-latest/agent-registry.json", + "https://dl.dbxio.com/agents/agent-registry.json", + ] + ); + } +} diff --git a/crates/dbx-platform/src/lib.rs b/crates/dbx-platform/src/lib.rs new file mode 100644 index 0000000000..5cc469a4d6 --- /dev/null +++ b/crates/dbx-platform/src/lib.rs @@ -0,0 +1,12 @@ +pub mod path_utils; +pub mod process; +pub mod proxy; +#[cfg(feature = "host-prompts")] +pub mod ssh_prompt; + +#[cfg(feature = "downloads")] +pub mod download; +pub mod version; + +#[cfg(all(target_os = "windows", target_env = "gnu"))] +mod nanosleep_stub; diff --git a/crates/dbx-core/src/nanosleep_stub.rs b/crates/dbx-platform/src/nanosleep_stub.rs similarity index 100% rename from crates/dbx-core/src/nanosleep_stub.rs rename to crates/dbx-platform/src/nanosleep_stub.rs diff --git a/crates/dbx-core/src/path_utils.rs b/crates/dbx-platform/src/path_utils.rs similarity index 100% rename from crates/dbx-core/src/path_utils.rs rename to crates/dbx-platform/src/path_utils.rs diff --git a/crates/dbx-core/src/process.rs b/crates/dbx-platform/src/process.rs similarity index 100% rename from crates/dbx-core/src/process.rs rename to crates/dbx-platform/src/process.rs diff --git a/crates/dbx-platform/src/proxy.rs b/crates/dbx-platform/src/proxy.rs new file mode 100644 index 0000000000..cc7e5215fd --- /dev/null +++ b/crates/dbx-platform/src/proxy.rs @@ -0,0 +1,177 @@ +pub fn system_proxy_url() -> Option { + system_proxy_url_from_platform() +} + +#[cfg(target_os = "macos")] +fn system_proxy_url_from_platform() -> Option { + let output = crate::process::new_std_command("scutil").arg("--proxy").output().ok()?; + if !output.status.success() { + return None; + } + let stdout = String::from_utf8(output.stdout).ok()?; + system_proxy_url_from_scutil_output(&stdout) +} + +#[cfg(target_os = "windows")] +fn system_proxy_url_from_platform() -> Option { + let key = r"HKCU\Software\Microsoft\Windows\CurrentVersion\Internet Settings"; + let proxy_enable = + crate::process::new_std_command("reg").args(["query", key, "/v", "ProxyEnable"]).output().ok()?; + let proxy_server = + crate::process::new_std_command("reg").args(["query", key, "/v", "ProxyServer"]).output().ok()?; + if !proxy_enable.status.success() || !proxy_server.status.success() { + return None; + } + let proxy_enable = String::from_utf8(proxy_enable.stdout).ok()?; + let proxy_server = String::from_utf8(proxy_server.stdout).ok()?; + system_proxy_url_from_windows_registry_output(&proxy_enable, &proxy_server) +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn system_proxy_url_from_platform() -> Option { + None +} + +#[cfg_attr(not(test), allow(dead_code))] +fn system_proxy_url_from_scutil_output(output: &str) -> Option { + let value = |key: &str| { + output.lines().find_map(|line| { + let (line_key, line_value) = line.split_once(':')?; + (line_key.trim() == key).then(|| line_value.trim()) + }) + }; + + if value("HTTPSEnable") == Some("1") { + if let Some(url) = proxy_url(value("HTTPSProxy")?, value("HTTPSPort")?) { + return Some(url); + } + } + + if value("HTTPEnable") == Some("1") { + if let Some(url) = proxy_url(value("HTTPProxy")?, value("HTTPPort")?) { + return Some(url); + } + } + + None +} + +#[cfg_attr(not(test), allow(dead_code))] +fn system_proxy_url_from_windows_registry_output(proxy_enable: &str, proxy_server: &str) -> Option { + let enabled = proxy_enable + .lines() + .find(|line| line.contains("ProxyEnable"))? + .split_whitespace() + .last() + .is_some_and(|value| value == "0x1" || value == "1"); + if !enabled { + return None; + } + + let server = proxy_server.lines().find(|line| line.contains("ProxyServer"))?.split_whitespace().last()?; + + proxy_url_from_windows_proxy_server(server) +} + +fn proxy_url_from_windows_proxy_server(server: &str) -> Option { + let entries = server.split(';').map(str::trim).filter(|entry| !entry.is_empty()).collect::>(); + + for key in ["https=", "http="] { + if let Some(entry) = entries.iter().find_map(|entry| entry.strip_prefix(key)) { + if let Some(url) = proxy_url_from_host_port(entry) { + return Some(url); + } + } + } + + entries.iter().find(|entry| !entry.contains('=')).and_then(|entry| proxy_url_from_host_port(entry)) +} + +fn proxy_url_from_host_port(value: &str) -> Option { + let value = value.trim(); + if value.starts_with("http://") || value.starts_with("https://") { + return Some(value.to_string()); + } + if value.starts_with("socks://") || value.starts_with("socks5://") || value.starts_with("socks5h://") { + return None; + } + + let (host, port) = if let Some(rest) = value.strip_prefix('[') { + let (host, rest) = rest.split_once(']')?; + let port = rest.strip_prefix(':')?; + (host, port) + } else { + value.rsplit_once(':')? + }; + proxy_url(host, port) +} + +fn proxy_url(host: &str, port: &str) -> Option { + if host.is_empty() || port.parse::().is_err() { + return None; + } + let host = if host.contains(':') && !host.starts_with('[') { format!("[{host}]") } else { host.to_string() }; + Some(format!("http://{host}:{port}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_macos_https_system_proxy() { + let output = r#" { + HTTPEnable : 1 + HTTPPort : 7890 + HTTPProxy : 127.0.0.1 + HTTPSEnable : 1 + HTTPSPort : 7891 + HTTPSProxy : 127.0.0.1 +}"#; + + assert_eq!(system_proxy_url_from_scutil_output(output), Some("http://127.0.0.1:7891".to_string())); + } + + #[test] + fn falls_back_to_macos_http_system_proxy() { + let output = r#" { + HTTPEnable : 1 + HTTPPort : 7890 + HTTPProxy : 127.0.0.1 + HTTPSEnable : 0 +}"#; + + assert_eq!(system_proxy_url_from_scutil_output(output), Some("http://127.0.0.1:7890".to_string())); + } + + #[test] + fn ignores_disabled_or_incomplete_macos_system_proxy() { + assert_eq!(system_proxy_url_from_scutil_output("HTTPEnable : 0\nHTTPProxy : 127.0.0.1\nHTTPPort : 7890"), None); + assert_eq!(system_proxy_url_from_scutil_output("HTTPEnable : 1\nHTTPProxy : 127.0.0.1"), None); + } + + #[test] + fn parses_windows_system_proxy() { + let enabled = r#" +HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Internet Settings + ProxyEnable REG_DWORD 0x1 +"#; + let server = r#" +HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Internet Settings + ProxyServer REG_SZ http=127.0.0.1:7890;https=127.0.0.1:7891 +"#; + + assert_eq!( + system_proxy_url_from_windows_registry_output(enabled, server), + Some("http://127.0.0.1:7891".to_string()) + ); + } + + #[test] + fn ignores_disabled_windows_system_proxy() { + let disabled = "ProxyEnable REG_DWORD 0x0"; + let server = "ProxyServer REG_SZ 127.0.0.1:7890"; + + assert_eq!(system_proxy_url_from_windows_registry_output(disabled, server), None); + } +} diff --git a/crates/dbx-core/src/db/ssh_prompt.rs b/crates/dbx-platform/src/ssh_prompt.rs similarity index 99% rename from crates/dbx-core/src/db/ssh_prompt.rs rename to crates/dbx-platform/src/ssh_prompt.rs index a6208f4c85..be35602c4d 100644 --- a/crates/dbx-core/src/db/ssh_prompt.rs +++ b/crates/dbx-platform/src/ssh_prompt.rs @@ -147,7 +147,7 @@ pub fn clear_ssh_prompt_gateway() { /// prompt and plugin Host API questions, so tests in any module must hold this /// lock while a gateway is installed — otherwise one test's gateway (or the /// fail-closed checks that rely on *no* gateway) clobbers another's. -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] pub fn prompt_gateway_test_lock() -> &'static tokio::sync::Mutex<()> { static LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); &LOCK diff --git a/crates/dbx-platform/src/version.rs b/crates/dbx-platform/src/version.rs new file mode 100644 index 0000000000..8421f683ae --- /dev/null +++ b/crates/dbx-platform/src/version.rs @@ -0,0 +1,45 @@ +pub fn normalize_version(version: &str) -> String { + version.trim().trim_start_matches('v').to_string() +} + +pub fn parse_version(version: &str) -> Vec { + normalize_version(version).split(['.', '-', '+']).map(|part| part.parse::().unwrap_or(0)).collect() +} + +pub fn is_newer_version(latest: &str, current: &str) -> bool { + let latest_parts = parse_version(latest); + let current_parts = parse_version(current); + let max_len = latest_parts.len().max(current_parts.len()); + + for i in 0..max_len { + let latest_part = *latest_parts.get(i).unwrap_or(&0); + let current_part = *current_parts.get(i).unwrap_or(&0); + if latest_part > current_part { + return true; + } + if latest_part < current_part { + return false; + } + } + + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_tag_versions() { + assert_eq!(normalize_version("v1.2.3"), "1.2.3"); + assert_eq!(normalize_version(" 0.2.0 "), "0.2.0"); + } + + #[test] + fn compares_semver_like_versions() { + assert!(is_newer_version("0.2.1", "0.2.0")); + assert!(is_newer_version("1.0.0", "0.9.9")); + assert!(!is_newer_version("0.2.0", "0.2.0")); + assert!(!is_newer_version("0.1.9", "0.2.0")); + } +} diff --git a/crates/dbx-plugin-runtime/Cargo.toml b/crates/dbx-plugin-runtime/Cargo.toml new file mode 100644 index 0000000000..70cc071529 --- /dev/null +++ b/crates/dbx-plugin-runtime/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "dbx-plugin-runtime" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +test-support = [] + +[dependencies] +dbx-platform = { path = "../dbx-platform", features = ["host-prompts"] } +dbx-types = { path = "../dbx-types", default-features = false } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +semver = "1" +ed25519-dalek = "2" +fs2 = "0.4" +log = "0.4" +tokio = { version = "1", features = ["process", "io-util", "sync", "time", "rt", "macros"] } +chrono = { version = "0.4", features = ["serde"] } +uuid = { version = "1", features = ["v4"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "socks"] } +futures = "0.3" +base64 = "0.22" +sha2 = "0.10" +bytes = "1" +zip = { version = "4", default-features = false, features = ["deflate"] } +tempfile = "3" + +[dev-dependencies] +dbx-platform = { path = "../dbx-platform", features = ["test-support"] } +tokio = { version = "1", features = ["test-util", "rt-multi-thread"] } diff --git a/crates/dbx-plugin-runtime/src/lib.rs b/crates/dbx-plugin-runtime/src/lib.rs new file mode 100644 index 0000000000..cdc5a9289a --- /dev/null +++ b/crates/dbx-plugin-runtime/src/lib.rs @@ -0,0 +1,4 @@ +pub use dbx_platform::process; +pub use dbx_types::models; + +pub mod plugins; diff --git a/crates/dbx-core/src/plugins.rs b/crates/dbx-plugin-runtime/src/plugins.rs similarity index 99% rename from crates/dbx-core/src/plugins.rs rename to crates/dbx-plugin-runtime/src/plugins.rs index 7c42f05bdb..8d6979036c 100644 --- a/crates/dbx-core/src/plugins.rs +++ b/crates/dbx-plugin-runtime/src/plugins.rs @@ -405,8 +405,8 @@ impl PluginDriverSession { self.sidecar.send_binary(channel, data).await } - #[cfg(all(test, unix))] - pub(crate) async fn start_for_test( + #[cfg(all(any(test, feature = "test-support"), unix))] + pub async fn start_for_test( plugin: InstalledPlugin, driver_id: String, env: PluginRuntimeEnv, diff --git a/crates/dbx-core/src/plugins/assets.rs b/crates/dbx-plugin-runtime/src/plugins/assets.rs similarity index 100% rename from crates/dbx-core/src/plugins/assets.rs rename to crates/dbx-plugin-runtime/src/plugins/assets.rs diff --git a/crates/dbx-core/src/plugins/filesystem.rs b/crates/dbx-plugin-runtime/src/plugins/filesystem.rs similarity index 100% rename from crates/dbx-core/src/plugins/filesystem.rs rename to crates/dbx-plugin-runtime/src/plugins/filesystem.rs diff --git a/crates/dbx-core/src/plugins/host.rs b/crates/dbx-plugin-runtime/src/plugins/host.rs similarity index 100% rename from crates/dbx-core/src/plugins/host.rs rename to crates/dbx-plugin-runtime/src/plugins/host.rs diff --git a/crates/dbx-core/src/plugins/installer.rs b/crates/dbx-plugin-runtime/src/plugins/installer.rs similarity index 100% rename from crates/dbx-core/src/plugins/installer.rs rename to crates/dbx-plugin-runtime/src/plugins/installer.rs diff --git a/crates/dbx-core/src/plugins/lifecycle.rs b/crates/dbx-plugin-runtime/src/plugins/lifecycle.rs similarity index 100% rename from crates/dbx-core/src/plugins/lifecycle.rs rename to crates/dbx-plugin-runtime/src/plugins/lifecycle.rs diff --git a/crates/dbx-core/src/plugins/manifest.rs b/crates/dbx-plugin-runtime/src/plugins/manifest.rs similarity index 100% rename from crates/dbx-core/src/plugins/manifest.rs rename to crates/dbx-plugin-runtime/src/plugins/manifest.rs diff --git a/crates/dbx-core/src/plugins/marketplace.rs b/crates/dbx-plugin-runtime/src/plugins/marketplace.rs similarity index 100% rename from crates/dbx-core/src/plugins/marketplace.rs rename to crates/dbx-plugin-runtime/src/plugins/marketplace.rs diff --git a/crates/dbx-core/src/plugins/runtime.rs b/crates/dbx-plugin-runtime/src/plugins/runtime.rs similarity index 98% rename from crates/dbx-core/src/plugins/runtime.rs rename to crates/dbx-plugin-runtime/src/plugins/runtime.rs index 7ada689514..58fbf56e91 100644 --- a/crates/dbx-core/src/plugins/runtime.rs +++ b/crates/dbx-plugin-runtime/src/plugins/runtime.rs @@ -17,7 +17,7 @@ use super::{ InstalledPlugin, PluginBackendTransport, PluginRuntimeEnv, SUPPORTED_PLUGIN_HOST_API_VERSION, SUPPORTED_PLUGIN_HOST_FEATURES, SUPPORTED_PLUGIN_PROTOCOL_VERSION, }; -use crate::db::ssh_prompt; +use dbx_platform::ssh_prompt; pub const PLUGIN_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// Host API method a plugin backend calls to ask the user for input @@ -1237,12 +1237,12 @@ sleep 30 #[cfg(unix)] fn install_prompt_harness( delay: Duration, - answer: crate::db::ssh_prompt::SshPromptAnswer, + answer: dbx_platform::ssh_prompt::SshPromptAnswer, ) -> tokio::sync::mpsc::UnboundedReceiver<( - crate::db::ssh_prompt::SshPromptRequest, - crate::db::ssh_prompt::SshPromptAnswer, + dbx_platform::ssh_prompt::SshPromptRequest, + dbx_platform::ssh_prompt::SshPromptAnswer, )> { - use crate::db::ssh_prompt; + use dbx_platform::ssh_prompt; let (tx, mut rx) = tokio::sync::mpsc::channel::(4); ssh_prompt::install_ssh_prompt_gateway(tx); @@ -1261,10 +1261,10 @@ sleep 30 /// answers, so the test can watch what happens to a prompt nobody answers. #[cfg(unix)] fn install_silent_prompt_harness() -> tokio::sync::mpsc::UnboundedReceiver<( - crate::db::ssh_prompt::SshPromptRequest, - tokio::sync::oneshot::Sender, + dbx_platform::ssh_prompt::SshPromptRequest, + tokio::sync::oneshot::Sender, )> { - use crate::db::ssh_prompt; + use dbx_platform::ssh_prompt; let (tx, mut rx) = tokio::sync::mpsc::channel::(4); ssh_prompt::install_ssh_prompt_gateway(tx); @@ -1280,7 +1280,7 @@ sleep 30 #[cfg(unix)] #[tokio::test] async fn fails_plugin_prompts_closed_without_a_prompt_ui() { - use crate::db::ssh_prompt; + use dbx_platform::ssh_prompt; let _guard = ssh_prompt::prompt_gateway_test_lock().lock().await; ssh_prompt::clear_ssh_prompt_gateway(); @@ -1305,7 +1305,7 @@ sleep 30 #[cfg(unix)] #[tokio::test] async fn relays_plugin_user_input_through_the_prompt_gateway() { - use crate::db::ssh_prompt::{self, SshPromptAnswer, SshPromptKind}; + use dbx_platform::ssh_prompt::{self, SshPromptAnswer, SshPromptKind}; let _guard = ssh_prompt::prompt_gateway_test_lock().lock().await; let mut prompts = install_prompt_harness(Duration::ZERO, SshPromptAnswer::Secret("123456".to_string())); @@ -1337,7 +1337,7 @@ sleep 30 #[cfg(unix)] #[tokio::test] async fn reports_plugin_user_input_cancel_when_the_user_dismisses_the_prompt() { - use crate::db::ssh_prompt::{self, SshPromptAnswer}; + use dbx_platform::ssh_prompt::{self, SshPromptAnswer}; let _guard = ssh_prompt::prompt_gateway_test_lock().lock().await; install_prompt_harness(Duration::ZERO, SshPromptAnswer::Reject); @@ -1360,7 +1360,7 @@ sleep 30 #[cfg(unix)] #[tokio::test] async fn connection_request_timeout_pauses_while_the_user_types() { - use crate::db::ssh_prompt::{self, SshPromptAnswer}; + use dbx_platform::ssh_prompt::{self, SshPromptAnswer}; let _guard = ssh_prompt::prompt_gateway_test_lock().lock().await; // The user needs longer than the caller's connect timeout. `connection/ @@ -1385,7 +1385,7 @@ sleep 30 #[cfg(unix)] #[tokio::test] async fn stopping_a_plugin_session_releases_its_open_prompt() { - use crate::db::ssh_prompt; + use dbx_platform::ssh_prompt; let _guard = ssh_prompt::prompt_gateway_test_lock().lock().await; let mut prompts = install_silent_prompt_harness(); @@ -1400,7 +1400,7 @@ sleep 30 ) }; let (request, responder) = prompts.recv().await.expect("the plugin should ask before answering"); - assert_eq!(request.kind, crate::db::ssh_prompt::SshPromptKind::UserInput); + assert_eq!(request.kind, dbx_platform::ssh_prompt::SshPromptKind::UserInput); assert!(!responder.is_closed()); session.shutdown().await; @@ -1419,7 +1419,7 @@ sleep 30 #[cfg(unix)] #[tokio::test] async fn stopping_a_plugin_session_releases_all_open_prompts() { - use crate::db::ssh_prompt; + use dbx_platform::ssh_prompt; let _guard = ssh_prompt::prompt_gateway_test_lock().lock().await; let mut prompts = install_silent_prompt_harness(); @@ -1529,7 +1529,7 @@ sleep 30 #[test] fn maps_user_answers_to_host_request_user_input_results() { - use crate::db::ssh_prompt::SshPromptAnswer; + use dbx_platform::ssh_prompt::SshPromptAnswer; assert_eq!( user_input_result(SshPromptAnswer::Secret("123456".to_string())), diff --git a/crates/dbx-sql/Cargo.toml b/crates/dbx-sql/Cargo.toml new file mode 100644 index 0000000000..f51cc86e8b --- /dev/null +++ b/crates/dbx-sql/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "dbx-sql" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +duckdb-sidecar = [] +openapi = ["dep:utoipa", "dbx-types/openapi"] + +[dependencies] +dbx-types = { path = "../dbx-types", default-features = false } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +utoipa = { version = "5.5", optional = true } +regex = "1" +sqlparser = { version = "0.62.0", features = ["visitor"] } +winnow = "1.0" +rayon = "1" +log = "0.4" +tokio = { version = "1", features = ["sync", "time", "rt", "macros"] } +chrono = { version = "0.4", features = ["serde"] } +percent-encoding = "2" +encoding_rs = "0.8" +unicode-ident = "1" +unicode-width = "0.2" +serde_yaml = "0.9" +notify = { version = "7", default-features = false, features = ["macos_kqueue"] } + +[dev-dependencies] +insta = { version = "1", features = ["json", "glob"] } +tempfile = "3" +rusqlite = { version = "0.32", default-features = false } +tokio = { version = "1", features = ["test-util", "rt-multi-thread"] } diff --git a/crates/dbx-sql/build.rs b/crates/dbx-sql/build.rs new file mode 100644 index 0000000000..086a063877 --- /dev/null +++ b/crates/dbx-sql/build.rs @@ -0,0 +1,56 @@ +use std::env; +use std::path::{Path, PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + generate_core_dialects(&manifest_dir.join("../../plugins/dialects"), &out_dir); +} + +fn yaml_entries(directory: &Path) -> Vec { + let mut entries: Vec<_> = std::fs::read_dir(directory) + .unwrap_or_else(|_| panic!("Cannot read {} directory", directory.display())) + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "yaml" || extension == "yml")) + .collect(); + entries.sort(); + entries +} + +fn generate_core_dialects(dialects_dir: &Path, out_dir: &Path) { + let dialects_dir = std::fs::canonicalize(dialects_dir).unwrap_or_else(|_| dialects_dir.to_path_buf()); + let dest_path = out_dir.join("core_dialects.rs"); + let entries = yaml_entries(&dialects_dir); + + // Watch the directory itself so additions/removals of dialect files trigger a rebuild. + println!("cargo::rerun-if-changed={}", dialects_dir.to_str().unwrap()); + + let mut code = String::from("{\n"); + + for path in &entries { + let canonical = std::fs::canonicalize(path).unwrap_or_else(|_| path.clone()); + let file_name = path.file_stem().unwrap().to_str().unwrap(); + let path_str = canonical.to_str().unwrap(); + + // Watch each dialect file individually. Editing a single YAML must invalidate + // the embedded `core_dialects.rs`, otherwise the compiled binary keeps a stale + // type catalog (e.g. old type names) and silently misbehaves (see field mapping). + println!("cargo::rerun-if-changed={}", path_str); + + code.push_str("match crate::sql_dialect::dialect_loader::DialectPluginLoader::load_from_string(\n"); + code.push_str(&format!(" include_str!(\"{}\"),\n", path_str.replace('\\', "\\\\"))); + code.push_str(" None,\n"); + code.push_str(") {\n"); + code.push_str(" Ok((_kind, yaml, descriptor)) => {\n"); + code.push_str(" let name = yaml.dialect.name.clone();\n"); + code.push_str(" registry.register_descriptor(&name, descriptor, yaml);\n"); + code.push_str(" }\n"); + code.push_str(&format!(" Err(e) => log::warn!(\"Failed to load core dialect '{}': {{e}}\"),\n", file_name)); + code.push_str("};\n"); + } + + code.push_str("}\n"); + + std::fs::write(&dest_path, code).expect("Failed to write core_dialects.rs"); +} diff --git a/crates/dbx-core/src/data_grid_extractors/contract.rs b/crates/dbx-sql/src/data_grid_extractors/contract.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/contract.rs rename to crates/dbx-sql/src/data_grid_extractors/contract.rs diff --git a/crates/dbx-core/src/data_grid_extractors/delimited.rs b/crates/dbx-sql/src/data_grid_extractors/delimited.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/delimited.rs rename to crates/dbx-sql/src/data_grid_extractors/delimited.rs diff --git a/crates/dbx-core/src/data_grid_extractors/document.rs b/crates/dbx-sql/src/data_grid_extractors/document.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/document.rs rename to crates/dbx-sql/src/data_grid_extractors/document.rs diff --git a/crates/dbx-core/src/data_grid_extractors/json.rs b/crates/dbx-sql/src/data_grid_extractors/json.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/json.rs rename to crates/dbx-sql/src/data_grid_extractors/json.rs diff --git a/crates/dbx-core/src/data_grid_extractors/mod.rs b/crates/dbx-sql/src/data_grid_extractors/mod.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/mod.rs rename to crates/dbx-sql/src/data_grid_extractors/mod.rs diff --git a/crates/dbx-core/src/data_grid_extractors/raw.rs b/crates/dbx-sql/src/data_grid_extractors/raw.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/raw.rs rename to crates/dbx-sql/src/data_grid_extractors/raw.rs diff --git a/crates/dbx-core/src/data_grid_extractors/sql.rs b/crates/dbx-sql/src/data_grid_extractors/sql.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/sql.rs rename to crates/dbx-sql/src/data_grid_extractors/sql.rs diff --git a/crates/dbx-core/src/data_grid_extractors/tests.rs b/crates/dbx-sql/src/data_grid_extractors/tests.rs similarity index 100% rename from crates/dbx-core/src/data_grid_extractors/tests.rs rename to crates/dbx-sql/src/data_grid_extractors/tests.rs diff --git a/crates/dbx-core/src/data_grid_iotdb_sql.rs b/crates/dbx-sql/src/data_grid_iotdb_sql.rs similarity index 100% rename from crates/dbx-core/src/data_grid_iotdb_sql.rs rename to crates/dbx-sql/src/data_grid_iotdb_sql.rs diff --git a/crates/dbx-core/src/data_grid_neo4j_sql.rs b/crates/dbx-sql/src/data_grid_neo4j_sql.rs similarity index 100% rename from crates/dbx-core/src/data_grid_neo4j_sql.rs rename to crates/dbx-sql/src/data_grid_neo4j_sql.rs diff --git a/crates/dbx-core/src/data_grid_sql.rs b/crates/dbx-sql/src/data_grid_sql.rs similarity index 99% rename from crates/dbx-core/src/data_grid_sql.rs rename to crates/dbx-sql/src/data_grid_sql.rs index 81f4988ab4..832db12dfa 100644 --- a/crates/dbx-core/src/data_grid_sql.rs +++ b/crates/dbx-sql/src/data_grid_sql.rs @@ -26,11 +26,11 @@ use crate::sql_dialect::{ firebird_rows_clause, quote_table_identifier, table_pagination_strategy, uses_oracle_row_id, uses_single_row_insert_statements, uses_synthetic_row_id, uses_xugu_row_id, TablePaginationStrategy, }; -use crate::transfer::{format_ch_array_sql_literal, format_pg_array_sql_literal}; +use crate::value_literals::{format_ch_array_sql_literal, format_pg_array_sql_literal}; const DBX_ROWID_COLUMN: &str = "__DBX_ROWID"; -pub(crate) const DBX_NEO4J_ELEMENT_ID_COLUMN: &str = "__DBX_ELEMENT_ID"; -pub(crate) const DBX_TDENGINE_TBNAME_COLUMN: &str = "tbname"; +pub const DBX_NEO4J_ELEMENT_ID_COLUMN: &str = "__DBX_ELEMENT_ID"; +pub const DBX_TDENGINE_TBNAME_COLUMN: &str = "tbname"; const DATA_GRID_COLUMN_DISTINCT_VALUES_DEFAULT_LIMIT: usize = 1000; const DATA_GRID_COLUMN_DISTINCT_VALUES_MAX_LIMIT: usize = 1000; /// Alias for the single cell a keyless guard query returns. @@ -377,7 +377,7 @@ pub fn prepare_data_grid_save_for_driver_profile( /// Relational SQL UPDATE/WHERE predicates are not meaningful for graph, /// document, or time-series stores that don't speak relational SQL. -pub(crate) fn supports_relational_copy_predicates(database_type: Option) -> bool { +pub fn supports_relational_copy_predicates(database_type: Option) -> bool { !matches!(database_type, Some(DatabaseType::Neo4j | DatabaseType::Tdengine | DatabaseType::MongoDb)) } @@ -1985,7 +1985,7 @@ fn build_mysql_insert_rollback_where( )) } -pub(crate) fn effective_columns(options: &DataGridSaveStatementOptions) -> Vec> { +pub fn effective_columns(options: &DataGridSaveStatementOptions) -> Vec> { let columns = match &options.source_columns { Some(source_columns) if source_columns.len() == options.columns.len() => source_columns.clone(), _ => options.columns.iter().map(|column| Some(column.clone())).collect(), @@ -2279,7 +2279,7 @@ pub fn format_grid_sql_literal( format_grid_sql_literal_with_identifier_quote(value, database_type, column_info, None) } -pub(crate) fn format_grid_sql_literal_with_identifier_quote( +pub fn format_grid_sql_literal_with_identifier_quote( value: &Value, database_type: Option, column_info: Option<&DataGridColumnInfo>, @@ -3077,7 +3077,7 @@ fn build_save_row_where( .join(" AND ") } -pub(crate) fn build_column_predicate( +pub fn build_column_predicate( database_type: Option, column: &str, value: &Value, @@ -3246,17 +3246,17 @@ fn is_synthetic_row_id(database_type: Option, name: Option<&str>) uses_synthetic_row_id(database_type) && name.is_some_and(|name| name.eq_ignore_ascii_case(DBX_ROWID_COLUMN)) } -pub(crate) fn is_neo4j_element_id(database_type: Option, name: Option<&str>) -> bool { +pub fn is_neo4j_element_id(database_type: Option, name: Option<&str>) -> bool { database_type == Some(DatabaseType::Neo4j) && name == Some(DBX_NEO4J_ELEMENT_ID_COLUMN) } -pub(crate) fn extra_is_auto_generated(extra: &str) -> bool { +pub fn extra_is_auto_generated(extra: &str) -> bool { extra.to_ascii_lowercase().split(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_').any(|part| { matches!(part, "auto_increment" | "autoincrement" | "identity" | "smallserial" | "serial" | "bigserial") }) } -pub(crate) fn is_auto_generated_column(column: &DataGridColumnInfo) -> bool { +pub fn is_auto_generated_column(column: &DataGridColumnInfo) -> bool { extra_is_auto_generated(column.extra.as_deref().unwrap_or("")) || column.column_default.as_deref().is_some_and(|default| default.to_ascii_lowercase().contains("nextval(")) } @@ -3265,7 +3265,7 @@ fn grid_value_is_empty(value: &Value) -> bool { value.is_null() || value.as_str().is_some_and(str::is_empty) } -pub(crate) fn is_grid_insert_omitted_column( +pub fn is_grid_insert_omitted_column( database_type: Option, column_info: Option<&DataGridColumnInfo>, name: Option<&str>, @@ -3321,7 +3321,7 @@ fn is_postgres_tsvector_type(data_type: &str) -> bool { normalized == "tsvector" || normalized.ends_with(".tsvector") } -pub(crate) fn is_non_identity_generated_column(column_info: Option<&DataGridColumnInfo>) -> bool { +pub fn is_non_identity_generated_column(column_info: Option<&DataGridColumnInfo>) -> bool { let extra = column_info.and_then(|column| column.extra.as_deref()).unwrap_or("").to_ascii_lowercase(); (extra.contains("generated always as") || extra.contains("virtual generated") || extra.contains("stored generated")) && !extra.contains("identity") @@ -3427,15 +3427,11 @@ fn predicate_ident(database_type: Option, name: &str, identifier_q } } -pub(crate) fn quote_ident(database_type: Option, name: &str) -> String { +pub fn quote_ident(database_type: Option, name: &str) -> String { quote_table_identifier(database_type, name) } -pub(crate) fn qualified_table_name( - database_type: Option, - schema: Option<&str>, - table_name: &str, -) -> String { +pub fn qualified_table_name(database_type: Option, schema: Option<&str>, table_name: &str) -> String { crate::sql_dialect::qualified_table_name(database_type, schema, table_name) } @@ -3446,7 +3442,7 @@ fn data_grid_identifier(database_type: Option, name: &str, identif crate::sql_dialect::quote_table_data_identifier(database_type, name, identifier_quote) } -pub(crate) fn data_grid_qualified_table_name( +pub fn data_grid_qualified_table_name( database_type: Option, catalog: Option<&str>, schema: Option<&str>, @@ -3665,7 +3661,7 @@ fn uses_keyless_row_predicate(database_type: Option) -> bool { ) } -pub(crate) fn column_info_for<'a>(columns: &'a [DataGridColumnInfo], name: &str) -> Option<&'a DataGridColumnInfo> { +pub fn column_info_for<'a>(columns: &'a [DataGridColumnInfo], name: &str) -> Option<&'a DataGridColumnInfo> { let normalized = normalize_column_name(name); columns.iter().find(|column| normalize_column_name(&column.name) == normalized) } diff --git a/crates/dbx-core/src/data_grid_tdengine_sql.rs b/crates/dbx-sql/src/data_grid_tdengine_sql.rs similarity index 100% rename from crates/dbx-core/src/data_grid_tdengine_sql.rs rename to crates/dbx-sql/src/data_grid_tdengine_sql.rs diff --git a/crates/dbx-core/src/database_search_sql.rs b/crates/dbx-sql/src/database_search_sql.rs similarity index 100% rename from crates/dbx-core/src/database_search_sql.rs rename to crates/dbx-sql/src/database_search_sql.rs diff --git a/crates/dbx-core/src/db_admin_sql.rs b/crates/dbx-sql/src/db_admin_sql.rs similarity index 100% rename from crates/dbx-core/src/db_admin_sql.rs rename to crates/dbx-sql/src/db_admin_sql.rs diff --git a/crates/dbx-core/src/dml_binding.rs b/crates/dbx-sql/src/dml_binding.rs similarity index 99% rename from crates/dbx-core/src/dml_binding.rs rename to crates/dbx-sql/src/dml_binding.rs index ce46d56d1e..d29e1cbacf 100644 --- a/crates/dbx-core/src/dml_binding.rs +++ b/crates/dbx-sql/src/dml_binding.rs @@ -57,7 +57,7 @@ fn normalize_type(t: &str) -> String { // ============================================================================ pub struct DmlCleanRuleRegistry { - pub(crate) rules: Vec, + pub rules: Vec, } impl DmlCleanRuleRegistry { diff --git a/crates/dbx-core/src/dml_preview_sql.rs b/crates/dbx-sql/src/dml_preview_sql.rs similarity index 100% rename from crates/dbx-core/src/dml_preview_sql.rs rename to crates/dbx-sql/src/dml_preview_sql.rs diff --git a/crates/dbx-sql/src/lib.rs b/crates/dbx-sql/src/lib.rs new file mode 100644 index 0000000000..5c1d1267a5 --- /dev/null +++ b/crates/dbx-sql/src/lib.rs @@ -0,0 +1,28 @@ +#![recursion_limit = "256"] + +pub use dbx_types::{database_manifest, models, types}; + +pub mod data_grid_extractors; +pub mod data_grid_sql; +pub mod database_search_sql; +pub mod db_admin_sql; +pub mod dml_binding; +pub mod dml_preview_sql; +pub mod mysql_ddl_normalize; +pub mod mysql_event_sql; +pub mod object_source_sql; +pub mod query_execution_sql; +pub mod query_result_sql; +pub mod schema_diff; +pub mod sql; +pub mod sql_analysis; +pub mod sql_diagnostics; +pub mod sql_dialect; +pub mod sql_editability; +pub mod sql_error_position; +pub mod sql_parser; +pub mod sql_risk; +pub mod sqlserver_temporal; +pub mod table_structure_sql; +pub mod tdsql_mysql; +pub mod value_literals; diff --git a/crates/dbx-core/src/mysql_ddl_normalize.rs b/crates/dbx-sql/src/mysql_ddl_normalize.rs similarity index 99% rename from crates/dbx-core/src/mysql_ddl_normalize.rs rename to crates/dbx-sql/src/mysql_ddl_normalize.rs index 2fdfc8edd3..e20042f688 100644 --- a/crates/dbx-core/src/mysql_ddl_normalize.rs +++ b/crates/dbx-sql/src/mysql_ddl_normalize.rs @@ -33,7 +33,7 @@ use winnow::ModalResult; /// Knobs for [`normalize_mysql_export_ddl`]. #[derive(Debug, Clone, Copy, Default)] -pub(crate) struct DdlNormalizeOptions { +pub struct DdlNormalizeOptions { /// When true, drop the table-level `AUTO_INCREMENT=N` option from the DDL. pub omit_auto_increment: bool, } @@ -56,7 +56,7 @@ const ROW_FORMAT: &str = "ROW_FORMAT"; /// Entry point. Returns the (possibly edited) DDL. On any structural surprise /// the input is returned unchanged. -pub(crate) fn normalize_mysql_export_ddl(ddl: &str, opts: DdlNormalizeOptions) -> String { +pub fn normalize_mysql_export_ddl(ddl: &str, opts: DdlNormalizeOptions) -> String { // One winnow pass over the whole DDL (wrapped in `LocatingSlice`), so every // option carries an absolute byte span into `ddl` — no offset math. let Some(options) = parse_create_table_options(ddl) else { diff --git a/crates/dbx-core/src/mysql_event_sql.rs b/crates/dbx-sql/src/mysql_event_sql.rs similarity index 89% rename from crates/dbx-core/src/mysql_event_sql.rs rename to crates/dbx-sql/src/mysql_event_sql.rs index 8795adf01f..c8be5ed2f9 100644 --- a/crates/dbx-core/src/mysql_event_sql.rs +++ b/crates/dbx-sql/src/mysql_event_sql.rs @@ -20,28 +20,7 @@ pub struct MysqlEventDefinition { pub body: String, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct MysqlEventInfo { - pub name: String, - pub schema: String, - pub definer: Option, - pub time_zone: Option, - pub event_type: Option, - pub execute_at: Option, - pub interval_value: Option, - pub interval_field: Option, - pub starts: Option, - pub ends: Option, - pub status: Option, - pub on_completion: Option, - pub comment: Option, - pub event_body: Option, - pub event_definition: Option, - pub created_at: Option, - pub updated_at: Option, - pub last_executed: Option, - pub source: Option, -} +pub use dbx_types::mysql_event::MysqlEventInfo; fn ident(value: &str) -> Result { let value = value.trim(); diff --git a/crates/dbx-core/src/object_source_sql.rs b/crates/dbx-sql/src/object_source_sql.rs similarity index 99% rename from crates/dbx-core/src/object_source_sql.rs rename to crates/dbx-sql/src/object_source_sql.rs index fb7cb144ff..253262d874 100644 --- a/crates/dbx-core/src/object_source_sql.rs +++ b/crates/dbx-sql/src/object_source_sql.rs @@ -376,7 +376,7 @@ fn quote_mysql_identifier(value: &str) -> String { format!("`{}`", value.replace('`', "``")) } -pub(crate) fn ensure_oracle_ddl_terminated(sql: &str) -> String { +pub fn ensure_oracle_ddl_terminated(sql: &str) -> String { let trimmed = sql.trim_end(); if trimmed.trim().is_empty() { return String::new(); diff --git a/crates/dbx-core/src/query_execution_sql.rs b/crates/dbx-sql/src/query_execution_sql.rs similarity index 99% rename from crates/dbx-core/src/query_execution_sql.rs rename to crates/dbx-sql/src/query_execution_sql.rs index 8e1f2f71dc..d2ca9c80bd 100644 --- a/crates/dbx-core/src/query_execution_sql.rs +++ b/crates/dbx-sql/src/query_execution_sql.rs @@ -272,16 +272,13 @@ fn classify_dynamodb_statement_write(source: &str) -> Option { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum SearchEngineQueryRisk { +pub enum SearchEngineQueryRisk { ReadOnly, Write, Dangerous, } -pub(crate) fn classify_search_engine_query_risk( - source: &str, - database_type: DatabaseType, -) -> Option { +pub fn classify_search_engine_query_risk(source: &str, database_type: DatabaseType) -> Option { if !matches!(database_type, DatabaseType::Elasticsearch | DatabaseType::Easysearch) { return None; } @@ -408,7 +405,7 @@ fn is_postgresql_family_database(database_type: DatabaseType) -> bool { /// Detects write-capable syntax that otherwise looks like a read query and is /// interpreted differently depending on the database dialect. -pub(crate) fn has_dialect_specific_write(sql: &str, database_type: DatabaseType) -> bool { +pub fn has_dialect_specific_write(sql: &str, database_type: DatabaseType) -> bool { let statements = crate::sql::split_sql_statements_for_database(sql, database_type); statements.iter().any(|statement| has_dialect_specific_write_statement(statement, database_type)) } @@ -444,7 +441,7 @@ fn contains_keyword_sequence(sql: &str, first: &str, second: &str) -> bool { false } -pub(crate) fn contains_unquoted_keyword(sql: &str, dialect: &dyn sqlparser::dialect::Dialect, keyword: &str) -> bool { +pub fn contains_unquoted_keyword(sql: &str, dialect: &dyn sqlparser::dialect::Dialect, keyword: &str) -> bool { Tokenizer::new(dialect, sql).tokenize().is_ok_and(|tokens| { tokens.into_iter().any(|token| { matches!(token, Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case(keyword)) @@ -709,7 +706,7 @@ fn has_extra_statement_after_semicolon(sql: &str) -> bool { stripped.split(';').skip(1).any(|part| !part.trim().is_empty()) } -pub(crate) fn strip_sql_comments(sql: &str) -> String { +pub fn strip_sql_comments(sql: &str) -> String { let mut output = String::with_capacity(sql.len()); let mut chars = sql.chars().peekable(); let mut in_line_comment = false; diff --git a/crates/dbx-core/src/query_result_sql.rs b/crates/dbx-sql/src/query_result_sql.rs similarity index 99% rename from crates/dbx-core/src/query_result_sql.rs rename to crates/dbx-sql/src/query_result_sql.rs index 8ec918ecb1..42f0afd744 100644 --- a/crates/dbx-core/src/query_result_sql.rs +++ b/crates/dbx-sql/src/query_result_sql.rs @@ -756,7 +756,7 @@ fn add_sql_server_rowcount_pagination(statement: &str, limit: usize, offset: usi ) } -pub(crate) fn sqlserver_result_offset(sql: &str) -> usize { +pub fn sqlserver_result_offset(sql: &str) -> usize { let sql = sql.trim_end(); if !sql.starts_with("EXEC sys.sp_executesql N'SET ROWCOUNT ") || !sql.ends_with(SQLSERVER_RESULT_OFFSET_SUFFIX) { return 0; @@ -940,7 +940,7 @@ fn leading_execution_hint_prefix_for_database(sql: &str, database_type: Option bool { /// True when the statement has a top-level `TOP` clause (SQL Server dialect). /// Kingbase's SQL Server compatibility mode treats TOP as a real clause, so a /// statement that already bounds rows with TOP must not receive a sibling LIMIT. -pub(crate) fn has_top_level_top(sql: &str) -> bool { +pub fn has_top_level_top(sql: &str) -> bool { top_level_sql_tokens(sql).iter().any(|token| token.text == "TOP") } @@ -1671,7 +1671,7 @@ pub(crate) fn has_top_level_top(sql: &str) -> bool { /// TOP clause has no literal at all. A parenthesized form is only accepted when /// it is exactly one integer literal followed by `)`, so the returned bound is /// always exact — never a silent under-count. -pub(crate) fn top_level_top_row_count(sql: &str) -> Option { +pub fn top_level_top_row_count(sql: &str) -> Option { let tokens = top_level_sql_tokens(sql); let top_index = tokens.iter().position(|token| token.text == "TOP")?; let top_token = &tokens[top_index]; diff --git a/crates/dbx-core/src/schema_diff.rs b/crates/dbx-sql/src/schema_diff.rs similarity index 96% rename from crates/dbx-core/src/schema_diff.rs rename to crates/dbx-sql/src/schema_diff.rs index 640aaed234..e19e0e242f 100644 --- a/crates/dbx-core/src/schema_diff.rs +++ b/crates/dbx-sql/src/schema_diff.rs @@ -3519,7 +3519,7 @@ pub fn diff_triggers(source: &[TriggerInfo], target: &[TriggerInfo]) -> Vec String { +pub fn normalize_definition(def: &str) -> String { def.replace("\r\n", "\n") .split('\n') .map(|line| line.split_whitespace().collect::>().join(" ")) @@ -4008,7 +4008,7 @@ fn postgres_index_column_sql(column: &str, is_expression: Option, db_type: } } -fn create_index_sql(table_name: &str, index: &IndexInfo, db_type: DatabaseType, schema: Option<&str>) -> String { +pub fn create_index_sql(table_name: &str, index: &IndexInfo, db_type: DatabaseType, schema: Option<&str>) -> String { use crate::sql_dialect::ddl_profile::IndexTypePlacement; let profile = profile_for(db_type); let table = qualified_name(table_name, db_type, schema); @@ -7410,276 +7410,6 @@ mod tests { assert!(executable[0].contains("EXEC sys.sp_executesql N'"), "single executable batch: {sql}"); } - #[tokio::test] - #[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"] - async fn live_sqlserver_alter_column_preserves_unchanged_runtime_dependencies() { - let database = std::env::var("DBX_LIVE_SQLSERVER_DATABASE").unwrap_or_else(|_| "tempdb".to_string()); - let host = std::env::var("DBX_LIVE_SQLSERVER_HOST").expect("DBX_LIVE_SQLSERVER_HOST"); - let port = std::env::var("DBX_LIVE_SQLSERVER_PORT") - .expect("DBX_LIVE_SQLSERVER_PORT") - .parse() - .expect("valid DBX_LIVE_SQLSERVER_PORT"); - let user = std::env::var("DBX_LIVE_SQLSERVER_USER").expect("DBX_LIVE_SQLSERVER_USER"); - let password = std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"); - let mut client = crate::db::sqlserver::connect( - &host, - port, - &user, - &password, - Some(&database), - None, - std::time::Duration::from_secs(10), - ) - .await - .expect("connect SQL Server"); - - let suffix = uuid::Uuid::new_v4().simple().to_string(); - let lookup_table = format!("dbx_dep_lookup_{suffix}"); - let parent_table = format!("dbx_dep_parent_{suffix}"); - let child_table = format!("dbx_dep_child_{suffix}"); - let primary_key = format!("PK_dbx_dep_parent_{suffix}"); - let unique_constraint = format!("UQ_dbx_dep_parent_code_{suffix}"); - let lookup_unique_constraint = format!("UQ_dbx_dep_lookup_code_{suffix}"); - let check_constraint = format!("CK_dbx_dep_parent_values_{suffix}"); - let outbound_foreign_key = format!("FK_dbx_dep_parent_lookup_{suffix}"); - let inbound_foreign_key = format!("FK_dbx_dep_child_parent_{suffix}"); - let ordinary_index = format!("IX_dbx_dep_parent_code_{suffix}"); - let cleanup = format!( - "DROP TABLE IF EXISTS [dbo].[{child_table}];\ - DROP TABLE IF EXISTS [dbo].[{parent_table}];\ - DROP TABLE IF EXISTS [dbo].[{lookup_table}];" - ); - let _ = crate::db::sqlserver::execute_batch(&mut client, &cleanup).await; - - let exercise = async { - let setup = format!( - "SET ANSI_NULLS ON;\ - SET QUOTED_IDENTIFIER ON;\ - SET ANSI_PADDING ON;\ - SET ANSI_WARNINGS ON;\ - SET CONCAT_NULL_YIELDS_NULL ON;\ - SET ARITHABORT ON;\ - SET NUMERIC_ROUNDABORT OFF;\ - CREATE TABLE [dbo].[{lookup_table}] (\ - [code] INT NOT NULL,\ - CONSTRAINT [{lookup_unique_constraint}] UNIQUE NONCLUSTERED ([code] ASC)\ - );\ - CREATE TABLE [dbo].[{parent_table}] (\ - [id] INT NOT NULL,\ - [code] INT NOT NULL,\ - [payload] NVARCHAR(64) NULL,\ - CONSTRAINT [{primary_key}] PRIMARY KEY CLUSTERED ([id] ASC),\ - CONSTRAINT [{unique_constraint}] UNIQUE NONCLUSTERED ([code] ASC),\ - CONSTRAINT [{check_constraint}] CHECK ([id] > 0 AND [code] >= 0),\ - CONSTRAINT [{outbound_foreign_key}] FOREIGN KEY ([code])\ - REFERENCES [dbo].[{lookup_table}] ([code])\ - );\ - CREATE TABLE [dbo].[{child_table}] (\ - [id] INT NOT NULL,\ - [parent_code] INT NULL,\ - CONSTRAINT [{inbound_foreign_key}] FOREIGN KEY ([parent_code])\ - REFERENCES [dbo].[{parent_table}] ([code])\ - ON UPDATE CASCADE ON DELETE SET NULL NOT FOR REPLICATION\ - );\ - CREATE NONCLUSTERED INDEX [{ordinary_index}]\ - ON [dbo].[{parent_table}] ([code] DESC, [id] ASC)\ - INCLUDE ([payload])\ - WHERE [code] IS NOT NULL\ - WITH (PAD_INDEX = ON, FILLFACTOR = 80, STATISTICS_NORECOMPUTE = ON,\ - ALLOW_ROW_LOCKS = OFF, ALLOW_PAGE_LOCKS = ON)\ - ON [PRIMARY];" - ); - crate::db::sqlserver::execute_batch(&mut client, &setup).await?; - - let target_detail = TableSchemaDetail { - name: parent_table.clone(), - columns: crate::db::sqlserver::get_columns(&mut client, "dbo", &parent_table).await?, - indexes: crate::db::sqlserver::list_indexes(&mut client, "dbo", &parent_table).await?, - foreign_keys: crate::db::sqlserver::list_foreign_keys(&mut client, "dbo", &parent_table).await?, - triggers: Vec::new(), - ddl: None, - }; - let mut source_detail = target_detail.clone(); - let source_id = source_detail - .columns - .iter_mut() - .find(|column| column.name == "id") - .ok_or_else(|| "live target metadata did not contain id".to_string())?; - source_id.data_type = "bigint".to_string(); - source_id.numeric_precision = Some(19); - let source_code = source_detail - .columns - .iter_mut() - .find(|column| column.name == "code") - .ok_or_else(|| "live target metadata did not contain code".to_string())?; - source_code.is_nullable = true; - - let prepared = prepare_schema_diff(SchemaDiffPreparationOptions { - source_tables: vec![table_info(&parent_table, "BASE TABLE")], - target_tables: vec![table_info(&parent_table, "BASE TABLE")], - source_details: vec![source_detail], - target_details: vec![target_detail], - database_type: DatabaseType::SqlServer, - target_schema: Some("dbo".to_string()), - source_dialect: Some(DialectKind::SqlServer), - target_dialect: Some(DialectKind::SqlServer), - ..Default::default() - }); - - crate::db::sqlserver::execute_batch(&mut client, "SET QUOTED_IDENTIFIER OFF;").await?; - crate::db::sqlserver::execute_batch(&mut client, &prepared.sync_sql).await?; - let verification_sql = format!( - "DECLARE @parent_id int = OBJECT_ID(N'[dbo].[{parent_table}]');\ - SELECT\ - CONVERT(int, CASE WHEN EXISTS (\ - SELECT 1\ - FROM sys.indexes AS idx\ - JOIN sys.stats AS stats\ - ON stats.object_id = idx.object_id AND stats.stats_id = idx.index_id\ - JOIN sys.data_spaces AS data_space ON data_space.data_space_id = idx.data_space_id\ - JOIN sys.index_columns AS key_ic\ - ON key_ic.object_id = idx.object_id AND key_ic.index_id = idx.index_id\ - JOIN sys.columns AS key_column\ - ON key_column.object_id = key_ic.object_id AND key_column.column_id = key_ic.column_id\ - WHERE idx.object_id = @parent_id\ - AND idx.name = N'{ordinary_index}'\ - AND idx.type_desc = N'NONCLUSTERED'\ - AND idx.is_primary_key = 0 AND idx.is_unique_constraint = 0\ - AND idx.is_disabled = 0 AND idx.has_filter = 1\ - AND idx.is_padded = 1 AND idx.fill_factor = 80\ - AND idx.allow_row_locks = 0 AND idx.allow_page_locks = 1\ - AND stats.no_recompute = 1 AND data_space.name = N'PRIMARY'\ - AND key_column.name = N'code' AND key_ic.key_ordinal = 1\ - AND key_ic.is_descending_key = 1\ - AND CHARINDEX(N'[code]', idx.filter_definition) > 0\ - AND EXISTS (\ - SELECT 1\ - FROM sys.index_columns AS include_ic\ - JOIN sys.columns AS include_column\ - ON include_column.object_id = include_ic.object_id\ - AND include_column.column_id = include_ic.column_id\ - WHERE include_ic.object_id = idx.object_id\ - AND include_ic.index_id = idx.index_id\ - AND include_ic.is_included_column = 1\ - AND include_column.name = N'payload'\ - )\ - ) THEN 1 ELSE 0 END) AS ordinary_index_ok,\ - CONVERT(int, CASE WHEN EXISTS (\ - SELECT 1\ - FROM sys.key_constraints AS key_constraint\ - JOIN sys.indexes AS idx\ - ON idx.object_id = key_constraint.parent_object_id\ - AND idx.index_id = key_constraint.unique_index_id\ - JOIN sys.index_columns AS ic\ - ON ic.object_id = idx.object_id AND ic.index_id = idx.index_id\ - JOIN sys.columns AS column_info\ - ON column_info.object_id = ic.object_id AND column_info.column_id = ic.column_id\ - JOIN sys.types AS column_type ON column_type.user_type_id = column_info.user_type_id\ - WHERE key_constraint.parent_object_id = @parent_id\ - AND key_constraint.name = N'{primary_key}'\ - AND key_constraint.type = N'PK'\ - AND idx.is_primary_key = 1 AND idx.type_desc = N'CLUSTERED'\ - AND ic.key_ordinal = 1 AND column_info.name = N'id'\ - AND column_type.name = N'bigint' AND column_info.is_nullable = 0\ - ) THEN 1 ELSE 0 END) AS primary_key_ok,\ - CONVERT(int, CASE WHEN EXISTS (\ - SELECT 1\ - FROM sys.check_constraints AS check_constraint\ - WHERE check_constraint.parent_object_id = @parent_id\ - AND check_constraint.name = N'{check_constraint}'\ - AND check_constraint.is_disabled = 0\ - AND check_constraint.is_not_trusted = 0\ - AND CHARINDEX(N'[id]', check_constraint.definition) > 0\ - AND CHARINDEX(N'[code]', check_constraint.definition) > 0\ - ) THEN 1 ELSE 0 END) AS check_constraint_ok,\ - CONVERT(int, CASE WHEN EXISTS (\ - SELECT 1\ - FROM sys.foreign_keys AS foreign_key\ - JOIN sys.foreign_key_columns AS fkc\ - ON fkc.constraint_object_id = foreign_key.object_id\ - JOIN sys.columns AS child_column\ - ON child_column.object_id = fkc.parent_object_id\ - AND child_column.column_id = fkc.parent_column_id\ - JOIN sys.columns AS parent_column\ - ON parent_column.object_id = fkc.referenced_object_id\ - AND parent_column.column_id = fkc.referenced_column_id\ - WHERE foreign_key.name = N'{inbound_foreign_key}'\ - AND foreign_key.parent_object_id = OBJECT_ID(N'[dbo].[{child_table}]')\ - AND foreign_key.referenced_object_id = @parent_id\ - AND foreign_key.type = N'F'\ - AND foreign_key.is_disabled = 0 AND foreign_key.is_not_trusted = 1\ - AND foreign_key.is_not_for_replication = 1\ - AND foreign_key.update_referential_action_desc = N'CASCADE'\ - AND foreign_key.delete_referential_action_desc = N'SET_NULL'\ - AND child_column.name = N'parent_code' AND parent_column.name = N'code'\ - AND EXISTS (\ - SELECT 1 FROM sys.key_constraints AS uq\ - WHERE uq.parent_object_id = @parent_id\ - AND uq.name = N'{unique_constraint}' AND uq.type = N'UQ'\ - )\ - ) THEN 1 ELSE 0 END) AS inbound_foreign_key_ok,\ - CONVERT(int, CASE WHEN\ - EXISTS (\ - SELECT 1 FROM sys.columns AS column_info\ - JOIN sys.types AS column_type ON column_type.user_type_id = column_info.user_type_id\ - WHERE column_info.object_id = @parent_id AND column_info.name = N'id'\ - AND column_type.name = N'bigint' AND column_info.is_nullable = 0\ - )\ - AND EXISTS (\ - SELECT 1 FROM sys.columns AS column_info\ - JOIN sys.types AS column_type ON column_type.user_type_id = column_info.user_type_id\ - WHERE column_info.object_id = @parent_id AND column_info.name = N'code'\ - AND column_type.name = N'int' AND column_info.is_nullable = 1\ - )\ - THEN 1 ELSE 0 END) AS altered_columns_ok;" - ); - let verification = crate::db::sqlserver::execute_query(&mut client, &verification_sql).await?; - Ok::<_, String>((prepared, verification)) - } - .await; - - let cleanup_result = crate::db::sqlserver::execute_batch(&mut client, &cleanup).await; - cleanup_result.expect("drop live SQL Server dependency test tables"); - let (prepared, verification) = exercise.expect("exercise live SQL Server dependency-aware ALTER COLUMN"); - - assert_eq!(prepared.diffs.len(), 1, "diffs={:?}", prepared.diffs); - let table_diff = &prepared.diffs[0]; - assert_eq!(table_diff.diff_type, "modified"); - assert_eq!(table_diff.columns.as_ref().map(Vec::len), Some(2)); - assert!(table_diff.indexes.is_none(), "unchanged indexes must be absent: {table_diff:?}"); - assert!(table_diff.foreign_keys.is_none(), "unchanged foreign keys must be absent: {table_diff:?}"); - assert!( - !prepared.sync_sql.contains(&ordinary_index), - "index must be discovered at runtime: {}", - prepared.sync_sql - ); - assert!( - !prepared.sync_sql.contains(&outbound_foreign_key), - "foreign keys must be discovered at runtime: {}", - prepared.sync_sql - ); - assert!( - !prepared.sync_sql.contains(&inbound_foreign_key), - "inbound foreign keys are not represented by the table diff: {}", - prepared.sync_sql - ); - - assert_eq!(verification.rows.len(), 1, "verification={verification:?}"); - assert_eq!( - verification.rows[0], - vec![ - serde_json::json!(1), - serde_json::json!(1), - serde_json::json!(1), - serde_json::json!(1), - serde_json::json!(1), - ], - "columns={:?}, sync_sql={} ", - verification.columns, - prepared.sync_sql - ); - } - #[test] fn sqlserver_index_and_foreign_key_forms_follow_tsql() { let btree = IndexInfo { @@ -8759,71 +8489,6 @@ mod tests { } } - #[tokio::test] - #[ignore = "requires DBX_TEST_POSTGRES_URL pointing at a writable PostgreSQL-family database"] - async fn real_postgres_round_trip_quotes_columns_and_leaves_expressions_bare() { - // PR #6312 review: exercise the full path end-to-end against a real PostgreSQL server - // instead of only asserting on generated text. Introspects a table whose unique index - // mixes a real column with an expression-hostile name ("order item", i.e. exactly the - // a.attname case the reviewer called out) and a genuine pg_get_indexdef expression key - // part, generates DDL with the same `create_index_sql` schema-diff sync uses, and - // executes that DDL back against the database to prove it's actually valid — not just - // plausible-looking text. - let url = std::env::var("DBX_TEST_POSTGRES_URL").expect("DBX_TEST_POSTGRES_URL"); - let pool = - crate::db::postgres::connect(&url, std::time::Duration::from_secs(5)).await.expect("connect postgres"); - let schema = format!("dbx_key_expr_{}", uuid::Uuid::new_v4().simple()); - crate::db::postgres::execute_query(&pool, &format!("CREATE SCHEMA {schema}")).await.expect("create schema"); - - let exercise = async { - crate::db::postgres::execute_query( - &pool, - &format!( - "CREATE TABLE {schema}.tankong_data (\ - \"order item\" integer, data_type text, data_time timestamp, height double precision)" - ), - ) - .await?; - crate::db::postgres::execute_query( - &pool, - &format!( - "CREATE UNIQUE INDEX uq_tankong_sta_type_time ON {schema}.tankong_data \ - (\"order item\", data_type, data_time, \ - (COALESCE(height, '-1'::integer::double precision)))" - ), - ) - .await?; - - let indexes = crate::db::postgres::list_indexes(&pool, &schema, "tankong_data").await?; - let index = indexes - .into_iter() - .find(|index| index.name == "uq_tankong_sta_type_time") - .ok_or_else(|| "introspection should return the created index".to_string())?; - - // Regenerate the index DDL through the exact same production function schema-diff - // sync calls, then execute it back against the real database to prove it's valid. - let ddl = create_index_sql("tankong_data", &index, DatabaseType::Highgo, Some(&schema)); - crate::db::postgres::execute_query(&pool, &format!("DROP INDEX {schema}.uq_tankong_sta_type_time")).await?; - crate::db::postgres::execute_query(&pool, &ddl).await?; - - Ok::<_, String>((index, ddl)) - } - .await; - - let cleanup = crate::db::postgres::execute_query(&pool, &format!("DROP SCHEMA {schema} CASCADE")).await; - cleanup.expect("drop schema"); - let (index, recreate_ddl) = exercise.expect("exercise real postgres round trip"); - - assert_eq!( - index.columns, - vec!["order item", "data_type", "data_time", "COALESCE(height, '-1'::integer::double precision)"] - ); - assert_eq!(index.key_is_expression, vec![false, false, false, true]); - assert!(recreate_ddl.contains("\"order item\"")); - assert!(recreate_ddl.contains("COALESCE(height, '-1'::integer::double precision)")); - assert!(!recreate_ddl.contains("\"COALESCE")); - } - #[test] fn detects_foreign_key_additions_removals_and_target_changes() { let diffs = diff_foreign_keys( diff --git a/crates/dbx-core/src/schema_diff/sqlserver_dependencies.rs b/crates/dbx-sql/src/schema_diff/sqlserver_dependencies.rs similarity index 100% rename from crates/dbx-core/src/schema_diff/sqlserver_dependencies.rs rename to crates/dbx-sql/src/schema_diff/sqlserver_dependencies.rs diff --git a/crates/dbx-core/src/sql.rs b/crates/dbx-sql/src/sql.rs similarity index 99% rename from crates/dbx-core/src/sql.rs rename to crates/dbx-sql/src/sql.rs index 923864d150..1654714b73 100644 --- a/crates/dbx-core/src/sql.rs +++ b/crates/dbx-sql/src/sql.rs @@ -60,7 +60,7 @@ pub struct SqlFileRequest { pub file_path: String, pub continue_on_error: bool, #[serde(default)] - pub selected_tables: Option>, + pub selected_tables: Option>, #[serde(default)] pub part_cooldown_ms: u64, /// Temporarily disable MySQL `FOREIGN_KEY_CHECKS` for this import and @@ -159,8 +159,9 @@ impl SqlDialectProfile { fn for_database_type(db_type: DatabaseType) -> Self { if matches!(db_type, DatabaseType::Mysql) { return Self { - preserves_tdsql_leading_directives: - crate::db::tdsql_mysql::preserves_leading_directives_for_database_type(db_type), + preserves_tdsql_leading_directives: crate::tdsql_mysql::preserves_leading_directives_for_database_type( + db_type, + ), ..Self::mysql_compatible() }; } @@ -383,7 +384,7 @@ pub struct SqlStatementSplitter { } #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct SqlStatementWithControl { +pub struct SqlStatementWithControl { pub sql: String, pub stop_on_error: bool, } @@ -397,7 +398,7 @@ impl SqlStatementSplitter { self.push_chunk_with_control(chunk).into_iter().map(|statement| statement.sql).collect() } - pub(crate) fn push_chunk_with_control(&mut self, chunk: &str) -> Vec { + pub fn push_chunk_with_control(&mut self, chunk: &str) -> Vec { let mut statements = Vec::new(); let chars = chunk.chars().collect::>(); let mut i = 0; @@ -642,7 +643,7 @@ impl SqlStatementSplitter { self.finish_with_control().into_iter().map(|statement| statement.sql).collect() } - pub(crate) fn finish_with_control(mut self) -> Vec { + pub fn finish_with_control(mut self) -> Vec { let mut statements = Vec::new(); if self.pending_mysql_line_comment_dashes { self.in_line_comment = true; @@ -2162,7 +2163,7 @@ fn is_mysql_compatible_import_profile(profile: &str) -> bool { ) } -pub(crate) fn supports_connection_level_database_bootstrap_target( +pub fn supports_connection_level_database_bootstrap_target( db_type: &DatabaseType, driver_profile: Option<&str>, ) -> bool { @@ -2172,7 +2173,7 @@ pub(crate) fn supports_connection_level_database_bootstrap_target( .is_some_and(|profile| is_mysql_compatible_import_profile(&profile) && profile != "manticoresearch") } -pub(crate) fn is_mysql_compatible_import_target(db_type: &DatabaseType, driver_profile: Option<&str>) -> bool { +pub fn is_mysql_compatible_import_target(db_type: &DatabaseType, driver_profile: Option<&str>) -> bool { matches!( db_type, DatabaseType::Mysql @@ -2789,7 +2790,7 @@ fn executable_sql_bounds(statement: &str, options: SqlParsingOptions) -> Option< } let executable_start = trimmed.len() - executable.len(); let start = if options.profile.preserves_tdsql_leading_directives { - crate::db::tdsql_mysql::leading_directive_start(trimmed, executable_start).unwrap_or(executable_start) + crate::tdsql_mysql::leading_directive_start(trimmed, executable_start).unwrap_or(executable_start) } else { executable_start }; diff --git a/crates/dbx-core/src/sql_analysis.rs b/crates/dbx-sql/src/sql_analysis.rs similarity index 100% rename from crates/dbx-core/src/sql_analysis.rs rename to crates/dbx-sql/src/sql_analysis.rs diff --git a/crates/dbx-core/src/sql_diagnostics.rs b/crates/dbx-sql/src/sql_diagnostics.rs similarity index 100% rename from crates/dbx-core/src/sql_diagnostics.rs rename to crates/dbx-sql/src/sql_diagnostics.rs diff --git a/crates/dbx-core/src/sql_dialect.rs b/crates/dbx-sql/src/sql_dialect.rs similarity index 98% rename from crates/dbx-core/src/sql_dialect.rs rename to crates/dbx-sql/src/sql_dialect.rs index 589faab5ba..c58a2ab598 100644 --- a/crates/dbx-core/src/sql_dialect.rs +++ b/crates/dbx-sql/src/sql_dialect.rs @@ -38,7 +38,7 @@ pub use descriptor::{ pub use identifiers::{ normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier, }; -pub(crate) use identifiers::{ +pub use identifiers::{ parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_iris_identifier, quote_transfer_identifier, transfer_column_identifier, }; @@ -46,7 +46,7 @@ pub use table_select::{ build_count_table_sql, build_table_data_select_sql, build_table_data_select_sql_with_database, build_table_select_sql, DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX, }; -pub(crate) use table_select::{ +pub use table_select::{ quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote, }; pub use type_rewrite::{ diff --git a/crates/dbx-core/src/sql_dialect/capabilities.rs b/crates/dbx-sql/src/sql_dialect/capabilities.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/capabilities.rs rename to crates/dbx-sql/src/sql_dialect/capabilities.rs diff --git a/crates/dbx-core/src/sql_dialect/ddl_profile.rs b/crates/dbx-sql/src/sql_dialect/ddl_profile.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/ddl_profile.rs rename to crates/dbx-sql/src/sql_dialect/ddl_profile.rs diff --git a/crates/dbx-core/src/sql_dialect/descriptor.rs b/crates/dbx-sql/src/sql_dialect/descriptor.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/descriptor.rs rename to crates/dbx-sql/src/sql_dialect/descriptor.rs diff --git a/crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs b/crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs rename to crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs diff --git a/crates/dbx-core/src/sql_dialect/dialect_loader.rs b/crates/dbx-sql/src/sql_dialect/dialect_loader.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/dialect_loader.rs rename to crates/dbx-sql/src/sql_dialect/dialect_loader.rs diff --git a/crates/dbx-core/src/sql_dialect/dialect_types.rs b/crates/dbx-sql/src/sql_dialect/dialect_types.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/dialect_types.rs rename to crates/dbx-sql/src/sql_dialect/dialect_types.rs diff --git a/crates/dbx-core/src/sql_dialect/dialect_yaml.rs b/crates/dbx-sql/src/sql_dialect/dialect_yaml.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/dialect_yaml.rs rename to crates/dbx-sql/src/sql_dialect/dialect_yaml.rs diff --git a/crates/dbx-core/src/sql_dialect/hot_reload.rs b/crates/dbx-sql/src/sql_dialect/hot_reload.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/hot_reload.rs rename to crates/dbx-sql/src/sql_dialect/hot_reload.rs diff --git a/crates/dbx-core/src/sql_dialect/identifiers.rs b/crates/dbx-sql/src/sql_dialect/identifiers.rs similarity index 97% rename from crates/dbx-core/src/sql_dialect/identifiers.rs rename to crates/dbx-sql/src/sql_dialect/identifiers.rs index 3277ea8a53..eefd4b7a52 100644 --- a/crates/dbx-core/src/sql_dialect/identifiers.rs +++ b/crates/dbx-sql/src/sql_dialect/identifiers.rs @@ -161,7 +161,7 @@ pub fn quote_table_identifier(database_type: Option, name: &str) - /// which is not valid in table references. Ordinary metadata names are /// case-insensitive and can be sent unquoted; names containing separators or /// other punctuation still need the identifier quote character. -pub(crate) fn quote_iris_identifier(name: &str, identifier_quote: Option<&str>) -> String { +pub fn quote_iris_identifier(name: &str, identifier_quote: Option<&str>) -> String { let mut chars = name.chars(); let ordinary = chars.next().is_some_and(|first| first == '_' || first == '%' || first.is_alphabetic()) && chars.all(|ch| ch == '_' || ch.is_alphanumeric()); @@ -172,7 +172,7 @@ pub(crate) fn quote_iris_identifier(name: &str, identifier_quote: Option<&str>) format!("{quote}{}{quote}", name.replace(quote, &format!("{quote}{quote}"))) } -pub(crate) fn quote_gaussdb_jdbc_identifier(name: &str, identifier_quote: &str) -> String { +pub fn quote_gaussdb_jdbc_identifier(name: &str, identifier_quote: &str) -> String { if is_explicitly_quoted_identifier(name) { return name.to_string(); } @@ -398,7 +398,7 @@ pub fn normalize_where_input(where_input: Option<&str>) -> String { } } -pub(crate) fn quote_transfer_identifier(name: &str, database_type: &DatabaseType) -> String { +pub fn quote_transfer_identifier(name: &str, database_type: &DatabaseType) -> String { match database_type { DatabaseType::Mysql | DatabaseType::ClickHouse @@ -415,11 +415,7 @@ pub(crate) fn quote_transfer_identifier(name: &str, database_type: &DatabaseType } } -pub(crate) fn transfer_column_identifier( - name: &str, - database_type: &DatabaseType, - quote_target_column_names: bool, -) -> String { +pub fn transfer_column_identifier(name: &str, database_type: &DatabaseType, quote_target_column_names: bool) -> String { if quote_target_column_names || !matches!(database_type, DatabaseType::Gaussdb | DatabaseType::OpenGauss) || !is_simple_unquoted_identifier(name) @@ -446,7 +442,7 @@ fn is_simple_unquoted_identifier(name: &str) -> bool { /// * With catalog AND the database type supports external catalogs (Doris/StarRocks): /// produces `catalog.schema.table` — the 3-part form those engines require to /// address objects in an external (non-internal) catalog. -pub(crate) fn qualified_transfer_table( +pub fn qualified_transfer_table( table_name: &str, schema: &str, database_type: &DatabaseType, diff --git a/crates/dbx-core/src/sql_dialect/inference.rs b/crates/dbx-sql/src/sql_dialect/inference.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/inference.rs rename to crates/dbx-sql/src/sql_dialect/inference.rs diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__all_dialects_info.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap similarity index 99% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__all_dialects_info.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap index 533535480c..cdb189ccd7 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__all_dialects_info.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__all_dialects_info.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 67 expression: all --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap similarity index 92% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap index f99e5beba9..cf1eb10fb6 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__duckdb_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 37 expression: desc --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_descriptor.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap similarity index 92% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_descriptor.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap index 40a95906cb..8ed1d1bab9 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_descriptor.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 7 expression: desc --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_info.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap similarity index 94% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_info.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap index bc23732d48..5650324e85 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_info.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_info.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 43 expression: info --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap similarity index 96% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap index 574907f791..6cf6254ca3 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__mysql_to_postgres_mapping.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 55 expression: "&matrix.rules" --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__oracle_descriptor.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap similarity index 92% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__oracle_descriptor.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap index e413bdd1ee..80c549bbad 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__oracle_descriptor.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__oracle_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 31 expression: desc --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_descriptor.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap similarity index 92% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_descriptor.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap index 7868298f7e..64762409ca 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_descriptor.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 13 expression: desc --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_info.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap similarity index 95% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_info.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap index 23fa9d4d7b..f97ed8c735 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_info.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_info.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 49 expression: info --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap similarity index 96% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap index 3aa4350ac8..d92ebe94dc 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__postgres_to_mysql_mapping.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 61 expression: "&matrix.rules" --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap similarity index 92% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap index 90aa9e45f1..ce3dda94b4 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlite_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 19 expression: desc --- diff --git a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap similarity index 92% rename from crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap rename to crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap index 8d4a2ab447..131899676a 100644 --- a/crates/dbx-core/src/sql_dialect/snapshots/dbx_core__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap +++ b/crates/dbx-sql/src/sql_dialect/snapshots/dbx_sql__sql_dialect__descriptor_snapshots__sqlserver_descriptor.snap @@ -1,5 +1,5 @@ --- -source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs +source: crates/dbx-sql/src/sql_dialect/descriptor_snapshots.rs assertion_line: 25 expression: desc --- diff --git a/crates/dbx-core/src/sql_dialect/table_select.rs b/crates/dbx-sql/src/sql_dialect/table_select.rs similarity index 99% rename from crates/dbx-core/src/sql_dialect/table_select.rs rename to crates/dbx-sql/src/sql_dialect/table_select.rs index 99a206fcf7..b1ddb6d8ae 100644 --- a/crates/dbx-core/src/sql_dialect/table_select.rs +++ b/crates/dbx-sql/src/sql_dialect/table_select.rs @@ -166,7 +166,7 @@ pub fn build_count_table_sql(database_type: Option, schema: Option format!("SELECT COUNT(*) AS row_count FROM {}", qualified_table_name(database_type, schema, table_name)) } -pub(crate) fn table_data_schema<'a>( +pub fn table_data_schema<'a>( database_type: Option, driver_profile: Option<&str>, schema: Option<&'a str>, @@ -457,7 +457,7 @@ fn database_qualified_table_name( } } -pub(crate) fn table_data_qualified_table_name( +pub fn table_data_qualified_table_name( database_type: Option, schema: Option<&str>, table_name: &str, @@ -482,7 +482,7 @@ pub(crate) fn table_data_qualified_table_name( .unwrap_or(table) } -pub(crate) fn quote_table_data_identifier( +pub fn quote_table_data_identifier( database_type: Option, name: &str, identifier_quote: Option<&str>, @@ -502,10 +502,7 @@ pub(crate) fn quote_table_data_identifier( format!("{quote}{}{quote}", name.replace(quote, &format!("{quote}{quote}"))) } -pub(crate) fn uses_connection_identifier_quote( - database_type: Option, - identifier_quote: Option<&str>, -) -> bool { +pub fn uses_connection_identifier_quote(database_type: Option, identifier_quote: Option<&str>) -> bool { database_type == Some(DatabaseType::Kingbase) // JDBC table-data requests carry the schema returned by DatabaseMetaData. // Keep the JDBC identifier unquoted when no driver quote was reported, but diff --git a/crates/dbx-core/src/sql_dialect/tests.rs b/crates/dbx-sql/src/sql_dialect/tests.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/tests.rs rename to crates/dbx-sql/src/sql_dialect/tests.rs diff --git a/crates/dbx-core/src/sql_dialect/type_rewrite.rs b/crates/dbx-sql/src/sql_dialect/type_rewrite.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/type_rewrite.rs rename to crates/dbx-sql/src/sql_dialect/type_rewrite.rs diff --git a/crates/dbx-core/src/sql_dialect/types.rs b/crates/dbx-sql/src/sql_dialect/types.rs similarity index 100% rename from crates/dbx-core/src/sql_dialect/types.rs rename to crates/dbx-sql/src/sql_dialect/types.rs diff --git a/crates/dbx-core/src/sql_editability.rs b/crates/dbx-sql/src/sql_editability.rs similarity index 100% rename from crates/dbx-core/src/sql_editability.rs rename to crates/dbx-sql/src/sql_editability.rs diff --git a/crates/dbx-core/src/sql_error_position.rs b/crates/dbx-sql/src/sql_error_position.rs similarity index 99% rename from crates/dbx-core/src/sql_error_position.rs rename to crates/dbx-sql/src/sql_error_position.rs index 242b9f0883..abbe15250f 100644 --- a/crates/dbx-core/src/sql_error_position.rs +++ b/crates/dbx-sql/src/sql_error_position.rs @@ -1,7 +1,7 @@ //! Driver-reported SQL error positions (currently PostgreSQL only). //! //! PostgreSQL reports a 1-based *character* cursor position for many SQL -//! errors ([`tokio_postgres::error::ErrorPosition::Original`]). The native +//! errors (`tokio_postgres::error::ErrorPosition::Original`). The native //! driver error path in `db::postgres` only exposes a `String` to the query //! layer, so the raw cursor is carried across that `String` ABI as a small //! machine-readable suffix and resolved back into a typed [`SqlErrorPosition`] diff --git a/crates/dbx-core/src/sql_parser/ast_filter.rs b/crates/dbx-sql/src/sql_parser/ast_filter.rs similarity index 100% rename from crates/dbx-core/src/sql_parser/ast_filter.rs rename to crates/dbx-sql/src/sql_parser/ast_filter.rs diff --git a/crates/dbx-core/src/sql_parser/git.rs b/crates/dbx-sql/src/sql_parser/git.rs similarity index 100% rename from crates/dbx-core/src/sql_parser/git.rs rename to crates/dbx-sql/src/sql_parser/git.rs diff --git a/crates/dbx-core/src/sql_parser/input.rs b/crates/dbx-sql/src/sql_parser/input.rs similarity index 100% rename from crates/dbx-core/src/sql_parser/input.rs rename to crates/dbx-sql/src/sql_parser/input.rs diff --git a/crates/dbx-core/src/sql_parser/meta.rs b/crates/dbx-sql/src/sql_parser/meta.rs similarity index 100% rename from crates/dbx-core/src/sql_parser/meta.rs rename to crates/dbx-sql/src/sql_parser/meta.rs diff --git a/crates/dbx-core/src/sql_parser/mod.rs b/crates/dbx-sql/src/sql_parser/mod.rs similarity index 100% rename from crates/dbx-core/src/sql_parser/mod.rs rename to crates/dbx-sql/src/sql_parser/mod.rs diff --git a/crates/dbx-core/src/sql_risk.rs b/crates/dbx-sql/src/sql_risk.rs similarity index 100% rename from crates/dbx-core/src/sql_risk.rs rename to crates/dbx-sql/src/sql_risk.rs diff --git a/crates/dbx-core/src/sqlserver_temporal.rs b/crates/dbx-sql/src/sqlserver_temporal.rs similarity index 100% rename from crates/dbx-core/src/sqlserver_temporal.rs rename to crates/dbx-sql/src/sqlserver_temporal.rs diff --git a/crates/dbx-core/src/table_structure_sql.rs b/crates/dbx-sql/src/table_structure_sql.rs similarity index 86% rename from crates/dbx-core/src/table_structure_sql.rs rename to crates/dbx-sql/src/table_structure_sql.rs index 6cbcb2f9c0..04dfab29ef 100644 --- a/crates/dbx-core/src/table_structure_sql.rs +++ b/crates/dbx-sql/src/table_structure_sql.rs @@ -8,7 +8,6 @@ mod foreign_keys; mod indexes; mod mysql_engine; mod owner; -mod sqlite_rebuild; mod triggers; mod types; mod util; @@ -20,14 +19,13 @@ mod tests; pub use column_alter::build_single_column_alter_sql; pub use create_table::build_create_table_sql; pub use owner::build_table_owner_change_sql; -pub use sqlite_rebuild::{apply_sqlite_table_structure_change, preview_sqlite_table_structure_change}; pub use types::*; -pub(crate) use column_alter::{ +pub use column_alter::{ build_sqlserver_alter_column_preserving_default_sql, build_sqlserver_drop_default_constraint_sql, }; -pub(crate) use comments::{build_sqlserver_column_comment_sql, build_sqlserver_table_comment_sql}; -pub(crate) use util::{oracle_new_object_reference, sqlserver_unicode_string_literal}; +pub use comments::{build_sqlserver_column_comment_sql, build_sqlserver_table_comment_sql}; +pub use util::{oracle_new_object_reference, sqlserver_unicode_string_literal}; use crate::models::connection::DatabaseType; @@ -38,7 +36,8 @@ use foreign_keys::build_foreign_key_sql; use indexes::build_index_sql; use mysql_engine::{build_mysql_engine_change_sql, validate_mysql_engine}; use triggers::build_trigger_sql; -use validation::{validate_concurrent_index_scope, validate_draft}; +use validation::validate_concurrent_index_scope; +pub use validation::validate_draft; pub fn build_table_structure_change_sql(mut options: TableStructureSqlOptions) -> TableStructureSqlResult { let mysql_engine_errors = validate_mysql_engine(&options); @@ -81,7 +80,7 @@ pub fn build_table_structure_change_sql(mut options: TableStructureSqlOptions) - /// for IRIS even though IRIS still reports descriptions on introspection. /// Callers using this as an introspection signal must corroborate it against /// what was actually collected rather than trust the flag alone. -pub(crate) fn supports_comments(database_type: crate::models::connection::DatabaseType) -> bool { +pub fn supports_comments(database_type: crate::models::connection::DatabaseType) -> bool { dialect::capabilities_for(Some(database_type), None).comment } @@ -93,7 +92,7 @@ pub(crate) fn supports_comments(database_type: crate::models::connection::Databa /// mismatch analogous to the IRIS comment case is possible for any future /// engine where DDL support and introspection support diverge — callers /// should corroborate against what was actually collected. -pub(crate) fn supports_foreign_keys(database_type: crate::models::connection::DatabaseType) -> bool { +pub fn supports_foreign_keys(database_type: crate::models::connection::DatabaseType) -> bool { dialect::capabilities_for(Some(database_type), None).foreign_key } @@ -104,6 +103,9 @@ pub(crate) fn supports_foreign_keys(database_type: crate::models::connection::Da /// `Debug` spelling (`Postgres`, `SqlServer`, `MongoDb`), which is an /// implementation detail, not something a consumer like dbdocs/dbdiagram /// should key off. -pub(crate) fn database_type_label(database_type: crate::models::connection::DatabaseType) -> String { +pub fn database_type_label(database_type: crate::models::connection::DatabaseType) -> String { dialect::database_label(Some(database_type)) } + +pub use indexes::has_existing_index_change; +pub use util::{clean, normalize_default, original_comment, original_default}; diff --git a/crates/dbx-core/src/table_structure_sql/column_alter.rs b/crates/dbx-sql/src/table_structure_sql/column_alter.rs similarity index 99% rename from crates/dbx-core/src/table_structure_sql/column_alter.rs rename to crates/dbx-sql/src/table_structure_sql/column_alter.rs index 5a8e363e44..61124bb80c 100644 --- a/crates/dbx-core/src/table_structure_sql/column_alter.rs +++ b/crates/dbx-sql/src/table_structure_sql/column_alter.rs @@ -792,7 +792,7 @@ pub(super) fn build_sqlserver_existing_column_sql( statements } -pub(crate) fn build_sqlserver_alter_column_preserving_default_sql( +pub fn build_sqlserver_alter_column_preserving_default_sql( table: &str, column_name: &str, data_type: &str, @@ -820,7 +820,7 @@ pub(crate) fn build_sqlserver_alter_column_preserving_default_sql( ) } -pub(crate) fn build_sqlserver_drop_default_constraint_sql(table: &str, column_name: &str) -> String { +pub fn build_sqlserver_drop_default_constraint_sql(table: &str, column_name: &str) -> String { let sql_var = sqlserver_default_constraint_sql_var(table, column_name); let table_literal = table.replace('\'', "''"); let column_literal = column_name.replace('\'', "''"); diff --git a/crates/dbx-core/src/table_structure_sql/column_format.rs b/crates/dbx-sql/src/table_structure_sql/column_format.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/column_format.rs rename to crates/dbx-sql/src/table_structure_sql/column_format.rs diff --git a/crates/dbx-core/src/table_structure_sql/columns.rs b/crates/dbx-sql/src/table_structure_sql/columns.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/columns.rs rename to crates/dbx-sql/src/table_structure_sql/columns.rs diff --git a/crates/dbx-core/src/table_structure_sql/comments.rs b/crates/dbx-sql/src/table_structure_sql/comments.rs similarity index 98% rename from crates/dbx-core/src/table_structure_sql/comments.rs rename to crates/dbx-sql/src/table_structure_sql/comments.rs index fd8ae018c5..728eaf0090 100644 --- a/crates/dbx-core/src/table_structure_sql/comments.rs +++ b/crates/dbx-sql/src/table_structure_sql/comments.rs @@ -73,7 +73,7 @@ fn build_sqlserver_extended_property_comment_sql( )] } -pub(crate) fn build_sqlserver_table_comment_sql( +pub fn build_sqlserver_table_comment_sql( qualified_table: &str, schema: Option<&str>, table_name: &str, @@ -82,7 +82,7 @@ pub(crate) fn build_sqlserver_table_comment_sql( build_sqlserver_table_comment_sql_for_profile(qualified_table, schema, table_name, new_comment, None) } -pub(crate) fn build_sqlserver_table_comment_sql_for_profile( +pub fn build_sqlserver_table_comment_sql_for_profile( qualified_table: &str, schema: Option<&str>, table_name: &str, @@ -156,7 +156,7 @@ pub(super) fn build_sqlserver_index_comment_sql_for_profile( build_sqlserver_extended_property_comment_sql(&exists, &levels, new_comment, procedure_prefix) } -pub(crate) fn build_sqlserver_column_comment_sql( +pub fn build_sqlserver_column_comment_sql( qualified_table: &str, schema: Option<&str>, table_name: &str, @@ -166,7 +166,7 @@ pub(crate) fn build_sqlserver_column_comment_sql( build_sqlserver_column_comment_sql_for_profile(qualified_table, schema, table_name, column_name, new_comment, None) } -pub(crate) fn build_sqlserver_column_comment_sql_for_profile( +pub fn build_sqlserver_column_comment_sql_for_profile( qualified_table: &str, schema: Option<&str>, table_name: &str, diff --git a/crates/dbx-core/src/table_structure_sql/create_table.rs b/crates/dbx-sql/src/table_structure_sql/create_table.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/create_table.rs rename to crates/dbx-sql/src/table_structure_sql/create_table.rs diff --git a/crates/dbx-core/src/table_structure_sql/dialect.rs b/crates/dbx-sql/src/table_structure_sql/dialect.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/dialect.rs rename to crates/dbx-sql/src/table_structure_sql/dialect.rs diff --git a/crates/dbx-core/src/table_structure_sql/foreign_keys.rs b/crates/dbx-sql/src/table_structure_sql/foreign_keys.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/foreign_keys.rs rename to crates/dbx-sql/src/table_structure_sql/foreign_keys.rs diff --git a/crates/dbx-core/src/table_structure_sql/indexes.rs b/crates/dbx-sql/src/table_structure_sql/indexes.rs similarity index 99% rename from crates/dbx-core/src/table_structure_sql/indexes.rs rename to crates/dbx-sql/src/table_structure_sql/indexes.rs index 4bcfe458bb..9ab1796a1e 100644 --- a/crates/dbx-core/src/table_structure_sql/indexes.rs +++ b/crates/dbx-sql/src/table_structure_sql/indexes.rs @@ -193,7 +193,7 @@ fn build_dameng_constraint_index_edit( statements } -pub(super) fn has_existing_index_change(index: &EditableStructureIndex) -> bool { +pub fn has_existing_index_change(index: &EditableStructureIndex) -> bool { let Some(original) = &index.original else { return false; }; diff --git a/crates/dbx-core/src/table_structure_sql/mysql_engine.rs b/crates/dbx-sql/src/table_structure_sql/mysql_engine.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/mysql_engine.rs rename to crates/dbx-sql/src/table_structure_sql/mysql_engine.rs diff --git a/crates/dbx-core/src/table_structure_sql/owner.rs b/crates/dbx-sql/src/table_structure_sql/owner.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/owner.rs rename to crates/dbx-sql/src/table_structure_sql/owner.rs diff --git a/crates/dbx-core/src/table_structure_sql/tests.rs b/crates/dbx-sql/src/table_structure_sql/tests.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/tests.rs rename to crates/dbx-sql/src/table_structure_sql/tests.rs diff --git a/crates/dbx-core/src/table_structure_sql/triggers.rs b/crates/dbx-sql/src/table_structure_sql/triggers.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/triggers.rs rename to crates/dbx-sql/src/table_structure_sql/triggers.rs diff --git a/crates/dbx-core/src/table_structure_sql/types.rs b/crates/dbx-sql/src/table_structure_sql/types.rs similarity index 100% rename from crates/dbx-core/src/table_structure_sql/types.rs rename to crates/dbx-sql/src/table_structure_sql/types.rs diff --git a/crates/dbx-core/src/table_structure_sql/util.rs b/crates/dbx-sql/src/table_structure_sql/util.rs similarity index 97% rename from crates/dbx-core/src/table_structure_sql/util.rs rename to crates/dbx-sql/src/table_structure_sql/util.rs index ed038e3779..83c3262203 100644 --- a/crates/dbx-core/src/table_structure_sql/util.rs +++ b/crates/dbx-sql/src/table_structure_sql/util.rs @@ -89,7 +89,7 @@ pub(super) fn quote_new_ident(database_type: Option, dialect: Stru /// later reference with the same unquoted spelling folds to the same object. Names that had to /// stay quoted at creation (special characters, reserved words) keep exact quoting here too, so /// generated references always line up with the created object. -pub(crate) fn oracle_new_object_reference(name: &str) -> String { +pub fn oracle_new_object_reference(name: &str) -> String { if is_simple_oracle_identifier(name) && !is_oracle_reserved_identifier(name) { name.to_string() } else { @@ -136,7 +136,7 @@ fn is_sql_string_literal(value: &str) -> bool { true } -pub(crate) fn sqlserver_unicode_string_literal(value: &str) -> Option { +pub fn sqlserver_unicode_string_literal(value: &str) -> Option { let mut candidate = value.trim(); let mut parenthesis_depth = 0usize; @@ -207,7 +207,7 @@ fn is_postgres_textual_cast_type(value: &str) -> bool { ) } -pub(super) fn clean(value: &str) -> String { +pub fn clean(value: &str) -> String { value.trim().to_string() } @@ -424,7 +424,7 @@ fn mysql_temporal_default_with_column_precision( } } -pub(super) fn normalize_default(value: Option<&String>) -> String { +pub fn normalize_default(value: Option<&String>) -> String { let trimmed = value.map(|value| value.trim()).unwrap_or(""); if trimmed.eq_ignore_ascii_case("null") { String::new() @@ -435,11 +435,11 @@ pub(super) fn normalize_default(value: Option<&String>) -> String { } } -pub(super) fn original_default(column: &EditableStructureColumn) -> String { +pub fn original_default(column: &EditableStructureColumn) -> String { normalize_default(column.original.as_ref().and_then(|original| original.column_default.as_ref())) } -pub(super) fn original_comment(column: &EditableStructureColumn) -> String { +pub fn original_comment(column: &EditableStructureColumn) -> String { clean(column.original.as_ref().and_then(|original| original.comment.as_deref()).unwrap_or("")) } diff --git a/crates/dbx-core/src/table_structure_sql/validation.rs b/crates/dbx-sql/src/table_structure_sql/validation.rs similarity index 98% rename from crates/dbx-core/src/table_structure_sql/validation.rs rename to crates/dbx-sql/src/table_structure_sql/validation.rs index 11008a22fd..a63d7eded6 100644 --- a/crates/dbx-core/src/table_structure_sql/validation.rs +++ b/crates/dbx-sql/src/table_structure_sql/validation.rs @@ -38,7 +38,7 @@ pub(super) fn validate_concurrent_index_scope(options: &TableStructureSqlOptions errors } -pub(super) fn validate_draft(options: &TableStructureSqlOptions) -> Vec { +pub fn validate_draft(options: &TableStructureSqlOptions) -> Vec { let mut warnings = Vec::new(); let active_columns: Vec<_> = options.columns.iter().filter(|column| !column.marked_for_drop).collect(); validate_columns(&active_columns, &mut warnings); diff --git a/crates/dbx-core/src/db/tdsql_mysql.rs b/crates/dbx-sql/src/tdsql_mysql.rs similarity index 92% rename from crates/dbx-core/src/db/tdsql_mysql.rs rename to crates/dbx-sql/src/tdsql_mysql.rs index 90ab5cf7c7..13e63856f4 100644 --- a/crates/dbx-core/src/db/tdsql_mysql.rs +++ b/crates/dbx-sql/src/tdsql_mysql.rs @@ -11,11 +11,11 @@ pub fn is_profile(db_type: &DatabaseType, driver_profile: Option<&str>) -> bool && driver_profile.is_some_and(|profile| profile.eq_ignore_ascii_case(DRIVER_PROFILE)) } -pub(crate) fn preserves_leading_directives_for_database_type(db_type: DatabaseType) -> bool { +pub fn preserves_leading_directives_for_database_type(db_type: DatabaseType) -> bool { db_type == DatabaseType::Mysql } -pub(crate) fn leading_directive_start(statement: &str, executable_start: usize) -> Option { +pub fn leading_directive_start(statement: &str, executable_start: usize) -> Option { let prefix = statement.get(..executable_start)?; let line_start = prefix.rfind(['\r', '\n']).map_or(0, |index| index + 1); let line_prefix = &prefix[line_start..]; diff --git a/crates/dbx-sql/src/value_literals.rs b/crates/dbx-sql/src/value_literals.rs new file mode 100644 index 0000000000..829283c872 --- /dev/null +++ b/crates/dbx-sql/src/value_literals.rs @@ -0,0 +1,150 @@ +pub fn format_pg_array_sql_literal(arr: &[serde_json::Value]) -> String { + if arr.is_empty() { + return "'{}'".to_string(); + } + let elements: Vec = arr.iter().map(format_pg_array_element).collect(); + let inner = format!("{{{}}}", elements.join(",")); + format!("'{}'", inner.replace('\\', "\\\\").replace('\'', "''")) +} + +pub fn format_pg_array_element(val: &serde_json::Value) -> String { + match val { + serde_json::Value::Null => "NULL".to_string(), + serde_json::Value::Array(arr) => { + if arr.is_empty() { + return "{}".to_string(); + } + let elements: Vec = arr.iter().map(format_pg_array_element).collect(); + format!("{{{}}}", elements.join(",")) + } + serde_json::Value::String(s) => { + let escaped = s.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{}\"", escaped) + } + serde_json::Value::Number(n) => n.to_string(), + serde_json::Value::Bool(b) => { + if *b { + "true".to_string() + } else { + "false".to_string() + } + } + serde_json::Value::Object(o) => { + let json = serde_json::to_string(o).unwrap_or_default(); + let escaped = json.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{}\"", escaped) + } + } +} + +pub fn format_ch_array_sql_literal(arr: &[serde_json::Value]) -> String { + if arr.is_empty() { + return "[]".to_string(); + } + let elements: Vec = arr.iter().map(format_ch_array_element).collect(); + format!("[{}]", elements.join(",")) +} + +pub fn format_ch_array_element(val: &serde_json::Value) -> String { + match val { + serde_json::Value::Null => "NULL".to_string(), + serde_json::Value::Array(arr) => { + if arr.is_empty() { + return "[]".to_string(); + } + let elements: Vec = arr.iter().map(format_ch_array_element).collect(); + format!("[{}]", elements.join(",")) + } + serde_json::Value::String(s) => { + let escaped = s.replace('\\', "\\\\").replace('\'', "''"); + format!("'{}'", escaped) + } + serde_json::Value::Number(n) => n.to_string(), + serde_json::Value::Bool(b) => { + if *b { + "true".to_string() + } else { + "false".to_string() + } + } + serde_json::Value::Object(o) => { + let json = serde_json::to_string(o).unwrap_or_default(); + format!("'{}'", json.replace('\\', "\\\\").replace('\'', "''")) + } + } +} + +pub fn quote_postgres_string_literal(value: &str) -> String { + if !value.contains('\\') && !value.chars().any(|character| character.is_ascii_control()) { + return quote_string_literal(value); + } + + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\\' => escaped.push_str("\\\\"), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + '\x08' => escaped.push_str("\\b"), + '\x0c' => escaped.push_str("\\f"), + '\'' => escaped.push_str("''"), + character if character.is_ascii_control() => { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let byte = character as u8; + escaped.push_str("\\x"); + escaped.push(HEX[(byte >> 4) as usize] as char); + escaped.push(HEX[(byte & 0x0F) as usize] as char); + } + character => escaped.push(character), + } + } + + // Escape string constants keep control characters out of the physical + // script and remain correct regardless of standard_conforming_strings. + format!("E'{escaped}'") +} + +pub fn is_postgres_vector_type(column_type: Option<&str>) -> bool { + column_type + .map(|column_type| { + let normalized = column_type.trim().trim_matches('"').to_ascii_lowercase(); + if normalized.trim_end().ends_with("[]") { + return false; + } + let base = normalized.split(['(', ' ', '\t', '\n']).next().unwrap_or("").trim_matches('"'); + matches!(base, "vector" | "halfvec") || base.ends_with(".vector") || base.ends_with(".halfvec") + }) + .unwrap_or(false) +} + +pub fn format_postgres_vector_sql_literal(value: &serde_json::Value) -> String { + if value.is_null() { + return "NULL".to_string(); + } + let text = match value { + // pgvector vector/halfvec are scalar extension types whose importable + // literal grammar uses square brackets, unlike PostgreSQL arrays. + serde_json::Value::Array(arr) => { + let elements = arr.iter().map(format_postgres_vector_element).collect::>(); + format!("[{}]", elements.join(",")) + } + serde_json::Value::String(text) => text.to_string(), + _ => value.to_string(), + }; + quote_postgres_string_literal(&text) +} + +pub fn format_postgres_vector_element(value: &serde_json::Value) -> String { + match value { + serde_json::Value::String(text) => text.trim().to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::Bool(value) => value.to_string(), + serde_json::Value::Null => "NULL".to_string(), + _ => value.to_string(), + } +} + +pub fn quote_string_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} diff --git a/crates/dbx-types/Cargo.toml b/crates/dbx-types/Cargo.toml new file mode 100644 index 0000000000..fca2128647 --- /dev/null +++ b/crates/dbx-types/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "dbx-types" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[features] +default = [] +mq-admin = ["dep:uuid"] +openapi = ["dep:utoipa"] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] } +percent-encoding = "2" +utoipa = { version = "5.5", optional = true } +uuid = { version = "1", features = ["v4"], optional = true } + +[build-dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml_ng = "0.10" diff --git a/crates/dbx-core/build.rs b/crates/dbx-types/build.rs similarity index 64% rename from crates/dbx-core/build.rs rename to crates/dbx-types/build.rs index e0913518da..ac9c65d04e 100644 --- a/crates/dbx-core/build.rs +++ b/crates/dbx-types/build.rs @@ -1,9 +1,8 @@ +use serde::Deserialize; use std::collections::HashSet; use std::env; use std::path::{Path, PathBuf}; -use serde::Deserialize; - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct DatabaseDescriptor { @@ -14,14 +13,9 @@ struct DatabaseDescriptor { } fn main() { - let cargo_manifest_dir_str = env::var("CARGO_MANIFEST_DIR").unwrap(); - let cargo_manifest_dir = Path::new(&cargo_manifest_dir_str); - let plugins_dir = cargo_manifest_dir.join("..").join("..").join("plugins"); - let out_dir_str = env::var("OUT_DIR").unwrap(); - let out_dir = Path::new(&out_dir_str); - - generate_core_dialects(&plugins_dir.join("dialects"), out_dir); - generate_connection_type_registry(&plugins_dir.join("connection-types"), out_dir); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + generate_connection_type_registry(&manifest_dir.join("../../plugins/connection-types"), &out_dir); } fn yaml_entries(directory: &Path) -> Vec { @@ -35,43 +29,6 @@ fn yaml_entries(directory: &Path) -> Vec { entries } -fn generate_core_dialects(dialects_dir: &Path, out_dir: &Path) { - let dialects_dir = std::fs::canonicalize(dialects_dir).unwrap_or_else(|_| dialects_dir.to_path_buf()); - let dest_path = out_dir.join("core_dialects.rs"); - let entries = yaml_entries(&dialects_dir); - - // Watch the directory itself so additions/removals of dialect files trigger a rebuild. - println!("cargo::rerun-if-changed={}", dialects_dir.to_str().unwrap()); - - let mut code = String::from("{\n"); - - for path in &entries { - let canonical = std::fs::canonicalize(path).unwrap_or_else(|_| path.clone()); - let file_name = path.file_stem().unwrap().to_str().unwrap(); - let path_str = canonical.to_str().unwrap(); - - // Watch each dialect file individually. Editing a single YAML must invalidate - // the embedded `core_dialects.rs`, otherwise the compiled binary keeps a stale - // type catalog (e.g. old type names) and silently misbehaves (see field mapping). - println!("cargo::rerun-if-changed={}", path_str); - - code.push_str("match crate::sql_dialect::dialect_loader::DialectPluginLoader::load_from_string(\n"); - code.push_str(&format!(" include_str!(\"{}\"),\n", path_str.replace('\\', "\\\\"))); - code.push_str(" None,\n"); - code.push_str(") {\n"); - code.push_str(" Ok((_kind, yaml, descriptor)) => {\n"); - code.push_str(" let name = yaml.dialect.name.clone();\n"); - code.push_str(" registry.register_descriptor(&name, descriptor, yaml);\n"); - code.push_str(" }\n"); - code.push_str(&format!(" Err(e) => log::warn!(\"Failed to load core dialect '{}': {{e}}\"),\n", file_name)); - code.push_str("};\n"); - } - - code.push_str("}\n"); - - std::fs::write(&dest_path, code).expect("Failed to write core_dialects.rs"); -} - fn generate_connection_type_registry(connection_types_dir: &Path, out_dir: &Path) { let connection_types_dir = std::fs::canonicalize(connection_types_dir).unwrap_or_else(|_| connection_types_dir.to_path_buf()); diff --git a/crates/dbx-core/src/database_manifest.rs b/crates/dbx-types/src/database_manifest.rs similarity index 100% rename from crates/dbx-core/src/database_manifest.rs rename to crates/dbx-types/src/database_manifest.rs diff --git a/crates/dbx-types/src/document.rs b/crates/dbx-types/src/document.rs new file mode 100644 index 0000000000..aa687e0348 --- /dev/null +++ b/crates/dbx-types/src/document.rs @@ -0,0 +1,21 @@ +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MongoGridFsFileInfo { + pub id: String, + pub filename: Option, + pub length: i64, + pub chunk_size: i32, + pub upload_date: Option, + pub metadata: Option, + pub md5: Option, + pub content_type: Option, + pub aliases: Option>, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MongoGridFsBucketInfo { + pub name: String, + pub file_count: u64, + pub total_bytes: i64, +} diff --git a/crates/dbx-types/src/json.rs b/crates/dbx-types/src/json.rs new file mode 100644 index 0000000000..cc703a4025 --- /dev/null +++ b/crates/dbx-types/src/json.rs @@ -0,0 +1,91 @@ +pub const JS_MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991; + +pub fn safe_i64_to_json(v: i64) -> serde_json::Value { + if !(-JS_MAX_SAFE_INTEGER..=JS_MAX_SAFE_INTEGER).contains(&v) { + serde_json::Value::String(v.to_string()) + } else { + serde_json::Value::Number(v.into()) + } +} + +pub fn safe_u64_to_json(v: u64) -> serde_json::Value { + if v > JS_MAX_SAFE_INTEGER as u64 { + serde_json::Value::String(v.to_string()) + } else { + serde_json::Value::Number(v.into()) + } +} + +pub fn json_value_for_js(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Number(number) => { + if let Some(value) = number.as_i64() { + safe_i64_to_json(value) + } else if let Some(value) = number.as_u64() { + safe_u64_to_json(value) + } else { + let value = number.to_string(); + if value.bytes().all(|byte| byte == b'-' || byte.is_ascii_digit()) { + serde_json::Value::String(value) + } else { + serde_json::Value::Number(number) + } + } + } + serde_json::Value::Array(values) => { + serde_json::Value::Array(values.into_iter().map(json_value_for_js).collect()) + } + serde_json::Value::Object(entries) => { + serde_json::Value::Object(entries.into_iter().map(|(key, value)| (key, json_value_for_js(value))).collect()) + } + value => value, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arbitrary_precision_integers_are_strings_for_js() { + let value = serde_json::from_str( + r#"{ + "positive": 20240302001417986771, + "negative": -20240302001417986771 + }"#, + ) + .unwrap(); + + assert_eq!( + json_value_for_js(value), + serde_json::json!({ + "positive": "20240302001417986771", + "negative": "-20240302001417986771" + }) + ); + } + + #[test] + fn json_value_for_js_preserves_existing_value_behavior_recursively() { + let value = serde_json::from_str( + r#"{ + "safe": [9007199254740991, -9007199254740991], + "unsafe": [9007199254740992, -9007199254740992], + "non_integer": [1.25, 1e-2], + "other": [null, true, "text"] + }"#, + ) + .unwrap(); + let expected: serde_json::Value = serde_json::from_str( + r#"{ + "safe": [9007199254740991, -9007199254740991], + "unsafe": ["9007199254740992", "-9007199254740992"], + "non_integer": [1.25, 1e-2], + "other": [null, true, "text"] + }"#, + ) + .unwrap(); + + assert_eq!(json_value_for_js(value), expected); + } +} diff --git a/crates/dbx-types/src/lib.rs b/crates/dbx-types/src/lib.rs new file mode 100644 index 0000000000..d5326c21b1 --- /dev/null +++ b/crates/dbx-types/src/lib.rs @@ -0,0 +1,10 @@ +pub mod database_manifest; +pub mod document; +pub mod json; +pub mod metadata_filter; +pub mod models; +#[cfg(feature = "mq-admin")] +pub mod mqtt; +pub mod mysql_event; +pub mod sql_file; +pub mod types; diff --git a/crates/dbx-types/src/metadata_filter.rs b/crates/dbx-types/src/metadata_filter.rs new file mode 100644 index 0000000000..e2663919ce --- /dev/null +++ b/crates/dbx-types/src/metadata_filter.rs @@ -0,0 +1,74 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TableNameFilter { + pub include_patterns: Vec, + pub exclude_patterns: Vec, +} + +impl TableNameFilter { + pub fn is_empty(&self) -> bool { + self.include_patterns.iter().all(|pattern| pattern.trim().is_empty()) + && self.exclude_patterns.iter().all(|pattern| pattern.trim().is_empty()) + } +} + +pub fn sql_like_pattern_matches_case_insensitive(pattern: &str, value: &str) -> bool { + #[derive(Clone, Copy)] + enum LikeToken { + Any, + One, + Literal(char), + } + + let mut tokens = Vec::new(); + let pattern = pattern.trim().to_lowercase(); + let mut pattern_chars = pattern.chars().peekable(); + while let Some(ch) = pattern_chars.next() { + match ch { + '%' => tokens.push(LikeToken::Any), + '_' => tokens.push(LikeToken::One), + '\\' => tokens.push(LikeToken::Literal(pattern_chars.next().unwrap_or('\\'))), + literal => tokens.push(LikeToken::Literal(literal)), + } + } + let value_chars: Vec = value.to_lowercase().chars().collect(); + + let mut previous = vec![false; value_chars.len() + 1]; + previous[0] = true; + for token in tokens { + let mut current = vec![false; value_chars.len() + 1]; + match token { + LikeToken::Any => { + current[0] = previous[0]; + for value_index in 1..=value_chars.len() { + current[value_index] = previous[value_index] || current[value_index - 1]; + } + } + LikeToken::One => { + current[1..].copy_from_slice(&previous[..value_chars.len()]); + } + LikeToken::Literal(literal) => { + for value_index in 1..=value_chars.len() { + current[value_index] = previous[value_index - 1] && value_chars[value_index - 1] == literal; + } + } + } + previous = current; + } + previous[value_chars.len()] +} + +pub fn table_name_filter_matches(name: &str, filter: Option<&TableNameFilter>) -> bool { + let Some(filter) = filter.filter(|filter| !filter.is_empty()) else { + return true; + }; + let include_patterns: Vec<&str> = + filter.include_patterns.iter().map(|pattern| pattern.trim()).filter(|pattern| !pattern.is_empty()).collect(); + let exclude_patterns: Vec<&str> = + filter.exclude_patterns.iter().map(|pattern| pattern.trim()).filter(|pattern| !pattern.is_empty()).collect(); + let included = include_patterns.is_empty() + || include_patterns.iter().any(|pattern| sql_like_pattern_matches_case_insensitive(pattern, name)); + included && !exclude_patterns.iter().any(|pattern| sql_like_pattern_matches_case_insensitive(pattern, name)) +} diff --git a/crates/dbx-core/src/models/connection.rs b/crates/dbx-types/src/models/connection.rs similarity index 96% rename from crates/dbx-core/src/models/connection.rs rename to crates/dbx-types/src/models/connection.rs index acccc7282c..bb23e90613 100644 --- a/crates/dbx-core/src/models/connection.rs +++ b/crates/dbx-types/src/models/connection.rs @@ -1562,7 +1562,7 @@ impl ConnectionConfig { fn mqtt_broker_url(&self) -> String { #[cfg(feature = "mq-admin")] { - use crate::mqtt::types::MqttConnectionConfig; + use crate::mqtt::MqttConnectionConfig; if let Ok(config) = MqttConnectionConfig::from_connection(self) { return config.broker_url(); } @@ -1615,7 +1615,7 @@ impl ConnectionConfig { } } - pub(crate) fn validate_native_url_params(&self) -> Result<(), String> { + pub fn validate_native_url_params(&self) -> Result<(), String> { if matches!(self.db_type, DatabaseType::Postgres | DatabaseType::Redshift) { validate_postgres_url_params(self.url_params.as_deref().unwrap_or("")) } else { @@ -1796,21 +1796,21 @@ fn mysql_url_params_require_tls(params: Option<&str>) -> bool { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum MysqlJdbcTlsMode { +pub enum MysqlJdbcTlsMode { Disabled, Preferred, Required, VerifyCa, } -pub(crate) fn is_mysql_jdbc_tls_param(key: &str) -> bool { +pub fn is_mysql_jdbc_tls_param(key: &str) -> bool { matches!( percent_decode_str(key).decode_utf8_lossy().trim().to_ascii_lowercase().as_str(), "usessl" | "requiressl" | "verifyservercertificate" ) } -pub(crate) fn mysql_jdbc_tls_mode(params: Option<&str>) -> Option { +pub fn mysql_jdbc_tls_mode(params: Option<&str>) -> Option { let mut has_jdbc_tls_param = false; let mut use_ssl = None; let mut require_ssl = None; @@ -2666,7 +2666,6 @@ mod tests { without_sqlserver_legacy_compatibility_param, ConnectionConfig, ConnectionTestResult, DatabaseConnectionInfo, DatabaseType, IdentifierCase, ProxyTunnelConfig, ProxyType, TransportLayerConfig, }; - use std::str::FromStr; #[test] fn default_query_timeout_is_sixty_seconds() { @@ -3332,21 +3331,6 @@ mod tests { assert_eq!(config.connection_url(), "mysql://user%40tenant%23cluster:secret@10.1.2.3:2883"); } - #[test] - fn doris_database_type_enables_cleartext_password_auth_for_direct_and_tunneled_urls() { - let mut config = mysql_config("root", "secret", Some("analytics")); - config.db_type = DatabaseType::Doris; - config.port = 9030; - - let direct_url = config.connection_url(); - assert_eq!(direct_url, "mysql://root:secret@10.1.2.3:9030/analytics?enable_cleartext_plugin=true"); - assert!(mysql_async::Opts::from_url(&direct_url).unwrap().enable_cleartext_plugin()); - assert_eq!( - config.connection_url_with_host("127.0.0.1", 19030), - "mysql://root:secret@127.0.0.1:19030/analytics?enable_cleartext_plugin=true" - ); - } - #[test] fn doris_family_database_types_and_profiles_enable_cleartext_password_auth() { for profile in ["doris", "selectdb", "starrocks", "manticoresearch"] { @@ -3837,78 +3821,6 @@ mod tests { assert_eq!(config.connection_url(), "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=require"); } - #[test] - fn postgres_url_normalizes_timezone_param_into_options() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("sslmode=require&timezone=Asia/Shanghai".to_string()); - - assert_eq!( - config.connection_url(), - "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=require&options=%2Dc%20TimeZone%3DAsia%2FShanghai" - ); - let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - assert_eq!(pg_config.get_options(), Some("-c TimeZone=Asia/Shanghai")); - } - - #[test] - fn postgres_url_maps_schema_param_into_search_path_options() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("schema=public".to_string()); - - assert_eq!( - config.connection_url(), - "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dpublic" - ); - let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - assert_eq!(pg_config.get_options(), Some("-c search_path=public")); - } - - #[test] - fn postgres_url_maps_current_schema_param_into_search_path_options() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("currentSchema=app".to_string()); - - assert_eq!( - config.connection_url(), - "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dapp" - ); - let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - assert_eq!(pg_config.get_options(), Some("-c search_path=app")); - } - - #[test] - fn postgres_url_ignores_jdbc_stringtype_param() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("currentSchema=public&stringtype=unspecified".to_string()); - - assert_eq!(config.validate_native_url_params(), Ok(())); - assert_eq!( - config.connection_url(), - "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dpublic" - ); - let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - assert_eq!(pg_config.get_options(), Some("-c search_path=public")); - } - - #[test] - fn postgres_url_accepts_encoded_jdbc_varchar_stringtype_param() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("currentSchema=app&%73tringtype=%76aRcHaR".to_string()); - - assert_eq!(config.validate_native_url_params(), Ok(())); - assert_eq!( - config.connection_url(), - "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer&options=%2Dc%20search%5Fpath%3Dapp" - ); - let pg_config = tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - assert_eq!(pg_config.get_options(), Some("-c search_path=app")); - } - #[test] fn postgres_url_rejects_invalid_or_empty_jdbc_stringtype_param() { let mut config = mysql_config("postgres", "secret", Some("test")); @@ -3927,43 +3839,6 @@ mod tests { ); } - #[test] - fn postgres_url_uses_only_the_structured_endpoint() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some( - "HOST=origin.example.com&%68ostaddr=203.0.113.10&%70ort=6432¤tSchema=app&application_name=dbx" - .to_string(), - ); - - let url = config.connection_url_with_host("127.0.0.1", 6543); - - assert_eq!( - url, - "postgres://postgres:secret@127.0.0.1:6543/test?sslmode=prefer&application_name=dbx&options=%2Dc%20search%5Fpath%3Dapp" - ); - let pg_config = tokio_postgres::Config::from_str(&url).unwrap(); - assert_eq!(pg_config.get_hosts().len(), 1); - assert_eq!(pg_config.get_ports(), &[6543]); - assert!(pg_config.get_hostaddrs().is_empty()); - assert_eq!(pg_config.get_options(), Some("-c search_path=app")); - } - - #[test] - fn postgres_url_query_password_keeps_priority_but_is_redacted() { - let mut config = mysql_config("postgres", "field-secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("%70assword=query-secret&application_name=dbx".to_string()); - - let url = config.connection_url(); - let pg_config = tokio_postgres::Config::from_str(&url).unwrap(); - assert_eq!(pg_config.get_password(), Some(b"query-secret".as_slice())); - assert_eq!( - config.redacted_connection_url(), - "postgres://10.1.2.3:2883/test?sslmode=prefer&application_name=dbx" - ); - } - #[test] fn postgres_url_redacts_case_insensitive_and_encoded_password_keys() { let mut config = mysql_config("postgres", "field-secret", Some("test")); @@ -3977,26 +3852,6 @@ mod tests { assert!(!url.contains("encoded-secret")); } - #[test] - fn postgres_url_ignores_mysql_only_params_from_saved_connections() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("ssl-mode=preferred&charset=utf8mb4".to_string()); - - assert_eq!(config.connection_url(), "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=prefer"); - tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - } - - #[test] - fn postgres_url_maps_mysql_ssl_mode_require_to_sslmode() { - let mut config = mysql_config("postgres", "secret", Some("test")); - config.db_type = DatabaseType::Postgres; - config.url_params = Some("ssl-mode=required&verify_ca=false&verify_identity=false".to_string()); - - assert_eq!(config.connection_url(), "postgres://postgres:secret@10.1.2.3:2883/test?sslmode=require"); - tokio_postgres::Config::from_str(&config.connection_url()).unwrap(); - } - #[test] fn postgres_url_appends_timezone_to_existing_options() { let mut config = mysql_config("postgres", "secret", Some("test")); diff --git a/crates/dbx-core/src/models/mod.rs b/crates/dbx-types/src/models/mod.rs similarity index 100% rename from crates/dbx-core/src/models/mod.rs rename to crates/dbx-types/src/models/mod.rs diff --git a/crates/dbx-core/src/mqtt/types.rs b/crates/dbx-types/src/mqtt.rs similarity index 98% rename from crates/dbx-core/src/mqtt/types.rs rename to crates/dbx-types/src/mqtt.rs index 2af493e39d..5dd63b3912 100644 --- a/crates/dbx-core/src/mqtt/types.rs +++ b/crates/dbx-types/src/mqtt.rs @@ -43,7 +43,7 @@ pub enum MqttAuth { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum MqttTlsVerificationMode { +pub enum MqttTlsVerificationMode { VerifyServerCert, SkipServerCertVerification, } @@ -173,7 +173,7 @@ impl MqttConnectionConfig { } } - pub(crate) fn tls_verification_mode(&self) -> Option { + pub fn tls_verification_mode(&self) -> Option { if !self.tls { None } else if self.tls_skip_verify { diff --git a/crates/dbx-types/src/mysql_event.rs b/crates/dbx-types/src/mysql_event.rs new file mode 100644 index 0000000000..6c5d97845c --- /dev/null +++ b/crates/dbx-types/src/mysql_event.rs @@ -0,0 +1,24 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct MysqlEventInfo { + pub name: String, + pub schema: String, + pub definer: Option, + pub time_zone: Option, + pub event_type: Option, + pub execute_at: Option, + pub interval_value: Option, + pub interval_field: Option, + pub starts: Option, + pub ends: Option, + pub status: Option, + pub on_completion: Option, + pub comment: Option, + pub event_body: Option, + pub event_definition: Option, + pub created_at: Option, + pub updated_at: Option, + pub last_executed: Option, + pub source: Option, +} diff --git a/crates/dbx-types/src/sql_file.rs b/crates/dbx-types/src/sql_file.rs new file mode 100644 index 0000000000..7aca098065 --- /dev/null +++ b/crates/dbx-types/src/sql_file.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +/// The database is the scope in the dump, not the destination selected in the UI. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlFileTable { + pub database: Option, + pub name: String, +} diff --git a/crates/dbx-core/src/types.rs b/crates/dbx-types/src/types.rs similarity index 98% rename from crates/dbx-core/src/types.rs rename to crates/dbx-types/src/types.rs index f302b06fab..d9a41f08a3 100644 --- a/crates/dbx-core/src/types.rs +++ b/crates/dbx-types/src/types.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -pub use crate::mysql_event_sql::MysqlEventInfo; +pub use crate::mysql_event::MysqlEventInfo; #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct DatabaseInfo { @@ -320,13 +320,13 @@ pub struct SpatialColumn { } #[derive(Debug, Default)] -pub(crate) struct SpatialColumnBuilder { +pub struct SpatialColumnBuilder { // column_index -> first non-null srid seen (sticky once set) columns: std::collections::BTreeMap>, } impl SpatialColumnBuilder { - pub(crate) fn new(column_indices: impl IntoIterator) -> Self { + pub fn new(column_indices: impl IntoIterator) -> Self { let mut builder = Self::default(); for column_index in column_indices { builder.columns.entry(column_index).or_insert(None); @@ -336,7 +336,7 @@ impl SpatialColumnBuilder { /// Record a geometry cell's SRID. The first non-null (and non-zero) value /// wins; later observations for the same column are ignored. - pub(crate) fn observe(&mut self, column_index: usize, srid: Option) { + pub fn observe(&mut self, column_index: usize, srid: Option) { let entry = self.columns.entry(column_index).or_insert(None); if entry.is_none() { if let Some(value) = srid.filter(|value| *value != 0) { @@ -345,11 +345,11 @@ impl SpatialColumnBuilder { } } - pub(crate) fn finish(self) -> Vec { + pub fn finish(self) -> Vec { self.columns.into_iter().map(|(column_index, srid)| SpatialColumn { column_index, srid }).collect() } - pub(crate) fn finish_with_values( + pub fn finish_with_values( self, spatial_values: Vec>>, ) -> (Vec, Vec>>) { @@ -547,11 +547,11 @@ impl Serialize for JsSafeCell<'_> { match self.0 { serde_json::Value::Number(number) => { if let Some(value) = number.as_i64() { - if !(-crate::db::JS_MAX_SAFE_INTEGER..=crate::db::JS_MAX_SAFE_INTEGER).contains(&value) { + if !(-crate::json::JS_MAX_SAFE_INTEGER..=crate::json::JS_MAX_SAFE_INTEGER).contains(&value) { return serializer.serialize_str(&value.to_string()); } } else if let Some(value) = number.as_u64() { - if value > crate::db::JS_MAX_SAFE_INTEGER as u64 { + if value > crate::json::JS_MAX_SAFE_INTEGER as u64 { return serializer.serialize_str(&value.to_string()); } } else { @@ -1158,7 +1158,7 @@ mod tests { let wire = serde_json::to_string(&result).unwrap(); let serialized: serde_json::Value = serde_json::from_str(&wire).unwrap(); - let expected = crate::db::json_value_for_js(cell); + let expected = crate::json::json_value_for_js(cell); assert_eq!(serialized["rows"][0][0], expected); assert_eq!(serialized["rows"][0][0]["ids"][0], "2026081810175800100000000000"); diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 8b7b8ce3b6..729a69b9a7 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -35,6 +35,14 @@ RUN dpkg --add-architecture arm64 \ # This layer is cached by GHA as long as Cargo.toml/Cargo.lock don't change. COPY Cargo.toml Cargo.lock ./ COPY crates/dbx-core/Cargo.toml crates/dbx-core/ +COPY crates/dbx-platform/Cargo.toml crates/dbx-platform/ +COPY crates/dbx-types/Cargo.toml crates/dbx-types/ +COPY crates/dbx-sql/Cargo.toml crates/dbx-sql/ +COPY crates/dbx-formats/Cargo.toml crates/dbx-formats/ +COPY crates/dbx-ai-provider/Cargo.toml crates/dbx-ai-provider/ +COPY crates/dbx-plugin-runtime/Cargo.toml crates/dbx-plugin-runtime/ +COPY crates/dbx-drivers/Cargo.toml crates/dbx-drivers/ +COPY crates/dbx-sqlite-worker/Cargo.toml crates/dbx-sqlite-worker/ COPY crates/dbx-web/Cargo.toml crates/dbx-web/ COPY crates/dbx-mcp/Cargo.toml crates/dbx-mcp/ COPY crates/dbx-cli/Cargo.toml crates/dbx-cli/ @@ -42,6 +50,10 @@ COPY src-tauri/Cargo.toml src-tauri/ # Copy every vendored crate: root Cargo.toml [patch.crates-io] points at # vendor/{ctor,rumqttc,dirs-sys,pageant,wry}; omitting any breaks the offline build. COPY vendor/ vendor/ +RUN for crate in dbx-platform dbx-types dbx-sql dbx-formats dbx-ai-provider dbx-plugin-runtime dbx-drivers dbx-sqlite-worker; do \ + mkdir -p "crates/$crate/src" && printf "" > "crates/$crate/src/lib.rs"; \ + done \ + && echo 'fn main() {}' > crates/dbx-sqlite-worker/src/main.rs RUN mkdir -p crates/dbx-core/src && echo '' > crates/dbx-core/src/lib.rs \ && mkdir -p crates/dbx-web/src && echo 'fn main() {}' > crates/dbx-web/src/main.rs \ && mkdir -p crates/dbx-mcp/src && echo '' > crates/dbx-mcp/src/lib.rs && echo 'fn main() {}' > crates/dbx-mcp/src/main.rs \ @@ -64,7 +76,7 @@ RUN case "$TARGETARCH" in \ # Copy real sources and rebuild (only application code recompiles) COPY crates/ crates/ -# dbx-core's build.rs embeds plugins/dialects (SQL dialects) and +# dbx-sql/build.rs embeds plugins/dialects; dbx-types/build.rs embeds # plugins/connection-types (database manifest + DatabaseType enum); # either directory missing makes the build script panic. COPY plugins/dialects/ plugins/dialects/ diff --git a/package.json b/package.json index 8fd84652a2..2aa3782f22 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "check": "node scripts/run-check.mjs", "db:env": "node scripts/database-env.mjs", "test:db-env": "node --test scripts/database-env.test.mjs", + "test:architecture": "node --test scripts/core-architecture.test.mjs", "test:ai-html-canary": "node scripts/ai-html-runtime-canary.mjs", "bench:redis-key-search": "node scripts/bench/redis-key-search.mjs", "bench:mongodb-count": "node scripts/bench/mongodb-count.mjs", diff --git a/packages/app-tests/piAgentMcpBridge.test.ts b/packages/app-tests/piAgentMcpBridge.test.ts index bcfedde644..b9fe1c7721 100644 --- a/packages/app-tests/piAgentMcpBridge.test.ts +++ b/packages/app-tests/piAgentMcpBridge.test.ts @@ -4,7 +4,7 @@ import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; -const bridgePath = resolve("crates/dbx-core/assets/pi-mcp-bridge.mjs"); +const bridgePath = resolve("crates/dbx-ai-provider/assets/pi-mcp-bridge.mjs"); const envNames = ["DBX_PI_MCP_PROGRAM", "DBX_PI_MCP_ARGS", "DBX_PI_ENABLED_TOOLS", "DBX_PI_BRIDGE_READY_FILE"] as const; const originalEnv = Object.fromEntries(envNames.map((name) => [name, process.env[name]])); diff --git a/packages/app-tests/scheduledDatabaseBackup.test.ts b/packages/app-tests/scheduledDatabaseBackup.test.ts index 87440119b5..89a229c47e 100644 --- a/packages/app-tests/scheduledDatabaseBackup.test.ts +++ b/packages/app-tests/scheduledDatabaseBackup.test.ts @@ -359,7 +359,7 @@ test("scheduled backup history exposes rename and overall percentage controls", test("scheduled backups prepare table scope before opening a consistent snapshot", () => { const scheduler = readFileSync("apps/desktop/src/composables/useScheduledDatabaseBackups.ts", "utf8"); - const exportCore = readFileSync("crates/dbx-core/src/database_export.rs", "utf8"); + const exportCore = readFileSync("crates/dbx-core/src/data/database_export.rs", "utf8"); const schemaIndex = scheduler.indexOf("await api.listSchemas(config.connectionId, database)"); const snapshotIndex = scheduler.indexOf("await api.beginDatabaseBackupSnapshot(config.connectionId, database, runId)"); const exportIndex = scheduler.indexOf("await runDatabaseExportUntilTerminal("); diff --git a/packages/app-tests/sqlserverResultFocus6189.test.ts b/packages/app-tests/sqlserverResultFocus6189.test.ts index 7d288d2968..e66dd256e3 100644 --- a/packages/app-tests/sqlserverResultFocus6189.test.ts +++ b/packages/app-tests/sqlserverResultFocus6189.test.ts @@ -51,7 +51,7 @@ function sqlServerConn(id: string): ConnectionConfig { const THREE_RESULTS = "SELECT 1 AS a; SELECT 2 AS b; SELECT 3 AS c;"; const TRAILING_MESSAGE = "SELECT 9 AS z; PRINT N'DBCC execution completed.';"; -/** Mirrors the shapes crates/dbx-core/src/db/sqlserver.rs:523-542 actually produces. */ +/** Mirrors the shapes crates/dbx-drivers/src/db/sqlserver.rs:523-542 actually produces. */ function resultsFor(sql: string) { if (sql.includes("SELECT 3 AS c")) { return [ diff --git a/scripts/core-architecture.test.mjs b/scripts/core-architecture.test.mjs new file mode 100644 index 0000000000..64d1bf4986 --- /dev/null +++ b/scripts/core-architecture.test.mjs @@ -0,0 +1,129 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +const read = (name) => readFileSync(path.join(root, name), "utf8"); +const metadata = JSON.parse(execFileSync("cargo", ["metadata", "--locked", "--offline", "--no-deps", "--format-version", "1"], { + cwd: root, + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, +})); +const members = new Map(metadata.packages.filter((pkg) => metadata.workspace_members.includes(pkg.id)).map((pkg) => [pkg.name, pkg])); +const allowedDependencies = { + "dbx-types": [], + "dbx-platform": [], + "dbx-formats": [], + "dbx-sql": ["dbx-types"], + "dbx-ai-provider": ["dbx-platform"], + "dbx-plugin-runtime": ["dbx-types", "dbx-platform"], + "dbx-drivers": ["dbx-types", "dbx-sql", "dbx-platform", "dbx-sqlite-worker"], +}; +const core = members.get("dbx-core"); + +for (const [name, allowed] of Object.entries(allowedDependencies)) { + test(`${name} has a one-way dependency boundary`, () => { + const pkg = members.get(name); + assert.ok(pkg, `${name} must be a workspace member`); + assert.deepEqual(pkg.publish, [], `${name} is an internal workspace crate`); + for (const dependency of pkg.dependencies) { + if (members.has(dependency.name)) { + assert.ok(allowed.includes(dependency.name), `${name} must not depend on ${dependency.name}, including test/build dependencies`); + } + if (dependency.kind !== "dev") { + assert.ok(!dependency.features.includes("test-support"), `${name} must not enable production test hooks`); + } + } + }); +} + +test("core owns application orchestration in real business directories", () => { + const ownedFiles = [ + "connection/mod.rs", "connection/connection_secrets.rs", "connection/driver_runtime.rs", + "query/mod.rs", "query/query_cancel.rs", "query/document_ops.rs", + "schema/mod.rs", "schema/table_structure_sql/sqlite_rebuild.rs", + "data/transfer.rs", "data/table_import.rs", "data/docs/export.rs", "data/cloudflare_d1/import.rs", + "ai/agent_loop.rs", "ai/agent_tools.rs", "admin/consul/mod.rs", "admin/mq/mod.rs", "admin/mqtt/mod.rs", + "persistence/storage.rs", "persistence/config/mod.rs", "safety/production_safety.rs", "host/update.rs", + ]; + for (const name of ownedFiles) { + assert.ok(existsSync(path.join(root, "crates/dbx-core/src", name)), `missing owned business source: ${name}`); + } + const rootSources = readdirSync(path.join(root, "crates/dbx-core/src")).filter((name) => name.endsWith(".rs")); + assert.deepEqual(rootSources, ["lib.rs"]); + assert.match(read("crates/dbx-core/src/ai/mod.rs"), /pub use dbx_ai_provider::ai::\*/); + assert.match(read("crates/dbx-core/src/db/mod.rs"), /pub use crate::data::cloudflare_d1/); + assert.ok(!existsSync(path.join(root, "crates/dbx-core/build.rs"))); +}); + +test("core forwards capability flags without enabling driver defaults or test hooks", () => { + for (const feature of ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-bundled", "sqlite-sqlcipher", "sqlite-multiple-ciphers"]) { + assert.ok(core.features[feature].includes(`dbx-drivers/${feature}`), `missing driver forwarding for ${feature}`); + } + for (const name of ["dbx-types", "dbx-sql"]) { + assert.ok(core.features.openapi.includes(`${name}/openapi`)); + } + for (const name of ["dbx-types", "dbx-sql", "dbx-drivers"]) { + const dependency = core.dependencies.find((dep) => dep.name === name && dep.kind === null); + assert.equal(dependency.uses_default_features, false, `${name} must follow explicit core features`); + } + for (const dependency of core.dependencies.filter((dep) => dep.kind !== "dev")) { + assert.ok(!dependency.features.includes("test-support")); + } + assert.deepEqual(core.features.default, ["duckdb-sidecar", "dynamodb", "mq-admin", "sqlite-sqlcipher", "system-fonts"]); +}); + +test("code generators and runtime assets belong to their implementation crates", () => { + assert.match(read("crates/dbx-types/build.rs"), /plugins\/connection-types/); + assert.match(read("crates/dbx-sql/build.rs"), /plugins\/dialects/); + assert.match(read("crates/dbx-ai-provider/src/ai_pi_agent_cli.rs"), /assets\/pi-mcp-bridge\.mjs/); + assert.ok(existsSync(path.join(root, "crates/dbx-ai-provider/assets/pi-mcp-bridge.mjs"))); + assert.deepEqual( + JSON.parse(read("crates/dbx-drivers/assets/agent-protocol-v2.json")), + JSON.parse(read("agents/common/src/main/resources/agent-protocol-v2.json")), + ); + assert.match(read("crates/dbx-platform/src/lib.rs"), /#\[cfg\(all\(target_os = "windows", target_env = "gnu"\)\)\]\s*mod nanosleep_stub/); +}); + +test("Docker dependency caching and Nix include the full workspace", () => { + const dockerfile = read("deploy/Dockerfile"); + const dependencyStage = dockerfile.split("COPY crates/ crates/")[0]; + for (const pkg of members.values()) { + const relativeManifest = path.relative(root, pkg.manifest_path).replaceAll(path.sep, "/"); + assert.ok(dependencyStage.includes(`COPY ${relativeManifest} `), `Docker dependency stage misses ${relativeManifest}`); + } + for (const input of ["plugins/dialects/", "plugins/connection-types/"]) { + assert.ok(dockerfile.includes(`COPY ${input} ${input}`)); + } + assert.ok(dockerfile.includes("dbx-sqlite-worker/src/main.rs")); + assert.match(read("flake.nix"), /src = pkgs\.lib\.cleanSource \.\/\.;/); +}); + +test("CI and release tracking follow the new source owners", () => { + const ci = read(".github/workflows/ci.yml"); + assert.ok(ci.includes("node --test scripts/core-architecture.test.mjs")); + assert.ok(ci.includes("'crates/**'")); + for (const input of ["plugins/dialects/**", "plugins/connection-types/**", "crates/dbx-drivers/assets/agent-protocol-v2.json"]) { + assert.ok(ci.includes(`'${input}'`), `CI misses ${input}`); + } + assert.ok(read("scripts/release.mjs").includes('"crates/dbx-drivers/src/mongo_shell.rs"')); +}); + +test("source-sensitive client tests point to existing owners", () => { + function inspect(directory) { + for (const entry of readdirSync(path.join(root, directory), { withFileTypes: true })) { + const name = `${directory}/${entry.name}`; + if (entry.isDirectory()) inspect(name); + else if (/\.(?:spec|test)\.[cm]?[jt]s$/.test(name)) { + for (const match of read(name).matchAll(/crates\/dbx-[a-z-]+\/(?:src|assets)\/[a-zA-Z0-9_/.-]+\.(?:rs|mjs)/g)) { + assert.ok(existsSync(path.join(root, match[0])), `${name} points at missing source ${match[0]}`); + } + } + } + } + inspect("apps/desktop/src"); + inspect("packages/app-tests"); +}); diff --git a/scripts/release.mjs b/scripts/release.mjs index a3e2dd279b..21ec5591a9 100644 --- a/scripts/release.mjs +++ b/scripts/release.mjs @@ -42,7 +42,7 @@ const PACKAGE_RELEASE_PATHS = [ "packages/mcp-win32-x64/", "crates/dbx-mcp/", "crates/dbx-cli/", - "crates/dbx-core/src/mongo_shell.rs", + "crates/dbx-drivers/src/mongo_shell.rs", "Cargo.toml", "Cargo.lock", ".github/workflows/mcp-release.yml",