diff --git a/apps/desktop/src/components/sidebar/ConnectionTree.vue b/apps/desktop/src/components/sidebar/ConnectionTree.vue index 3f8a6edbd2..86dfafc0df 100644 --- a/apps/desktop/src/components/sidebar/ConnectionTree.vue +++ b/apps/desktop/src/components/sidebar/ConnectionTree.vue @@ -107,7 +107,7 @@ watch(deferredSearchQuery, (newQuery, oldQuery) => { .catch(() => {}); }); -const searchableObjectGroupTypes = new Set(["group-tables", "group-views", "group-materialized-views"]); +const searchableObjectGroupTypes = new Set(["group-tables", "group-views", "group-materialized-views", "group-procedures", "group-functions", "group-sequences", "group-packages"]); const simpleObjectParentTypes = new Set(["database", "schema", "linked-server-schema"]); const simpleObjectChildTypes = new Set(["table", "view", "materialized_view", "procedure", "function", "sequence", "package", "package-body", "load-more"]); diff --git a/apps/desktop/src/stores/__tests__/connectionStore.metadataLoading.spec.ts b/apps/desktop/src/stores/__tests__/connectionStore.metadataLoading.spec.ts index d561836305..edb777a168 100644 --- a/apps/desktop/src/stores/__tests__/connectionStore.metadataLoading.spec.ts +++ b/apps/desktop/src/stores/__tests__/connectionStore.metadataLoading.spec.ts @@ -1,6 +1,6 @@ import { createPinia, setActivePinia } from "pinia"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { ConnectionConfig, TableInfo, TreeNode } from "@/types/database"; +import type { ConnectionConfig, ObjectInfo, TableInfo, TreeNode } from "@/types/database"; function installLocalStorage() { const data = new Map(); @@ -24,6 +24,32 @@ function postgresConnection(): ConnectionConfig { } as ConnectionConfig; } +function mysqlConnection(): ConnectionConfig { + return { + id: "mysql-1", + name: "MySQL", + db_type: "mysql", + host: "127.0.0.1", + port: 3306, + username: "root", + password: "", + database: "app", + } as ConnectionConfig; +} + +function procedure(name: string): ObjectInfo { + return { + name, + object_type: "PROCEDURE", + schema: "app", + comment: null, + created_at: null, + updated_at: null, + parent_schema: null, + parent_name: null, + }; +} + describe("connectionStore metadata loading", () => { beforeEach(() => { vi.resetModules(); @@ -96,4 +122,88 @@ describe("connectionStore metadata loading", () => { expect(listObjects).toHaveBeenCalled(); expect(schemaNode.children?.map((node) => node.label)).toEqual(["users"]); }); + + it("paginates procedure groups and appends the next page", async () => { + const firstPage = Array.from({ length: 201 }, (_, index) => procedure(`p_${String(index + 1).padStart(4, "0")}`)); + const listObjects = vi + .fn() + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce([procedure("p_0201"), procedure("p_0202")]) + .mockResolvedValueOnce([procedure("p_0999")]); + + vi.doMock("@/lib/backend/tauriRuntime", () => ({ isTauriRuntime: () => false })); + vi.doMock("@/lib/backend/api", () => ({ + checkConnectionHealth: vi.fn().mockResolvedValue(undefined), + deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined), + listObjects, + loadSchemaCache: vi.fn().mockResolvedValue(null), + saveSchemaCache: vi.fn().mockResolvedValue(undefined), + saveConnections: vi.fn().mockResolvedValue(undefined), + saveSidebarLayout: vi.fn().mockResolvedValue(undefined), + })); + + const { useConnectionStore } = await import("@/stores/connectionStore"); + const { useSettingsStore } = await import("@/stores/settingsStore"); + const store = useConnectionStore(); + const settingsStore = useSettingsStore(); + settingsStore.editorSettings.sidebarObjectDisplay = "grouped"; + settingsStore.desktopSettings.sidebar_table_page_size = 200; + + const connection = mysqlConnection(); + const procedureGroup: TreeNode = { + id: "mysql-1:app:__procedures", + label: "tree.procedures", + type: "group-procedures", + connectionId: connection.id, + database: "app", + isExpanded: false, + children: [], + }; + store.connections = [connection]; + store.connectedIds.add(connection.id); + store.treeNodes = [ + { + id: connection.id, + label: connection.name, + type: "connection", + connectionId: connection.id, + isExpanded: true, + children: [ + { + id: "mysql-1:app", + label: "app", + type: "database", + connectionId: connection.id, + database: "app", + isExpanded: true, + children: [procedureGroup], + }, + ], + }, + ]; + + const storedProcedureGroup = store.treeNodes[0].children?.[0].children?.[0]; + expect(storedProcedureGroup?.type).toBe("group-procedures"); + await store.loadObjectGroupChildren(storedProcedureGroup!); + + expect(listObjects).toHaveBeenNthCalledWith(1, connection.id, "app", "app", ["PROCEDURE"], undefined, 201, 0); + expect(storedProcedureGroup?.children).toHaveLength(201); + expect(storedProcedureGroup?.children?.[0].label).toBe("p_0001"); + expect(storedProcedureGroup?.children?.[199].label).toBe("p_0200"); + expect(storedProcedureGroup?.children?.[200].label).toBe("tree.loadMore"); + + const loadMoreNode = storedProcedureGroup?.children?.at(-1); + expect(loadMoreNode?.type).toBe("load-more"); + await store.loadMoreObjectGroupChildren(loadMoreNode!); + + expect(listObjects).toHaveBeenNthCalledWith(2, connection.id, "app", "app", ["PROCEDURE"], undefined, 201, 200); + expect(storedProcedureGroup?.children).toHaveLength(202); + expect(storedProcedureGroup?.children?.at(-1)?.label).toBe("p_0202"); + + store.sidebarSearchQuery = "p_0999"; + await store.loadObjectGroupChildren(storedProcedureGroup!, { force: true, searchFilter: "p_0999" }); + + expect(listObjects).toHaveBeenNthCalledWith(3, connection.id, "app", "app", ["PROCEDURE"], "p_0999", undefined, undefined); + expect(storedProcedureGroup?.children?.map((node) => node.label)).toEqual(["p_0999"]); + }); }); diff --git a/apps/desktop/src/stores/connectionStore.ts b/apps/desktop/src/stores/connectionStore.ts index fad0c767e3..ed716c78d1 100644 --- a/apps/desktop/src/stores/connectionStore.ts +++ b/apps/desktop/src/stores/connectionStore.ts @@ -1206,6 +1206,56 @@ export const useConnectionStore = defineStore("connection", () => { }; } + async function loadPagedObjectGroupChildren(options: { + node: TreeNode; + parentNodeId: string; + querySchema: string; + effectiveSchema?: string; + objectTypes: DatabaseObjectTreeKind[]; + offset: number; + pageSize: number; + searchFilter?: string; + force?: boolean; + }): Promise<{ children: TreeNode[]; objectCount: number; hasMore: boolean; nextOffset: number }> { + if (!options.node.connectionId || !options.node.database) { + return { children: [], objectCount: 0, hasMore: false, nextOffset: options.offset }; + } + const searchFilter = options.searchFilter || undefined; + const fetchLimit = searchFilter ? undefined : options.pageSize + 1; + const fetchOffset = searchFilter ? undefined : options.offset; + const objects = await loadCachedMetadataListPage( + metadataListCacheScope({ + kind: "object-list-page", + connectionId: options.node.connectionId, + database: options.node.database, + schema: options.querySchema, + nodeKind: options.node.type, + objectTypes: options.objectTypes, + searchFilter, + limit: fetchLimit, + offset: fetchOffset, + sidebarDisplayMode: useSettingsStore().editorSettings.sidebarObjectDisplay, + }), + () => api.listObjects(options.node.connectionId!, options.node.database!, options.querySchema, options.objectTypes, searchFilter, fetchLimit, fetchOffset), + { force: options.force }, + ); + const hasMore = searchFilter ? false : objects.length > options.pageSize; + const pageObjects = hasMore ? objects.slice(0, options.pageSize) : objects; + const children = objectGroupChildrenFromObjects({ + node: options.node, + parentNodeId: options.parentNodeId, + effectiveSchema: options.effectiveSchema, + objectTypes: options.objectTypes, + objects: pageObjects, + }); + return { + children, + objectCount: children.length, + hasMore, + nextOffset: options.offset + pageObjects.length, + }; + } + async function loadPagedSimpleTableChildren(options: { nodeId: string; connectionId: string; @@ -3003,6 +3053,7 @@ export const useConnectionStore = defineStore("connection", () => { async function loadObjectGroupChildren(node: TreeNode, options?: LoadTreeOptions) { const configForScope = node.connectionId ? getConfig(node.connectionId) : undefined; const objectTypesForScope = objectTypesForGroupNode(node.type); + const pageSizeForScope = sidebarObjectGroupPageSize(); return runTreeMetadataLoad( { kind: "object-group", @@ -3012,7 +3063,7 @@ export const useConnectionStore = defineStore("connection", () => { nodeKind: node.type, objectTypes: objectTypesForScope, searchFilter: activeTreeLoadSearchFilter(options), - limit: sidebarObjectGroupPageSize() + 1, + limit: pageSizeForScope + 1, offset: 0, sidebarDisplayMode: useSettingsStore().editorSettings.sidebarObjectDisplay, driverProfile: metadataDriverProfile(configForScope), @@ -3060,28 +3111,20 @@ export const useConnectionStore = defineStore("connection", () => { children = page.hasMore && !searchFilter ? appendTableTreeLoadMoreNode(page.children, buildLoadMoreNode(node, page.nextOffset, sidebarObjectGroupPageSize()), page.loadMoreParent) : page.children; nextObjectCount = page.objectCount; } else { - const objects = await loadCachedMetadataListPage( - metadataListCacheScope({ - kind: "object-list-page", - connectionId: node.connectionId, - database: node.database, - schema: querySchema, - nodeKind: node.type, - objectTypes, - searchFilter: searchFilter || undefined, - sidebarDisplayMode: useSettingsStore().editorSettings.sidebarObjectDisplay, - }), - () => api.listObjects(node.connectionId!, node.database!, querySchema, objectTypes, searchFilter || undefined), - { force: options?.force }, - ); - children = objectGroupChildrenFromObjects({ + const pageSize = sidebarObjectGroupPageSize(); + const page = await loadPagedObjectGroupChildren({ node, parentNodeId, + querySchema, effectiveSchema, objectTypes, - objects, + offset: 0, + pageSize, + searchFilter: searchFilter || undefined, + force: options?.force, }); - nextObjectCount = children.length; + children = page.hasMore && !searchFilter ? [...page.children, buildLoadMoreNode(node, page.nextOffset, pageSize)] : page.children; + nextObjectCount = page.objectCount; } if (isTreeLoadSearchChanged(searchFilter, options)) return; if (!canApplyTreeMetadataResult(node)) return; @@ -3186,26 +3229,44 @@ export const useConnectionStore = defineStore("connection", () => { const objectTypes = objectTypesForGroupNode(parent.type); const parentNodeId = objectGroupRefreshParentId(parent); if (!objectTypes || !parentNodeId) return; - if (!objectTypes.every((objectType) => objectType === "TABLE" || objectType === "VIEW" || objectType === "MATERIALIZED_VIEW")) return; const config = getConfig(parentConnectionId); const parentDatabase = parent.database; if (!parentDatabase) return; const querySchema = connectionObjectTreeQuerySchema(config, parentDatabase, parent.schema); const effectiveSchema = connectionObjectTreeNodeSchema(config, parentDatabase, parent.schema); - const page = await loadPagedTableGroupChildren({ - node: parent, - parentNodeId, - querySchema, - effectiveSchema, - objectTypes, - offset: loadMore.offset, - pageSize: loadMore.pageSize, - force: false, - }); - const currentChildren = withoutTableTreeLoadMoreNodes(parent.children); - const mergedChildren = mergeTableTreePageChildren(currentChildren, page.children, parentConnectionId, parentDatabase); - const nextChildren = page.hasMore ? appendTableTreeLoadMoreNode(mergedChildren, buildLoadMoreNode(parent, page.nextOffset, loadMore.pageSize), page.loadMoreParent) : mergedChildren; + const wantsOnlyTablesOrViews = objectTypes.every((objectType) => objectType === "TABLE" || objectType === "VIEW" || objectType === "MATERIALIZED_VIEW"); + let mergedChildren: TreeNode[]; + let nextChildren: TreeNode[]; + if (wantsOnlyTablesOrViews) { + const page = await loadPagedTableGroupChildren({ + node: parent, + parentNodeId, + querySchema, + effectiveSchema, + objectTypes, + offset: loadMore.offset, + pageSize: loadMore.pageSize, + force: false, + }); + const currentChildren = withoutTableTreeLoadMoreNodes(parent.children); + mergedChildren = mergeTableTreePageChildren(currentChildren, page.children, parentConnectionId, parentDatabase); + nextChildren = page.hasMore ? appendTableTreeLoadMoreNode(mergedChildren, buildLoadMoreNode(parent, page.nextOffset, loadMore.pageSize), page.loadMoreParent) : mergedChildren; + } else { + const page = await loadPagedObjectGroupChildren({ + node: parent, + parentNodeId, + querySchema, + effectiveSchema, + objectTypes, + offset: loadMore.offset, + pageSize: loadMore.pageSize, + force: false, + }); + const currentChildren = withoutLoadMoreNodes(parent.children); + mergedChildren = mergeLocatedTreeChildren(parent, currentChildren, page.children, parentConnectionId, parentDatabase); + nextChildren = page.hasMore ? [...mergedChildren, buildLoadMoreNode(parent, page.nextOffset, loadMore.pageSize)] : mergedChildren; + } if (!canApplyTreeMetadataResult(parent)) return; parent.objectCount = mergedChildren.length; setChildren(parent, nextChildren); diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index e0b4e33dea..e686bea038 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -1957,7 +1957,29 @@ pub async fn list_tables_show(pool: &MySqlPool, database: &str) -> Result String { +fn requested_object_type(object_types: Option<&[String]>, object_type: &str) -> bool { + object_types.is_none_or(|types| { + types.is_empty() || types.iter().any(|candidate| candidate.eq_ignore_ascii_case(object_type)) + }) +} + +fn sql_pagination(limit: Option, offset: Option) -> String { + limit.map_or_else(String::new, |limit| format!(" LIMIT {limit} OFFSET {}", offset.unwrap_or(0))) +} + +fn list_tables_objects_sql( + database: &str, + object_types: Option<&[String]>, + limit: Option, + offset: Option, +) -> String { + let wants_tables = requested_object_type(object_types, "TABLE"); + let wants_views = requested_object_type(object_types, "VIEW"); + let type_filter = match (wants_tables, wants_views) { + (true, false) => " AND TABLE_TYPE <> 'VIEW'", + (false, true) => " AND TABLE_TYPE = 'VIEW'", + _ => "", + }; format!( "SELECT TABLE_NAME AS object_name, \ CASE WHEN TABLE_TYPE = 'VIEW' THEN 'VIEW' ELSE 'TABLE' END AS object_type, \ @@ -1967,22 +1989,37 @@ fn list_tables_objects_sql(database: &str) -> String { NULL AS parent_schema, NULL AS parent_name, \ CASE WHEN TABLE_TYPE = 'VIEW' THEN 1 ELSE 0 END AS sort_order \ FROM information_schema.TABLES \ - WHERE TABLE_SCHEMA = {db} \ - ORDER BY sort_order, object_name", + WHERE TABLE_SCHEMA = {db}{type_filter} \ + ORDER BY sort_order, object_name{pagination}", db = quote_value(database), + pagination = sql_pagination(limit, offset), ) } -fn list_routines_sql(database: &str) -> String { +fn list_routines_sql( + database: &str, + object_types: Option<&[String]>, + limit: Option, + offset: Option, +) -> String { + let routine_types = [ + ("PROCEDURE", requested_object_type(object_types, "PROCEDURE")), + ("FUNCTION", requested_object_type(object_types, "FUNCTION")), + ] + .into_iter() + .filter_map(|(routine_type, requested)| requested.then_some(format!("'{}'", routine_type))) + .collect::>() + .join(", "); format!( "SELECT ROUTINE_NAME AS object_name, ROUTINE_TYPE AS object_type, NULL AS object_comment, \ NULL AS created_at, NULL AS updated_at, \ NULL AS parent_schema, NULL AS parent_name, \ CASE WHEN ROUTINE_TYPE = 'PROCEDURE' THEN 2 ELSE 3 END AS sort_order \ FROM information_schema.ROUTINES \ - WHERE ROUTINE_SCHEMA = {db} AND ROUTINE_TYPE IN ('PROCEDURE', 'FUNCTION') \ - ORDER BY sort_order, object_name", + WHERE ROUTINE_SCHEMA = {db} AND ROUTINE_TYPE IN ({routine_types}) \ + ORDER BY sort_order, object_name{pagination}", db = quote_value(database), + pagination = sql_pagination(limit, offset), ) } @@ -2015,47 +2052,91 @@ fn row_to_object(row: &mysql_async::Row, database: &str) -> ObjectInfo { } } -pub async fn list_objects(pool: &MySqlPool, database: &str) -> Result, String> { - let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?; +pub struct PagedObjectList { + pub objects: Vec, + pub paging_applied: bool, +} - let tables_sql = list_tables_objects_sql(database); - let result = match conn.query_iter(&tables_sql).await { - Ok(result) => result, - Err(err) => { - log::debug!( - "Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES failed: {err}" - ); - return list_table_objects_show(pool, database).await; - } +fn object_query_supports_paging(object_types: Option<&[String]>) -> bool { + let Some(object_types) = object_types.filter(|types| !types.is_empty()) else { + return false; }; - let table_rows: Vec = result.collect_and_drop().await.map_err(|e| e.to_string())?; - if table_rows.is_empty() { - log::debug!( - "Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES returned no named tables" - ); - return list_table_objects_show(pool, database).await; + let uses_table_source = object_types + .iter() + .any(|object_type| object_type.eq_ignore_ascii_case("TABLE") || object_type.eq_ignore_ascii_case("VIEW")); + let uses_routine_source = object_types.iter().any(|object_type| { + object_type.eq_ignore_ascii_case("PROCEDURE") || object_type.eq_ignore_ascii_case("FUNCTION") + }); + let all_types_supported = object_types.iter().all(|object_type| { + object_type.eq_ignore_ascii_case("TABLE") + || object_type.eq_ignore_ascii_case("VIEW") + || object_type.eq_ignore_ascii_case("PROCEDURE") + || object_type.eq_ignore_ascii_case("FUNCTION") + }); + all_types_supported && uses_table_source != uses_routine_source +} + +pub async fn list_objects( + pool: &MySqlPool, + database: &str, + object_types: Option<&[String]>, + limit: Option, + offset: Option, +) -> Result { + let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?; + let wants_tables = requested_object_type(object_types, "TABLE") || requested_object_type(object_types, "VIEW"); + let wants_routines = + requested_object_type(object_types, "PROCEDURE") || requested_object_type(object_types, "FUNCTION"); + let paging_applied = limit.is_some() && object_query_supports_paging(object_types); + let (query_limit, query_offset) = if paging_applied { (limit, offset) } else { (None, None) }; + let mut objects = Vec::new(); + + if wants_tables { + let tables_sql = list_tables_objects_sql(database, object_types, query_limit, query_offset); + let result = match conn.query_iter(&tables_sql).await { + Ok(result) => result, + Err(err) => { + log::debug!( + "Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES failed: {err}" + ); + return list_table_objects_show(pool, database) + .await + .map(|objects| PagedObjectList { objects, paging_applied: false }); + } + }; + let table_rows: Vec = result.collect_and_drop().await.map_err(|e| e.to_string())?; + if table_rows.is_empty() && !wants_routines { + log::debug!( + "Falling back to SHOW TABLES for object browser database `{database}` after information_schema.TABLES returned no named tables" + ); + return list_table_objects_show(pool, database) + .await + .map(|objects| PagedObjectList { objects, paging_applied: false }); + } + objects.extend(table_rows.iter().map(|row| row_to_object(row, database))); } - let mut objects: Vec = table_rows.iter().map(|row| row_to_object(row, database)).collect(); // Routines are queried separately: some MySQL-compatible servers (sharding proxies, // OceanBase/TiDB variants, restricted accounts) reject information_schema.ROUTINES with // ER_UNKNOWN_ERROR (1105). Degrading gracefully keeps tables/views usable. - let routines_sql = list_routines_sql(database); - match conn.query_iter(&routines_sql).await { - Ok(result) => match result.collect_and_drop::().await { - Ok(routine_rows) => { - objects.extend(routine_rows.iter().map(|row| row_to_object(row, database))); - } + if wants_routines { + let routines_sql = list_routines_sql(database, object_types, query_limit, query_offset); + match conn.query_iter(&routines_sql).await { + Ok(result) => match result.collect_and_drop::().await { + Ok(routine_rows) => { + objects.extend(routine_rows.iter().map(|row| row_to_object(row, database))); + } + Err(e) => { + log::warn!("Skipping routines for database `{}` in object browser: {}", database, e); + } + }, Err(e) => { log::warn!("Skipping routines for database `{}` in object browser: {}", database, e); } - }, - Err(e) => { - log::warn!("Skipping routines for database `{}` in object browser: {}", database, e); } } - Ok(objects) + Ok(PagedObjectList { objects, paging_applied }) } pub async fn list_object_statistics(pool: &MySqlPool, database: &str) -> Result, String> { @@ -2115,7 +2196,7 @@ pub async fn list_table_objects_show(pool: &MySqlPool, database: &str) -> Result async fn list_routine_objects(pool: &MySqlPool, database: &str) -> Result, String> { let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?; - let routines_sql = list_routines_sql(database); + let routines_sql = list_routines_sql(database, None, None, None); let result = conn.query_iter(&routines_sql).await.map_err(|e| e.to_string())?; let rows: Vec = result.collect_and_drop().await.map_err(|e| e.to_string())?; Ok(rows.iter().map(|row| row_to_object(row, database)).collect()) @@ -2125,7 +2206,7 @@ pub async fn list_completion_objects(pool: &MySqlPool, database: &str) -> Result let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?; let mut objects = Vec::new(); - let routines_sql = list_routines_sql(database); + let routines_sql = list_routines_sql(database, None, None, None); match conn.query_iter(&routines_sql).await { Ok(result) => match result.collect_and_drop::().await { Ok(rows) => objects.extend(rows.iter().map(|row| row_to_object(row, database))), @@ -3797,7 +3878,7 @@ mod tests { #[test] fn mysql_list_tables_objects_sql_includes_timestamps() { - let sql = list_tables_objects_sql("app"); + let sql = list_tables_objects_sql("app", None, None, None); assert!(sql.contains("information_schema.TABLES")); assert!(!sql.contains("information_schema.ROUTINES")); @@ -3939,7 +4020,7 @@ mod tests { #[test] fn mysql_list_routines_sql_is_independent_of_tables() { - let sql = list_routines_sql("app"); + let sql = list_routines_sql("app", None, None, None); assert!(sql.contains("information_schema.ROUTINES")); assert!(!sql.contains("information_schema.TABLES")); @@ -3950,6 +4031,33 @@ mod tests { assert!(!sql.contains("CREATED AS created_at")); } + #[test] + fn mysql_list_routines_sql_honors_requested_type_and_paging() { + let object_types = vec!["PROCEDURE".to_string()]; + let sql = list_routines_sql("app", Some(&object_types), Some(101), Some(200)); + + assert!(sql.contains("ROUTINE_TYPE IN ('PROCEDURE')")); + assert!(!sql.contains("'FUNCTION'")); + assert!(sql.ends_with("LIMIT 101 OFFSET 200")); + } + + #[test] + fn mysql_list_tables_objects_sql_honors_requested_type_and_paging() { + let object_types = vec!["VIEW".to_string()]; + let sql = list_tables_objects_sql("app", Some(&object_types), Some(51), Some(100)); + + assert!(sql.contains("TABLE_TYPE = 'VIEW'")); + assert!(sql.ends_with("LIMIT 51 OFFSET 100")); + } + + #[test] + fn mysql_object_query_only_pages_within_one_metadata_source() { + assert!(object_query_supports_paging(Some(&["PROCEDURE".to_string()]))); + assert!(object_query_supports_paging(Some(&["TABLE".to_string(), "VIEW".to_string()]))); + assert!(!object_query_supports_paging(Some(&["TABLE".to_string(), "PROCEDURE".to_string()]))); + assert!(!object_query_supports_paging(None)); + } + #[test] fn mysql_completion_triggers_sql_lists_database_triggers() { let sql = list_completion_triggers_sql("app"); diff --git a/crates/dbx-core/src/schema.rs b/crates/dbx-core/src/schema.rs index 3997a391b2..15d894f47d 100644 --- a/crates/dbx-core/src/schema.rs +++ b/crates/dbx-core/src/schema.rs @@ -3216,14 +3216,15 @@ pub async fn list_objects_core( retry_metadata_connection(state, connection_id, Some(database), || async { let objects = list_objects_once(state, connection_id, database, schema, filter, limit, offset, object_types) .await - .map(|objects| { - let final_offset = if oracle_agent_paging_likely_applied(use_oracle_agent_paging, limit, objects.len()) + .map(|outcome| { + let final_offset = if outcome.paging_applied + || oracle_agent_paging_likely_applied(use_oracle_agent_paging, limit, outcome.objects.len()) { Some(0) } else { offset }; - filter_object_infos(objects, filter, limit, final_offset, object_types) + filter_object_infos(outcome.objects, filter, limit, final_offset, object_types) })?; Ok(objects) }) @@ -3534,6 +3535,15 @@ async fn list_object_statistics_once( } } +struct ObjectListOutcome { + objects: Vec, + paging_applied: bool, +} + +fn unpaged_object_list(objects: Vec) -> ObjectListOutcome { + ObjectListOutcome { objects, paging_applied: false } +} + async fn list_objects_once( state: &AppState, connection_id: &str, @@ -3543,9 +3553,11 @@ async fn list_objects_once( limit: Option, offset: Option, object_types: Option<&[String]>, -) -> Result, String> { +) -> Result { let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?; let db_config = connection_config(state, connection_id).await; + let (mysql_limit, mysql_offset) = + if filter.is_none_or(|value| value.trim().is_empty()) { (limit, offset) } else { (None, None) }; { let connections = state.connections.read().await; @@ -3553,7 +3565,7 @@ async fn list_objects_once( if let Some(ext_pool) = extract_pool!(&connections, &pool_key, ExternalTabular) { drop(connections); let cache = ext_pool.cache.clone(); - return tokio::task::spawn_blocking(move || { + let objects = tokio::task::spawn_blocking(move || { let con = cache.lock().map_err(|e| e.to_string())?; Ok(duckdb_query_tables(&con)? .into_iter() @@ -3572,6 +3584,7 @@ async fn list_objects_once( }) .await .map_err(|e| e.to_string())?; + return objects.map(unpaged_object_list); } if let Some(PoolKind::ExternalDriver { config, session, .. }) = connections.get(&pool_key) { let config = config.clone(); @@ -3586,7 +3599,8 @@ async fn list_objects_once( filter, object_types, ) - .await; + .await + .map(unpaged_object_list); } let mut params = serde_json::json!({ "connection": config.as_ref(), "database": database, "schema": schema }); @@ -3602,9 +3616,14 @@ async fn list_objects_once( params, agent_metadata_timeout(Some(config.as_ref())), ) - .await; + .await + .map(unpaged_object_list); + } + if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { + drop(connections); + let mut client = client.lock().await; + return db::sqlserver::list_objects(&mut client, schema).await.map(unpaged_object_list); } - try_sqlserver!(connections, &pool_key, list_objects, schema); if let Some(client) = extract_pool!(&connections, &pool_key, Agent) { let is_oracle = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle); let use_oracle_agent_paging = db_config.as_ref().is_some_and(is_default_oracle_agent_config); @@ -3614,7 +3633,9 @@ async fn list_objects_once( let fallback_config = db_config.clone(); drop(connections); if is_oracle && !use_oracle_agent_paging { - return oracle_agent_list_objects(client, database, schema, timeout_duration).await; + return oracle_agent_list_objects(client, database, schema, timeout_duration) + .await + .map(unpaged_object_list); } let mut client = client.lock().await; let agent_filter = if filter_locally_after_oracle_comments { None } else { filter }; @@ -3655,13 +3676,15 @@ async fn list_objects_once( ) .await?; } - return Ok(objects); + return Ok(unpaged_object_list(objects)); } Ok(objects) => { if let Some(config) = fallback_config.as_ref() { match native_postgres_metadata_pool(state, connection_id, database, config).await { - Ok(Some(pool)) => return db::postgres::list_objects(&pool, schema).await, - Ok(None) => return Ok(objects), + Ok(Some(pool)) => { + return db::postgres::list_objects(&pool, schema).await.map(unpaged_object_list) + } + Ok(None) => return Ok(unpaged_object_list(objects)), Err(error) => { log::warn!( "[schema][agent:list_objects:fallback-failed] connection_id={} database={} schema={} error={}", @@ -3673,16 +3696,20 @@ async fn list_objects_once( } } } - return Ok(objects); + return Ok(unpaged_object_list(objects)); } Err(agent_error) => { if let Some(config) = fallback_config.as_ref() { if let Some(pool) = native_postgres_metadata_pool(state, connection_id, database, config).await? { - return db::postgres::list_objects(&pool, schema).await.map_err(|fallback_error| { - format!("{agent_error}\n\nNative PostgreSQL metadata fallback failed: {fallback_error}") - }); + return db::postgres::list_objects(&pool, schema).await.map(unpaged_object_list).map_err( + |fallback_error| { + format!( + "{agent_error}\n\nNative PostgreSQL metadata fallback failed: {fallback_error}" + ) + }, + ); } } return Err(agent_error); @@ -3698,36 +3725,40 @@ async fn list_objects_once( PoolKind::Mysql(p, mode) => { // Note: mysql and ob_oracle take different second args (database vs schema) if *mode == MysqlMode::OceanBaseOracle { - db::ob_oracle::list_objects(p, schema).await + db::ob_oracle::list_objects(p, schema).await.map(unpaged_object_list) } else if db_config.as_ref().is_some_and(is_manticoresearch_config) { - db::manticoresearch::list_objects(p, database).await + db::manticoresearch::list_objects(p, database).await.map(unpaged_object_list) } else if db_config.as_ref().is_some_and(is_doris_family_config) { - db::mysql::list_table_objects_show(p, database).await + db::mysql::list_table_objects_show(p, database).await.map(unpaged_object_list) } else { - db::mysql::list_objects(p, database).await + db::mysql::list_objects(p, database, object_types, mysql_limit, mysql_offset) + .await + .map(|result| ObjectListOutcome { objects: result.objects, paging_applied: result.paging_applied }) } } PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_questdb_config) => { - db::questdb::list_objects(p, schema).await + db::questdb::list_objects(p, schema).await.map(unpaged_object_list) } - PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await, + PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(unpaged_object_list), _ => { drop(connections); - Ok(list_tables_core(state, connection_id, database, schema, None, None, None, None) - .await? - .into_iter() - .map(|table| db::ObjectInfo { - name: table.name, - object_type: table.table_type, - schema: if schema.is_empty() { None } else { Some(schema.to_string()) }, - signature: None, - comment: table.comment, - created_at: None, - updated_at: None, - parent_schema: table.parent_schema, - parent_name: table.parent_name, - }) - .collect()) + Ok(unpaged_object_list( + list_tables_core(state, connection_id, database, schema, None, None, None, None) + .await? + .into_iter() + .map(|table| db::ObjectInfo { + name: table.name, + object_type: table.table_type, + schema: if schema.is_empty() { None } else { Some(schema.to_string()) }, + signature: None, + comment: table.comment, + created_at: None, + updated_at: None, + parent_schema: table.parent_schema, + parent_name: table.parent_name, + }) + .collect(), + )) } } } @@ -3826,8 +3857,8 @@ async fn list_completion_objects_once( PoolKind::Postgres(p) => db::postgres::list_objects(p, schema).await.map(filter_completion_objects), PoolKind::SqlServer(_) => { drop(connections); - let objects = list_objects_once(state, connection_id, database, schema, None, None, None, None).await?; - Ok(filter_completion_objects(objects)) + let outcome = list_objects_once(state, connection_id, database, schema, None, None, None, None).await?; + Ok(filter_completion_objects(outcome.objects)) } _ => Ok(Vec::new()), } diff --git a/crates/dbx-core/src/schema/providers/native.rs b/crates/dbx-core/src/schema/providers/native.rs index 77b0b39af8..b407c870ce 100644 --- a/crates/dbx-core/src/schema/providers/native.rs +++ b/crates/dbx-core/src/schema/providers/native.rs @@ -82,7 +82,9 @@ pub(in crate::schema) async fn list_objects( PoolKind::Mysql(p, _) if config.is_some_and(is_doris_family_config) => { db::mysql::list_table_objects_show(p, database).await.map(Some) } - PoolKind::Mysql(p, _) => db::mysql::list_objects(p, database).await.map(Some), + PoolKind::Mysql(p, _) => db::mysql::list_objects(p, database, None, None, None) + .await + .map(|result| Some(result.objects)), PoolKind::Postgres(p) if config.is_some_and(is_questdb_config) => { db::questdb::list_objects(p, schema).await.map(Some) }