fix(schema): filter JDBC table names before paging

This commit is contained in:
zipg authored and GitHub committed 2026-10-02 09:04:02 +08:00
1 parent b16a16aadf
commit 99a8a5c318
4 files changed
+134 -7

No files matched your search

@@ -0,0 +1,119 @@
use super::{list_tables_core, TableNameFilter};
use crate::connection::{AppState, PoolKind};
use crate::models::connection::ConnectionConfig;
use crate::plugins::{
InstalledPlugin, PluginCompatibility, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv,
};
use std::os::unix::fs::PermissionsExt;
use std::sync::Arc;
use std::time::Duration;
async fn jdbc_state() -> (tempfile::TempDir, AppState) {
let dir = tempfile::tempdir().unwrap();
let executable = dir.path().join("plugin.py");
std::fs::write(
&executable,
r#"#!/usr/bin/env python3
import json, pathlib, sys
tables = [{'name': 'T%04d' % i, 'table_type': 'TABLE'} for i in range(1005)]
for line in sys.stdin:
request = json.loads(line)
params = request.get('params', {})
if request['method'] == 'listTables':
with pathlib.Path(__file__).with_name('calls.jsonl').open('a') as log:
log.write(json.dumps(params) + '\n')
start = params.get('offset', 0)
end = start + params['limit'] if params.get('limit', 0) > 0 else len(tables)
result = tables[start:end]
else:
result = {}
print(json.dumps({'id': request['id'], 'result': result}), flush=True)
"#,
)
.unwrap();
std::fs::set_permissions(&executable, std::fs::Permissions::from_mode(0o755)).unwrap();
let plugin = InstalledPlugin {
manifest: PluginManifest {
id: "jdbc".into(),
name: "JDBC".into(),
version: "test".into(),
protocol_version: 1,
executable: Some("plugin.py".into()),
drivers: vec![PluginDriverManifest {
id: "jdbc".into(),
label: "JDBC".into(),
kind: "external".into(),
database_type: Some("jdbc".into()),
}],
..PluginManifest::default()
},
path: dir.path().to_path_buf(),
compatibility: PluginCompatibility {
compatible: true,
backend_executable: Some(executable),
..PluginCompatibility::default()
},
provenance: None,
};
let session = Arc::new(
PluginDriverSession::start_for_test(plugin, "jdbc".into(), PluginRuntimeEnv::default()).await.unwrap(),
);
let state = AppState::new(crate::persistence::test_storage::open(&dir.path().join("dbx.db")).await.unwrap());
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
"id": "jdbc-tables", "name": "JDBC metadata test", "db_type": "jdbc",
"host": "127.0.0.1", "port": 1521, "username": "test", "password": "",
"database": "demo", "connection_string": "jdbc:oracle:thin:@127.0.0.1:1521/demo"
}))
.unwrap();
state.configs.write().await.insert(config.id.clone(), config.clone());
state
.update_connection_pools(|pools| {
pools.insert(
config.id.clone(),
PoolKind::ExternalDriver { driver_id: "jdbc".into(), config: Arc::new(config), session },
);
})
.await;
(dir, state)
}
#[tokio::test]
async fn jdbc_table_filters_find_matches_beyond_the_plugin_page_and_page_after_filtering() {
let (dir, state) = jdbc_state().await;
let filter = TableNameFilter { include_patterns: vec!["T100%".into()], exclude_patterns: vec!["T1002".into()] };
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1), None, Some(&filter))
.await
.unwrap();
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T1001", "T1003"]);
let calls = std::fs::read_to_string(dir.path().join("calls.jsonl")).unwrap();
let params: serde_json::Value = serde_json::from_str(calls.lines().last().unwrap()).unwrap();
assert!(params.get("limit").is_none());
assert!(params.get("offset").is_none());
state.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test]
async fn jdbc_table_filters_apply_exclusions_before_pagination() {
let (_dir, state) = jdbc_state().await;
let filter = TableNameFilter { include_patterns: vec![], exclude_patterns: vec!["T000%".into()] };
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1), None, Some(&filter))
.await
.unwrap();
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T0011", "T0012"]);
state.shutdown(Duration::from_secs(1)).await;
}
#[tokio::test]
async fn jdbc_table_filters_keep_plugin_paging_for_empty_filters() {
let (dir, state) = jdbc_state().await;
let filter = TableNameFilter { include_patterns: vec![" ".into()], exclude_patterns: vec![] };
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1001), None, Some(&filter))
.await
.unwrap();
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T1001", "T1002"]);
let calls = std::fs::read_to_string(dir.path().join("calls.jsonl")).unwrap();
let params: serde_json::Value = serde_json::from_str(calls.lines().last().unwrap()).unwrap();
assert_eq!(params["limit"], 2);
assert_eq!(params["offset"], 1001);
state.shutdown(Duration::from_secs(1)).await;
}
+15 -7
View File
@@ -18,6 +18,8 @@ use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
mod agent_pg_sequences;
#[cfg(all(test, unix))]
mod external_table_filter_tests;
mod kingbase;
mod mongodb_columns;
pub mod plugin_metadata;
@@ -2260,8 +2262,8 @@ async fn list_tables_once(
let driver_id = driver_id.clone();
let config = config.clone();
let session = session.clone();
let force_local_table_name_filter = table_name_filter.is_some_and(|filter| !filter.is_empty());
if uses_presto_like_information_schema_tables(&config.db_type) {
let force_local_table_name_filter = table_name_filter.is_some_and(|filter| !filter.is_empty());
return external_driver_presto_like_tables(
session,
config.as_ref(),
@@ -2282,11 +2284,15 @@ async fn list_tables_once(
if let Some(object_types) = object_types {
params["object_types"] = serde_json::json!(object_types);
}
if let Some(limit) = limit {
params["limit"] = serde_json::json!(limit);
}
if let Some(offset) = offset {
params["offset"] = serde_json::json!(offset);
// Include/exclude name patterns are evaluated in core, so the plugin
// must return the full candidate list before core applies pagination.
if !force_local_table_name_filter {
if let Some(limit) = limit {
params["limit"] = serde_json::json!(limit);
}
if let Some(offset) = offset {
params["offset"] = serde_json::json!(offset);
}
}
return session
.invoke_with_timeout::<Vec<db::TableInfo>>(
@@ -2296,7 +2302,9 @@ async fn list_tables_once(
)
.await
.map(|tables| {
let final_offset = if external_driver_paging_likely_applied(&driver_id, limit, tables.len()) {
let final_offset = if !force_local_table_name_filter
&& external_driver_paging_likely_applied(&driver_id, limit, tables.len())
{
Some(0)
} else {
offset