fix(schema): filter JDBC table names before paging
This commit is contained in:
4 files changed
+134
-7
No files matched your search
@@ -0,0 +1,119 @@
|
||||
use super::{list_tables_core, TableNameFilter};
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
use crate::plugins::{
|
||||
InstalledPlugin, PluginCompatibility, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv,
|
||||
};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
async fn jdbc_state() -> (tempfile::TempDir, AppState) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let executable = dir.path().join("plugin.py");
|
||||
std::fs::write(
|
||||
&executable,
|
||||
r#"#!/usr/bin/env python3
|
||||
import json, pathlib, sys
|
||||
tables = [{'name': 'T%04d' % i, 'table_type': 'TABLE'} for i in range(1005)]
|
||||
for line in sys.stdin:
|
||||
request = json.loads(line)
|
||||
params = request.get('params', {})
|
||||
if request['method'] == 'listTables':
|
||||
with pathlib.Path(__file__).with_name('calls.jsonl').open('a') as log:
|
||||
log.write(json.dumps(params) + '\n')
|
||||
start = params.get('offset', 0)
|
||||
end = start + params['limit'] if params.get('limit', 0) > 0 else len(tables)
|
||||
result = tables[start:end]
|
||||
else:
|
||||
result = {}
|
||||
print(json.dumps({'id': request['id'], 'result': result}), flush=True)
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::set_permissions(&executable, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
let plugin = InstalledPlugin {
|
||||
manifest: PluginManifest {
|
||||
id: "jdbc".into(),
|
||||
name: "JDBC".into(),
|
||||
version: "test".into(),
|
||||
protocol_version: 1,
|
||||
executable: Some("plugin.py".into()),
|
||||
drivers: vec![PluginDriverManifest {
|
||||
id: "jdbc".into(),
|
||||
label: "JDBC".into(),
|
||||
kind: "external".into(),
|
||||
database_type: Some("jdbc".into()),
|
||||
}],
|
||||
..PluginManifest::default()
|
||||
},
|
||||
path: dir.path().to_path_buf(),
|
||||
compatibility: PluginCompatibility {
|
||||
compatible: true,
|
||||
backend_executable: Some(executable),
|
||||
..PluginCompatibility::default()
|
||||
},
|
||||
provenance: None,
|
||||
};
|
||||
let session = Arc::new(
|
||||
PluginDriverSession::start_for_test(plugin, "jdbc".into(), PluginRuntimeEnv::default()).await.unwrap(),
|
||||
);
|
||||
let state = AppState::new(crate::persistence::test_storage::open(&dir.path().join("dbx.db")).await.unwrap());
|
||||
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||
"id": "jdbc-tables", "name": "JDBC metadata test", "db_type": "jdbc",
|
||||
"host": "127.0.0.1", "port": 1521, "username": "test", "password": "",
|
||||
"database": "demo", "connection_string": "jdbc:oracle:thin:@127.0.0.1:1521/demo"
|
||||
}))
|
||||
.unwrap();
|
||||
state.configs.write().await.insert(config.id.clone(), config.clone());
|
||||
state
|
||||
.update_connection_pools(|pools| {
|
||||
pools.insert(
|
||||
config.id.clone(),
|
||||
PoolKind::ExternalDriver { driver_id: "jdbc".into(), config: Arc::new(config), session },
|
||||
);
|
||||
})
|
||||
.await;
|
||||
(dir, state)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn jdbc_table_filters_find_matches_beyond_the_plugin_page_and_page_after_filtering() {
|
||||
let (dir, state) = jdbc_state().await;
|
||||
let filter = TableNameFilter { include_patterns: vec!["T100%".into()], exclude_patterns: vec!["T1002".into()] };
|
||||
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1), None, Some(&filter))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T1001", "T1003"]);
|
||||
let calls = std::fs::read_to_string(dir.path().join("calls.jsonl")).unwrap();
|
||||
let params: serde_json::Value = serde_json::from_str(calls.lines().last().unwrap()).unwrap();
|
||||
assert!(params.get("limit").is_none());
|
||||
assert!(params.get("offset").is_none());
|
||||
state.shutdown(Duration::from_secs(1)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn jdbc_table_filters_apply_exclusions_before_pagination() {
|
||||
let (_dir, state) = jdbc_state().await;
|
||||
let filter = TableNameFilter { include_patterns: vec![], exclude_patterns: vec!["T000%".into()] };
|
||||
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1), None, Some(&filter))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T0011", "T0012"]);
|
||||
state.shutdown(Duration::from_secs(1)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn jdbc_table_filters_keep_plugin_paging_for_empty_filters() {
|
||||
let (dir, state) = jdbc_state().await;
|
||||
let filter = TableNameFilter { include_patterns: vec![" ".into()], exclude_patterns: vec![] };
|
||||
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1001), None, Some(&filter))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T1001", "T1002"]);
|
||||
let calls = std::fs::read_to_string(dir.path().join("calls.jsonl")).unwrap();
|
||||
let params: serde_json::Value = serde_json::from_str(calls.lines().last().unwrap()).unwrap();
|
||||
assert_eq!(params["limit"], 2);
|
||||
assert_eq!(params["offset"], 1001);
|
||||
state.shutdown(Duration::from_secs(1)).await;
|
||||
}
|
||||
@@ -18,6 +18,8 @@ use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
mod agent_pg_sequences;
|
||||
#[cfg(all(test, unix))]
|
||||
mod external_table_filter_tests;
|
||||
mod kingbase;
|
||||
mod mongodb_columns;
|
||||
pub mod plugin_metadata;
|
||||
@@ -2260,8 +2262,8 @@ async fn list_tables_once(
|
||||
let driver_id = driver_id.clone();
|
||||
let config = config.clone();
|
||||
let session = session.clone();
|
||||
let force_local_table_name_filter = table_name_filter.is_some_and(|filter| !filter.is_empty());
|
||||
if uses_presto_like_information_schema_tables(&config.db_type) {
|
||||
let force_local_table_name_filter = table_name_filter.is_some_and(|filter| !filter.is_empty());
|
||||
return external_driver_presto_like_tables(
|
||||
session,
|
||||
config.as_ref(),
|
||||
@@ -2282,11 +2284,15 @@ async fn list_tables_once(
|
||||
if let Some(object_types) = object_types {
|
||||
params["object_types"] = serde_json::json!(object_types);
|
||||
}
|
||||
if let Some(limit) = limit {
|
||||
params["limit"] = serde_json::json!(limit);
|
||||
}
|
||||
if let Some(offset) = offset {
|
||||
params["offset"] = serde_json::json!(offset);
|
||||
// Include/exclude name patterns are evaluated in core, so the plugin
|
||||
// must return the full candidate list before core applies pagination.
|
||||
if !force_local_table_name_filter {
|
||||
if let Some(limit) = limit {
|
||||
params["limit"] = serde_json::json!(limit);
|
||||
}
|
||||
if let Some(offset) = offset {
|
||||
params["offset"] = serde_json::json!(offset);
|
||||
}
|
||||
}
|
||||
return session
|
||||
.invoke_with_timeout::<Vec<db::TableInfo>>(
|
||||
@@ -2296,7 +2302,9 @@ async fn list_tables_once(
|
||||
)
|
||||
.await
|
||||
.map(|tables| {
|
||||
let final_offset = if external_driver_paging_likely_applied(&driver_id, limit, tables.len()) {
|
||||
let final_offset = if !force_local_table_name_filter
|
||||
&& external_driver_paging_likely_applied(&driver_id, limit, tables.len())
|
||||
{
|
||||
Some(0)
|
||||
} else {
|
||||
offset
|
||||
|
||||
Reference in new issue
Block a user