fix(schema): filter JDBC table names before paging
This commit is contained in:
4 files changed
+134
-7
No files matched your search
@@ -0,0 +1,119 @@
|
|||||||
|
use super::{list_tables_core, TableNameFilter};
|
||||||
|
use crate::connection::{AppState, PoolKind};
|
||||||
|
use crate::models::connection::ConnectionConfig;
|
||||||
|
use crate::plugins::{
|
||||||
|
InstalledPlugin, PluginCompatibility, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv,
|
||||||
|
};
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
async fn jdbc_state() -> (tempfile::TempDir, AppState) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let executable = dir.path().join("plugin.py");
|
||||||
|
std::fs::write(
|
||||||
|
&executable,
|
||||||
|
r#"#!/usr/bin/env python3
|
||||||
|
import json, pathlib, sys
|
||||||
|
tables = [{'name': 'T%04d' % i, 'table_type': 'TABLE'} for i in range(1005)]
|
||||||
|
for line in sys.stdin:
|
||||||
|
request = json.loads(line)
|
||||||
|
params = request.get('params', {})
|
||||||
|
if request['method'] == 'listTables':
|
||||||
|
with pathlib.Path(__file__).with_name('calls.jsonl').open('a') as log:
|
||||||
|
log.write(json.dumps(params) + '\n')
|
||||||
|
start = params.get('offset', 0)
|
||||||
|
end = start + params['limit'] if params.get('limit', 0) > 0 else len(tables)
|
||||||
|
result = tables[start:end]
|
||||||
|
else:
|
||||||
|
result = {}
|
||||||
|
print(json.dumps({'id': request['id'], 'result': result}), flush=True)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::set_permissions(&executable, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||||
|
let plugin = InstalledPlugin {
|
||||||
|
manifest: PluginManifest {
|
||||||
|
id: "jdbc".into(),
|
||||||
|
name: "JDBC".into(),
|
||||||
|
version: "test".into(),
|
||||||
|
protocol_version: 1,
|
||||||
|
executable: Some("plugin.py".into()),
|
||||||
|
drivers: vec![PluginDriverManifest {
|
||||||
|
id: "jdbc".into(),
|
||||||
|
label: "JDBC".into(),
|
||||||
|
kind: "external".into(),
|
||||||
|
database_type: Some("jdbc".into()),
|
||||||
|
}],
|
||||||
|
..PluginManifest::default()
|
||||||
|
},
|
||||||
|
path: dir.path().to_path_buf(),
|
||||||
|
compatibility: PluginCompatibility {
|
||||||
|
compatible: true,
|
||||||
|
backend_executable: Some(executable),
|
||||||
|
..PluginCompatibility::default()
|
||||||
|
},
|
||||||
|
provenance: None,
|
||||||
|
};
|
||||||
|
let session = Arc::new(
|
||||||
|
PluginDriverSession::start_for_test(plugin, "jdbc".into(), PluginRuntimeEnv::default()).await.unwrap(),
|
||||||
|
);
|
||||||
|
let state = AppState::new(crate::persistence::test_storage::open(&dir.path().join("dbx.db")).await.unwrap());
|
||||||
|
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": "jdbc-tables", "name": "JDBC metadata test", "db_type": "jdbc",
|
||||||
|
"host": "127.0.0.1", "port": 1521, "username": "test", "password": "",
|
||||||
|
"database": "demo", "connection_string": "jdbc:oracle:thin:@127.0.0.1:1521/demo"
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
state.configs.write().await.insert(config.id.clone(), config.clone());
|
||||||
|
state
|
||||||
|
.update_connection_pools(|pools| {
|
||||||
|
pools.insert(
|
||||||
|
config.id.clone(),
|
||||||
|
PoolKind::ExternalDriver { driver_id: "jdbc".into(), config: Arc::new(config), session },
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
(dir, state)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn jdbc_table_filters_find_matches_beyond_the_plugin_page_and_page_after_filtering() {
|
||||||
|
let (dir, state) = jdbc_state().await;
|
||||||
|
let filter = TableNameFilter { include_patterns: vec!["T100%".into()], exclude_patterns: vec!["T1002".into()] };
|
||||||
|
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1), None, Some(&filter))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T1001", "T1003"]);
|
||||||
|
let calls = std::fs::read_to_string(dir.path().join("calls.jsonl")).unwrap();
|
||||||
|
let params: serde_json::Value = serde_json::from_str(calls.lines().last().unwrap()).unwrap();
|
||||||
|
assert!(params.get("limit").is_none());
|
||||||
|
assert!(params.get("offset").is_none());
|
||||||
|
state.shutdown(Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn jdbc_table_filters_apply_exclusions_before_pagination() {
|
||||||
|
let (_dir, state) = jdbc_state().await;
|
||||||
|
let filter = TableNameFilter { include_patterns: vec![], exclude_patterns: vec!["T000%".into()] };
|
||||||
|
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1), None, Some(&filter))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T0011", "T0012"]);
|
||||||
|
state.shutdown(Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn jdbc_table_filters_keep_plugin_paging_for_empty_filters() {
|
||||||
|
let (dir, state) = jdbc_state().await;
|
||||||
|
let filter = TableNameFilter { include_patterns: vec![" ".into()], exclude_patterns: vec![] };
|
||||||
|
let tables = list_tables_core(&state, "jdbc-tables", "demo", "APP", None, Some(2), Some(1001), None, Some(&filter))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(tables.iter().map(|table| table.name.as_str()).collect::<Vec<_>>(), ["T1001", "T1002"]);
|
||||||
|
let calls = std::fs::read_to_string(dir.path().join("calls.jsonl")).unwrap();
|
||||||
|
let params: serde_json::Value = serde_json::from_str(calls.lines().last().unwrap()).unwrap();
|
||||||
|
assert_eq!(params["limit"], 2);
|
||||||
|
assert_eq!(params["offset"], 1001);
|
||||||
|
state.shutdown(Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
@@ -18,6 +18,8 @@ use std::sync::{Arc, OnceLock};
|
|||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
mod agent_pg_sequences;
|
mod agent_pg_sequences;
|
||||||
|
#[cfg(all(test, unix))]
|
||||||
|
mod external_table_filter_tests;
|
||||||
mod kingbase;
|
mod kingbase;
|
||||||
mod mongodb_columns;
|
mod mongodb_columns;
|
||||||
pub mod plugin_metadata;
|
pub mod plugin_metadata;
|
||||||
@@ -2260,8 +2262,8 @@ async fn list_tables_once(
|
|||||||
let driver_id = driver_id.clone();
|
let driver_id = driver_id.clone();
|
||||||
let config = config.clone();
|
let config = config.clone();
|
||||||
let session = session.clone();
|
let session = session.clone();
|
||||||
|
let force_local_table_name_filter = table_name_filter.is_some_and(|filter| !filter.is_empty());
|
||||||
if uses_presto_like_information_schema_tables(&config.db_type) {
|
if uses_presto_like_information_schema_tables(&config.db_type) {
|
||||||
let force_local_table_name_filter = table_name_filter.is_some_and(|filter| !filter.is_empty());
|
|
||||||
return external_driver_presto_like_tables(
|
return external_driver_presto_like_tables(
|
||||||
session,
|
session,
|
||||||
config.as_ref(),
|
config.as_ref(),
|
||||||
@@ -2282,11 +2284,15 @@ async fn list_tables_once(
|
|||||||
if let Some(object_types) = object_types {
|
if let Some(object_types) = object_types {
|
||||||
params["object_types"] = serde_json::json!(object_types);
|
params["object_types"] = serde_json::json!(object_types);
|
||||||
}
|
}
|
||||||
if let Some(limit) = limit {
|
// Include/exclude name patterns are evaluated in core, so the plugin
|
||||||
params["limit"] = serde_json::json!(limit);
|
// must return the full candidate list before core applies pagination.
|
||||||
}
|
if !force_local_table_name_filter {
|
||||||
if let Some(offset) = offset {
|
if let Some(limit) = limit {
|
||||||
params["offset"] = serde_json::json!(offset);
|
params["limit"] = serde_json::json!(limit);
|
||||||
|
}
|
||||||
|
if let Some(offset) = offset {
|
||||||
|
params["offset"] = serde_json::json!(offset);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return session
|
return session
|
||||||
.invoke_with_timeout::<Vec<db::TableInfo>>(
|
.invoke_with_timeout::<Vec<db::TableInfo>>(
|
||||||
@@ -2296,7 +2302,9 @@ async fn list_tables_once(
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map(|tables| {
|
.map(|tables| {
|
||||||
let final_offset = if external_driver_paging_likely_applied(&driver_id, limit, tables.len()) {
|
let final_offset = if !force_local_table_name_filter
|
||||||
|
&& external_driver_paging_likely_applied(&driver_id, limit, tables.len())
|
||||||
|
{
|
||||||
Some(0)
|
Some(0)
|
||||||
} else {
|
} else {
|
||||||
offset
|
offset
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 73 KiB |
Reference in new issue
Block a user