feat(es): paginate past 10k documents via pit and search_after

This commit is contained in:
Lan-zk authored and GitHub committed 2026-08-30 03:57:03 +08:00
1 parent 1d71a5fc23
commit 73a7b3f2c7
35 files changed
+1622 -153

No files matched your search

+28 -1
View File
@@ -61,6 +61,23 @@ pub async fn find_documents(
.map_err(easysearch_error)
}
pub async fn find_documents_with_cursor(
client: &EasysearchClient,
index: &str,
limit: i64,
filter: Option<&str>,
sort: Option<&str>,
cursor: Option<&str>,
) -> Result<DocumentQueryResult, String> {
elasticsearch_driver::find_documents_with_cursor(&client.inner, index, limit, filter, sort, cursor)
.await
.map_err(easysearch_error)
}
pub async fn close_cursor(client: &EasysearchClient, cursor: &str) -> Result<(), String> {
elasticsearch_driver::close_cursor(&client.inner, cursor).await.map_err(easysearch_error)
}
pub async fn count_documents(client: &EasysearchClient, index: &str, filter: Option<&str>) -> Result<u64, String> {
elasticsearch_driver::count_documents(&client.inner, index, filter).await.map_err(easysearch_error)
}
@@ -97,7 +114,17 @@ pub async fn delete_document(
}
pub async fn execute_rest_query(client: &EasysearchClient, input: &str) -> Result<QueryResult, String> {
elasticsearch_driver::execute_rest_query_with_sql_parser(&client.inner, input, parse_sql_response)
elasticsearch_driver::execute_rest_query_with_sql_parser(&client.inner, input, parse_sql_response, None)
.await
.map_err(easysearch_error)
}
pub async fn execute_rest_query_with_cursor(
client: &EasysearchClient,
input: &str,
cursor: Option<&str>,
) -> Result<QueryResult, String> {
elasticsearch_driver::execute_rest_query_with_sql_parser(&client.inner, input, parse_sql_response, cursor)
.await
.map_err(easysearch_error)
}
File diff suppressed because it is too large. Load diff
+11 -2
View File
@@ -424,6 +424,7 @@ pub async fn find_documents_core(
sort: Option<&str>,
collation: Option<&str>,
cursor: Option<&str>,
cursor_pagination: bool,
) -> Result<DocumentQueryResult, String> {
ensure_document_pool(state, connection_id).await?;
let connections = state.connections.read().await;
@@ -445,12 +446,20 @@ pub async fn find_documents_core(
PoolKind::Elasticsearch(client) => {
let client = client.clone();
drop(connections);
elasticsearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
if cursor_pagination {
elasticsearch_driver::find_documents_with_cursor(&client, collection, limit, filter, sort, cursor).await
} else {
elasticsearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
}
}
PoolKind::Easysearch(client) => {
let client = client.clone();
drop(connections);
easysearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
if cursor_pagination {
easysearch_driver::find_documents_with_cursor(&client, collection, limit, filter, sort, cursor).await
} else {
easysearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
}
}
PoolKind::Meilisearch(client) => {
let client = client.clone();
+1
View File
@@ -225,6 +225,7 @@ pub async fn mongo_find_documents_core(
sort,
collation,
None,
false,
)
.await
}
+22 -2
View File
@@ -1997,7 +1997,11 @@ async fn do_execute_typed(
let result = wait_for_query_opt(
cancel_token,
query_timeout,
db::elasticsearch_driver::execute_rest_query(&client, &sql),
db::elasticsearch_driver::execute_rest_query_with_cursor(
&client,
&sql,
options.result_session_id.as_deref(),
),
)
.await
.map(|result| truncate_result_with_max_rows(result, max_rows));
@@ -2014,7 +2018,11 @@ async fn do_execute_typed(
let result = wait_for_query_opt(
cancel_token,
query_timeout,
db::easysearch_driver::execute_rest_query(&client, &sql),
db::easysearch_driver::execute_rest_query_with_cursor(
&client,
&sql,
options.result_session_id.as_deref(),
),
)
.await
.map(|result| truncate_result_with_max_rows(result, max_rows));
@@ -2763,6 +2771,18 @@ pub async fn close_query_session(
.await
.map(|value| value.get("ok").and_then(|ok| ok.as_bool()).unwrap_or(false))
}
PoolKind::Elasticsearch(client) => {
let client = client.clone();
drop(connections);
db::elasticsearch_driver::close_cursor(&client, session_id).await?;
Ok(true)
}
PoolKind::Easysearch(client) => {
let client = client.clone();
drop(connections);
db::easysearch_driver::close_cursor(&client, session_id).await?;
Ok(true)
}
_ => Ok(false),
}
}