feat(es): paginate past 10k documents via pit and search_after
This commit is contained in:
1 parent
1d71a5fc23
commit
73a7b3f2c7
35 files changed
+1622
-153
No files matched your search
@@ -61,6 +61,23 @@ pub async fn find_documents(
|
||||
.map_err(easysearch_error)
|
||||
}
|
||||
|
||||
pub async fn find_documents_with_cursor(
|
||||
client: &EasysearchClient,
|
||||
index: &str,
|
||||
limit: i64,
|
||||
filter: Option<&str>,
|
||||
sort: Option<&str>,
|
||||
cursor: Option<&str>,
|
||||
) -> Result<DocumentQueryResult, String> {
|
||||
elasticsearch_driver::find_documents_with_cursor(&client.inner, index, limit, filter, sort, cursor)
|
||||
.await
|
||||
.map_err(easysearch_error)
|
||||
}
|
||||
|
||||
pub async fn close_cursor(client: &EasysearchClient, cursor: &str) -> Result<(), String> {
|
||||
elasticsearch_driver::close_cursor(&client.inner, cursor).await.map_err(easysearch_error)
|
||||
}
|
||||
|
||||
pub async fn count_documents(client: &EasysearchClient, index: &str, filter: Option<&str>) -> Result<u64, String> {
|
||||
elasticsearch_driver::count_documents(&client.inner, index, filter).await.map_err(easysearch_error)
|
||||
}
|
||||
@@ -97,7 +114,17 @@ pub async fn delete_document(
|
||||
}
|
||||
|
||||
pub async fn execute_rest_query(client: &EasysearchClient, input: &str) -> Result<QueryResult, String> {
|
||||
elasticsearch_driver::execute_rest_query_with_sql_parser(&client.inner, input, parse_sql_response)
|
||||
elasticsearch_driver::execute_rest_query_with_sql_parser(&client.inner, input, parse_sql_response, None)
|
||||
.await
|
||||
.map_err(easysearch_error)
|
||||
}
|
||||
|
||||
pub async fn execute_rest_query_with_cursor(
|
||||
client: &EasysearchClient,
|
||||
input: &str,
|
||||
cursor: Option<&str>,
|
||||
) -> Result<QueryResult, String> {
|
||||
elasticsearch_driver::execute_rest_query_with_sql_parser(&client.inner, input, parse_sql_response, cursor)
|
||||
.await
|
||||
.map_err(easysearch_error)
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -424,6 +424,7 @@ pub async fn find_documents_core(
|
||||
sort: Option<&str>,
|
||||
collation: Option<&str>,
|
||||
cursor: Option<&str>,
|
||||
cursor_pagination: bool,
|
||||
) -> Result<DocumentQueryResult, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
@@ -445,12 +446,20 @@ pub async fn find_documents_core(
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
|
||||
if cursor_pagination {
|
||||
elasticsearch_driver::find_documents_with_cursor(&client, collection, limit, filter, sort, cursor).await
|
||||
} else {
|
||||
elasticsearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
|
||||
}
|
||||
}
|
||||
PoolKind::Easysearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
easysearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
|
||||
if cursor_pagination {
|
||||
easysearch_driver::find_documents_with_cursor(&client, collection, limit, filter, sort, cursor).await
|
||||
} else {
|
||||
easysearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
|
||||
}
|
||||
}
|
||||
PoolKind::Meilisearch(client) => {
|
||||
let client = client.clone();
|
||||
|
||||
@@ -225,6 +225,7 @@ pub async fn mongo_find_documents_core(
|
||||
sort,
|
||||
collation,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1997,7 +1997,11 @@ async fn do_execute_typed(
|
||||
let result = wait_for_query_opt(
|
||||
cancel_token,
|
||||
query_timeout,
|
||||
db::elasticsearch_driver::execute_rest_query(&client, &sql),
|
||||
db::elasticsearch_driver::execute_rest_query_with_cursor(
|
||||
&client,
|
||||
&sql,
|
||||
options.result_session_id.as_deref(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map(|result| truncate_result_with_max_rows(result, max_rows));
|
||||
@@ -2014,7 +2018,11 @@ async fn do_execute_typed(
|
||||
let result = wait_for_query_opt(
|
||||
cancel_token,
|
||||
query_timeout,
|
||||
db::easysearch_driver::execute_rest_query(&client, &sql),
|
||||
db::easysearch_driver::execute_rest_query_with_cursor(
|
||||
&client,
|
||||
&sql,
|
||||
options.result_session_id.as_deref(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map(|result| truncate_result_with_max_rows(result, max_rows));
|
||||
@@ -2763,6 +2771,18 @@ pub async fn close_query_session(
|
||||
.await
|
||||
.map(|value| value.get("ok").and_then(|ok| ok.as_bool()).unwrap_or(false))
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
db::elasticsearch_driver::close_cursor(&client, session_id).await?;
|
||||
Ok(true)
|
||||
}
|
||||
PoolKind::Easysearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
db::easysearch_driver::close_cursor(&client, session_id).await?;
|
||||
Ok(true)
|
||||
}
|
||||
_ => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user