feat(redis): add hash field search
This commit is contained in:
1 parent
2c33570351
commit
d502b4ce6c
13 files changed
+176
-23
No files matched your search
@@ -15,6 +15,7 @@ use super::json_value_for_js;
|
||||
|
||||
const STREAM_ENTRY_LIMIT: usize = 100;
|
||||
const COLLECTION_PAGE_SIZE: usize = 200;
|
||||
const HASH_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
|
||||
const DEFAULT_REDIS_DATABASES: u32 = 16;
|
||||
const CLUSTER_CURSOR_NODE_BITS: u64 = 16;
|
||||
const CLUSTER_CURSOR_NODE_MASK: u64 = (1 << CLUSTER_CURSOR_NODE_BITS) - 1;
|
||||
@@ -1758,7 +1759,7 @@ where
|
||||
}
|
||||
"hash" => {
|
||||
let len: u64 = redis::cmd("HLEN").arg(key).query_async(con).await.unwrap_or(0);
|
||||
let (next_cursor, items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE).await?;
|
||||
let (next_cursor, items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
|
||||
let cursor = if next_cursor > 0 { Some(next_cursor) } else { None };
|
||||
(serde_json::Value::Array(items), false, Some(len), cursor)
|
||||
}
|
||||
@@ -2204,6 +2205,7 @@ pub async fn load_more_collection<C>(
|
||||
key_type: &str,
|
||||
cursor: u64,
|
||||
count: usize,
|
||||
match_pattern: Option<&str>,
|
||||
) -> Result<RedisValue, String>
|
||||
where
|
||||
C: ConnectionLike + Send + Sync + Unpin,
|
||||
@@ -2230,7 +2232,11 @@ where
|
||||
(serde_json::Value::Array(items), cursor)
|
||||
}
|
||||
"hash" => {
|
||||
let (next, items) = hscan_page_raw(con, key, cursor, count).await?;
|
||||
let (next, items) = if let Some(pattern) = match_pattern {
|
||||
hscan_matching_page_raw(con, key, cursor, count, pattern).await?
|
||||
} else {
|
||||
hscan_page_raw(con, key, cursor, count, None).await?
|
||||
};
|
||||
let cursor = if next > 0 { Some(next) } else { None };
|
||||
(serde_json::Value::Array(items), cursor)
|
||||
}
|
||||
@@ -2254,21 +2260,48 @@ async fn hscan_page_raw<C>(
|
||||
key: &[u8],
|
||||
cursor: u64,
|
||||
count: usize,
|
||||
match_pattern: Option<&str>,
|
||||
) -> Result<(u64, Vec<serde_json::Value>), String>
|
||||
where
|
||||
C: ConnectionLike + Send + Sync + Unpin,
|
||||
{
|
||||
let raw: RedisRawValue = redis::cmd("HSCAN")
|
||||
.arg(key)
|
||||
.arg(cursor)
|
||||
.arg("COUNT")
|
||||
.arg(count)
|
||||
.query_async(con)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut cmd = redis::cmd("HSCAN");
|
||||
cmd.arg(key).arg(cursor).arg("COUNT").arg(count);
|
||||
if let Some(pattern) = match_pattern {
|
||||
cmd.arg("MATCH").arg(pattern);
|
||||
}
|
||||
let raw: RedisRawValue = cmd.query_async(con).await.map_err(|e| e.to_string())?;
|
||||
parse_scan_pairs(raw, "hash")
|
||||
}
|
||||
|
||||
async fn hscan_matching_page_raw<C>(
|
||||
con: &mut C,
|
||||
key: &[u8],
|
||||
cursor: u64,
|
||||
count: usize,
|
||||
pattern: &str,
|
||||
) -> Result<(u64, Vec<serde_json::Value>), String>
|
||||
where
|
||||
C: ConnectionLike + Send + Sync + Unpin,
|
||||
{
|
||||
let mut cur = cursor;
|
||||
let mut items = Vec::new();
|
||||
let target = count.max(1);
|
||||
|
||||
for _ in 0..HASH_FILTER_SCAN_MAX_ITERATIONS {
|
||||
let (next, page) = hscan_page_raw(con, key, cur, target, Some(pattern)).await?;
|
||||
items.extend(page);
|
||||
cur = next;
|
||||
// Redis applies MATCH during incremental scans and may return empty pages.
|
||||
// Stop only after a bounded number of chunks so sparse matches progress without an unbounded full-hash scan.
|
||||
if cur == 0 || items.len() >= target {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((cur, items))
|
||||
}
|
||||
|
||||
async fn sscan_page_raw<C>(
|
||||
con: &mut C,
|
||||
key: &[u8],
|
||||
@@ -2430,6 +2463,11 @@ mod tests {
|
||||
])
|
||||
}
|
||||
|
||||
fn hscan_response(cursor: &str, pairs: Vec<(&str, &str)>) -> RedisRawValue {
|
||||
let entries = pairs.into_iter().flat_map(|(field, value)| [bulk(field), bulk(value)]).collect();
|
||||
RedisRawValue::Array(vec![bulk(cursor), RedisRawValue::Array(entries)])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_stream_entries() {
|
||||
let raw = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||
@@ -2560,6 +2598,36 @@ mod tests {
|
||||
assert_eq!(con.command_count("SCAN"), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn filtered_hash_load_more_keeps_scan_cursor_instead_of_full_iteration() {
|
||||
let mut con = FakeRedisConnection::new(vec![
|
||||
hscan_response("512", vec![("user:1", "Ada")]),
|
||||
hscan_response("0", vec![("user:2", "Bob")]),
|
||||
]);
|
||||
|
||||
let result = super::load_more_collection(&mut con, b"hash-key", "hash", 0, 1, Some("*user*")).await.unwrap();
|
||||
|
||||
assert_eq!(result.scan_cursor, Some(512));
|
||||
assert_eq!(result.value, serde_json::json!([{ "field": "user:1", "value": "Ada" }]));
|
||||
assert_eq!(con.command_count("HSCAN"), 1);
|
||||
assert!(con.commands[0].contains("\r\nMATCH\r\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn filtered_hash_load_more_caps_sparse_scan_iterations() {
|
||||
let responses = (1..=super::HASH_FILTER_SCAN_MAX_ITERATIONS + 1)
|
||||
.map(|cursor| hscan_response(&cursor.to_string(), vec![]))
|
||||
.collect();
|
||||
let mut con = FakeRedisConnection::new(responses);
|
||||
|
||||
let result =
|
||||
super::load_more_collection(&mut con, b"hash-key", "hash", 0, 20, Some("*missing*")).await.unwrap();
|
||||
|
||||
assert_eq!(result.scan_cursor, Some(super::HASH_FILTER_SCAN_MAX_ITERATIONS as u64));
|
||||
assert_eq!(result.value, serde_json::json!([]));
|
||||
assert_eq!(con.command_count("HSCAN"), super::HASH_FILTER_SCAN_MAX_ITERATIONS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_binary_string_values_like_rdm() {
|
||||
let raw = RedisRawValue::BulkString(vec![0xAC, 0xED, 0x00, 0x05, b's', b'r']);
|
||||
|
||||
@@ -789,6 +789,7 @@ pub async fn redis_load_more_in_db_core(
|
||||
key_type: &str,
|
||||
cursor: u64,
|
||||
count: usize,
|
||||
filter: Option<&str>,
|
||||
) -> Result<redis_driver::RedisValue, String> {
|
||||
ensure_redis_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
@@ -799,12 +800,12 @@ pub async fn redis_load_more_in_db_core(
|
||||
RedisConnection::Direct(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::select_db(&mut *con, db).await?;
|
||||
redis_driver::load_more_collection(&mut *con, &key, key_type, cursor, count).await
|
||||
redis_driver::load_more_collection(&mut *con, &key, key_type, cursor, count, filter).await
|
||||
}
|
||||
RedisConnection::Cluster(cluster) => {
|
||||
redis_driver::ensure_cluster_db(db)?;
|
||||
let mut con = redis_driver::cluster_key_connection(cluster, &key).await?;
|
||||
redis_driver::load_more_collection(&mut con, &key, key_type, cursor, count).await
|
||||
redis_driver::load_more_collection(&mut con, &key, key_type, cursor, count, filter).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user