feat(redis): add stream consumer group monitoring
This commit is contained in:
1 parent
99cfa1b516
commit
31f0b7d6ef
23 files changed
+2092
-44
No files matched your search
@@ -15,9 +15,11 @@ use tokio::sync::{Mutex, MutexGuard};
|
||||
use super::json_value_for_js;
|
||||
|
||||
const STREAM_ENTRY_LIMIT: usize = 100;
|
||||
const STREAM_PENDING_PAGE_SIZE: usize = 100;
|
||||
const COLLECTION_PAGE_SIZE: usize = 200;
|
||||
const HASH_FILTER_SCAN_MAX_ITERATIONS: usize = 10;
|
||||
const DEFAULT_REDIS_DATABASES: u32 = 16;
|
||||
const JS_MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
|
||||
const CLUSTER_CURSOR_NODE_BITS: u64 = 16;
|
||||
const CLUSTER_CURSOR_NODE_MASK: u64 = (1 << CLUSTER_CURSOR_NODE_BITS) - 1;
|
||||
const CLUSTER_CURSOR_SCAN_MASK: u64 = (1 << (64 - CLUSTER_CURSOR_NODE_BITS)) - 1;
|
||||
@@ -122,6 +124,70 @@ pub struct RedisStreamEntry {
|
||||
pub fields: Vec<RedisStreamField>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RedisStreamGroup {
|
||||
pub name: RedisBlob,
|
||||
#[serde(serialize_with = "serialize_redis_u64_for_js")]
|
||||
pub consumers: u64,
|
||||
#[serde(serialize_with = "serialize_redis_u64_for_js")]
|
||||
pub pending: u64,
|
||||
pub last_delivered_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", serialize_with = "serialize_optional_redis_u64_for_js")]
|
||||
pub entries_read: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", serialize_with = "serialize_optional_redis_u64_for_js")]
|
||||
pub lag: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RedisStreamConsumer {
|
||||
pub name: RedisBlob,
|
||||
#[serde(serialize_with = "serialize_redis_u64_for_js")]
|
||||
pub pending: u64,
|
||||
#[serde(serialize_with = "serialize_redis_u64_for_js")]
|
||||
pub idle_ms: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", serialize_with = "serialize_optional_redis_u64_for_js")]
|
||||
pub inactive_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RedisStreamPendingEntry {
|
||||
pub id: String,
|
||||
pub consumer: RedisBlob,
|
||||
#[serde(serialize_with = "serialize_redis_u64_for_js")]
|
||||
pub idle_ms: u64,
|
||||
#[serde(serialize_with = "serialize_redis_u64_for_js")]
|
||||
pub deliveries: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RedisStreamPendingPage {
|
||||
pub entries: Vec<RedisStreamPendingEntry>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
|
||||
// JSON and Tauri IPC both deliver numbers to JavaScript, so preserve large Redis counters as text.
|
||||
fn serialize_redis_u64_for_js<S>(value: &u64, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
if *value > JS_MAX_SAFE_INTEGER {
|
||||
serializer.serialize_str(&value.to_string())
|
||||
} else {
|
||||
serializer.serialize_u64(*value)
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_optional_redis_u64_for_js<S>(value: &Option<u64>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match value {
|
||||
Some(value) => serialize_redis_u64_for_js(value, serializer),
|
||||
None => serializer.serialize_none(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum RedisValueData {
|
||||
@@ -1888,8 +1954,9 @@ fn redis_raw_value_to_command_arg(v: &RedisRawValue) -> Option<String> {
|
||||
/// Try to convert a RedisRawValue to a u64.
|
||||
fn redis_value_to_u64(v: &RedisRawValue) -> Option<u64> {
|
||||
match v {
|
||||
RedisRawValue::Int(i) => Some(*i as u64),
|
||||
RedisRawValue::Int(i) => u64::try_from(*i).ok(),
|
||||
RedisRawValue::BulkString(bytes) => std::str::from_utf8(bytes).ok().and_then(|s| s.parse().ok()),
|
||||
RedisRawValue::SimpleString(value) => value.parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -2358,6 +2425,64 @@ where
|
||||
Ok(parse_stream_entries(raw))
|
||||
}
|
||||
|
||||
pub async fn get_stream_groups<C>(con: &mut C, key: &[u8]) -> Result<Vec<RedisStreamGroup>, String>
|
||||
where
|
||||
C: ConnectionLike + Send + Sync + Unpin,
|
||||
{
|
||||
let raw: RedisRawValue =
|
||||
redis::cmd("XINFO").arg("GROUPS").arg(key).query_async(con).await.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(parse_stream_groups(raw))
|
||||
}
|
||||
|
||||
pub async fn get_stream_consumers<C>(con: &mut C, key: &[u8], group: &[u8]) -> Result<Vec<RedisStreamConsumer>, String>
|
||||
where
|
||||
C: ConnectionLike + Send + Sync + Unpin,
|
||||
{
|
||||
let raw: RedisRawValue =
|
||||
redis::cmd("XINFO").arg("CONSUMERS").arg(key).arg(group).query_async(con).await.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(parse_stream_consumers(raw))
|
||||
}
|
||||
|
||||
pub async fn get_stream_pending_page<C>(
|
||||
con: &mut C,
|
||||
key: &[u8],
|
||||
group: &[u8],
|
||||
cursor: Option<&str>,
|
||||
consumer: Option<&[u8]>,
|
||||
) -> Result<RedisStreamPendingPage, String>
|
||||
where
|
||||
C: ConnectionLike + Send + Sync + Unpin,
|
||||
{
|
||||
let cursor = cursor.filter(|value| !value.is_empty());
|
||||
let start = cursor.unwrap_or("-");
|
||||
// Redis before 6.2 has no exclusive XPENDING ranges. A cursor page needs
|
||||
// a duplicate candidate and a lookahead row to page without skipping IDs.
|
||||
let requested_count = STREAM_PENDING_PAGE_SIZE + 1 + usize::from(cursor.is_some());
|
||||
let mut command = redis::cmd("XPENDING");
|
||||
command.arg(key).arg(group).arg(start).arg("+").arg(requested_count);
|
||||
if let Some(consumer) = consumer {
|
||||
command.arg(consumer);
|
||||
}
|
||||
let raw: RedisRawValue = command.query_async(con).await.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut entries = parse_stream_pending_entries(raw);
|
||||
if let Some(cursor) = cursor {
|
||||
if entries.first().is_some_and(|entry| entry.id == cursor) {
|
||||
entries.remove(0);
|
||||
}
|
||||
}
|
||||
let next_cursor = if entries.len() > STREAM_PENDING_PAGE_SIZE {
|
||||
entries.truncate(STREAM_PENDING_PAGE_SIZE);
|
||||
entries.last().map(|entry| entry.id.clone())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(RedisStreamPendingPage { entries, next_cursor })
|
||||
}
|
||||
|
||||
fn parse_scan_keys(raw: RedisRawValue) -> Result<(u64, Vec<Vec<u8>>), String> {
|
||||
let RedisRawValue::Array(parts) = raw else {
|
||||
return Err("Invalid Redis SCAN response".to_string());
|
||||
@@ -2417,6 +2542,81 @@ fn parse_stream_entry(entry: RedisRawValue) -> Option<RedisStreamEntry> {
|
||||
Some(RedisStreamEntry { id, fields: parsed_fields })
|
||||
}
|
||||
|
||||
fn parse_stream_groups(raw: RedisRawValue) -> Vec<RedisStreamGroup> {
|
||||
match raw {
|
||||
RedisRawValue::Array(groups) => groups.into_iter().filter_map(parse_stream_group).collect(),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_stream_group(group: RedisRawValue) -> Option<RedisStreamGroup> {
|
||||
let mut attributes = parse_stream_info_attributes(group)?;
|
||||
Some(RedisStreamGroup {
|
||||
name: redis_value_to_blob(attributes.remove("name")?)?,
|
||||
consumers: redis_value_to_u64(&attributes.remove("consumers")?)?,
|
||||
pending: redis_value_to_u64(&attributes.remove("pending")?)?,
|
||||
last_delivered_id: redis_value_to_string(attributes.remove("last-delivered-id")?)?,
|
||||
entries_read: attributes.remove("entries-read").and_then(|value| redis_value_to_u64(&value)),
|
||||
lag: attributes.remove("lag").and_then(|value| redis_value_to_u64(&value)),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_stream_consumers(raw: RedisRawValue) -> Vec<RedisStreamConsumer> {
|
||||
match raw {
|
||||
RedisRawValue::Array(consumers) => consumers.into_iter().filter_map(parse_stream_consumer).collect(),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_stream_consumer(consumer: RedisRawValue) -> Option<RedisStreamConsumer> {
|
||||
let mut attributes = parse_stream_info_attributes(consumer)?;
|
||||
Some(RedisStreamConsumer {
|
||||
name: redis_value_to_blob(attributes.remove("name")?)?,
|
||||
pending: redis_value_to_u64(&attributes.remove("pending")?)?,
|
||||
idle_ms: redis_value_to_u64(&attributes.remove("idle")?)?,
|
||||
inactive_ms: attributes.remove("inactive").and_then(|value| redis_value_to_u64(&value)),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_stream_pending_entries(raw: RedisRawValue) -> Vec<RedisStreamPendingEntry> {
|
||||
match raw {
|
||||
RedisRawValue::Array(entries) => entries.into_iter().filter_map(parse_stream_pending_entry).collect(),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_stream_pending_entry(entry: RedisRawValue) -> Option<RedisStreamPendingEntry> {
|
||||
let mut parts = match entry {
|
||||
RedisRawValue::Array(parts) if parts.len() == 4 => parts.into_iter(),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
Some(RedisStreamPendingEntry {
|
||||
id: redis_value_to_string(parts.next()?)?,
|
||||
consumer: redis_value_to_blob(parts.next()?)?,
|
||||
idle_ms: redis_value_to_u64(&parts.next()?)?,
|
||||
deliveries: redis_value_to_u64(&parts.next()?)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_stream_info_attributes(raw: RedisRawValue) -> Option<HashMap<String, RedisRawValue>> {
|
||||
let RedisRawValue::Array(parts) = raw else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let mut attributes = HashMap::new();
|
||||
let mut parts = parts.into_iter();
|
||||
while let Some(name) = parts.next() {
|
||||
let value = parts.next()?;
|
||||
attributes.insert(redis_value_to_string(name)?, value);
|
||||
}
|
||||
Some(attributes)
|
||||
}
|
||||
|
||||
fn redis_value_to_blob(value: RedisRawValue) -> Option<RedisBlob> {
|
||||
redis_value_to_bytes(value).map(|bytes| redis_blob_from_bytes(&bytes))
|
||||
}
|
||||
|
||||
fn redis_value_to_string(value: RedisRawValue) -> Option<String> {
|
||||
match value {
|
||||
RedisRawValue::BulkString(bytes) => Some(redis_bytes_to_display(&bytes)),
|
||||
@@ -2975,13 +3175,14 @@ mod tests {
|
||||
use super::{
|
||||
classify_command, connection_info, decode_cluster_cursor, encode_cluster_cursor, is_redis_json_type,
|
||||
parse_cluster_slots, parse_command_argv, parse_database_count, parse_redis_endpoint, parse_scan_keys,
|
||||
parse_stream_entries, redis_auth_candidates, redis_blob_from_bytes, redis_cluster_slot,
|
||||
redis_command_raw_to_json, redis_database_index, redis_key_bytes_to_display, redis_key_bytes_to_raw,
|
||||
redis_key_matches_query, redis_key_raw_to_bytes, redis_key_value_preview, redis_sentinel_master_endpoint,
|
||||
redis_value_matches_query, redis_value_to_bytes, standalone_connection_infos, RedisAuthCandidate, RedisBlob,
|
||||
RedisBlobEncoding, RedisClusterSlotRange, RedisCollectionPage, RedisCommandSafety, RedisHashItem,
|
||||
RedisNodeEndpoint, RedisNodeRoute, RedisRawValue, RedisSetItem, RedisStreamEntry, RedisStreamField, RedisValue,
|
||||
RedisValueData,
|
||||
parse_stream_consumers, parse_stream_entries, parse_stream_groups, parse_stream_pending_entries,
|
||||
redis_auth_candidates, redis_blob_from_bytes, redis_cluster_slot, redis_command_raw_to_json,
|
||||
redis_database_index, redis_key_bytes_to_display, redis_key_bytes_to_raw, redis_key_matches_query,
|
||||
redis_key_raw_to_bytes, redis_key_value_preview, redis_sentinel_master_endpoint, redis_value_matches_query,
|
||||
redis_value_to_bytes, standalone_connection_infos, RedisAuthCandidate, RedisBlob, RedisBlobEncoding,
|
||||
RedisClusterSlotRange, RedisCollectionPage, RedisCommandSafety, RedisHashItem, RedisNodeEndpoint,
|
||||
RedisNodeRoute, RedisRawValue, RedisSetItem, RedisStreamConsumer, RedisStreamEntry, RedisStreamField,
|
||||
RedisStreamGroup, RedisStreamPendingEntry, RedisValue, RedisValueData,
|
||||
};
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
use redis::{aio::ConnectionLike, Cmd, ConnectionAddr, Pipeline, RedisFuture};
|
||||
@@ -3167,6 +3368,260 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_stream_groups_with_binary_names_and_compatible_optional_fields() {
|
||||
let raw = RedisRawValue::Array(vec![
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("name"),
|
||||
RedisRawValue::BulkString(vec![0xFF, b'g']),
|
||||
bulk("consumers"),
|
||||
RedisRawValue::Int(2),
|
||||
bulk("pending"),
|
||||
bulk("3"),
|
||||
bulk("last-delivered-id"),
|
||||
bulk("1714470000000-0"),
|
||||
bulk("entries-read"),
|
||||
RedisRawValue::Int(11),
|
||||
bulk("lag"),
|
||||
RedisRawValue::Nil,
|
||||
bulk("future-field"),
|
||||
bulk("ignored"),
|
||||
]),
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("name"),
|
||||
bulk("legacy-group"),
|
||||
bulk("consumers"),
|
||||
RedisRawValue::Int(0),
|
||||
bulk("pending"),
|
||||
RedisRawValue::Int(0),
|
||||
bulk("last-delivered-id"),
|
||||
bulk("0-0"),
|
||||
]),
|
||||
]);
|
||||
|
||||
let parsed = parse_stream_groups(raw);
|
||||
|
||||
assert_eq!(parsed.len(), 2);
|
||||
assert_eq!(parsed[0].name, redis_blob_from_bytes(&[0xFF, b'g']));
|
||||
assert_eq!(
|
||||
parsed[0],
|
||||
RedisStreamGroup {
|
||||
name: redis_blob_from_bytes(&[0xFF, b'g']),
|
||||
consumers: 2,
|
||||
pending: 3,
|
||||
last_delivered_id: "1714470000000-0".to_string(),
|
||||
entries_read: Some(11),
|
||||
lag: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parsed[1],
|
||||
RedisStreamGroup {
|
||||
name: text_blob("legacy-group"),
|
||||
consumers: 0,
|
||||
pending: 0,
|
||||
last_delivered_id: "0-0".to_string(),
|
||||
entries_read: None,
|
||||
lag: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_stream_consumers_and_pending_entries() {
|
||||
let consumers = parse_stream_consumers(RedisRawValue::Array(vec![
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("name"),
|
||||
bulk("worker-a"),
|
||||
bulk("pending"),
|
||||
RedisRawValue::Int(4),
|
||||
bulk("idle"),
|
||||
bulk("1200"),
|
||||
bulk("inactive"),
|
||||
RedisRawValue::Int(800),
|
||||
]),
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("name"),
|
||||
bulk("legacy-worker"),
|
||||
bulk("pending"),
|
||||
RedisRawValue::Int(0),
|
||||
bulk("idle"),
|
||||
RedisRawValue::Int(0),
|
||||
]),
|
||||
]));
|
||||
let pending = parse_stream_pending_entries(RedisRawValue::Array(vec![
|
||||
RedisRawValue::Array(vec![
|
||||
bulk("1714470000000-0"),
|
||||
RedisRawValue::BulkString(vec![0xFE, b'c']),
|
||||
RedisRawValue::Int(2_400),
|
||||
bulk("3"),
|
||||
]),
|
||||
RedisRawValue::Array(vec![bulk("malformed")]),
|
||||
]));
|
||||
|
||||
assert_eq!(
|
||||
consumers,
|
||||
vec![
|
||||
RedisStreamConsumer { name: text_blob("worker-a"), pending: 4, idle_ms: 1_200, inactive_ms: Some(800) },
|
||||
RedisStreamConsumer { name: text_blob("legacy-worker"), pending: 0, idle_ms: 0, inactive_ms: None },
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
pending,
|
||||
vec![RedisStreamPendingEntry {
|
||||
id: "1714470000000-0".to_string(),
|
||||
consumer: redis_blob_from_bytes(&[0xFE, b'c']),
|
||||
idle_ms: 2_400,
|
||||
deliveries: 3,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serializes_stream_metrics_without_losing_javascript_precision() {
|
||||
let unsafe_value = super::JS_MAX_SAFE_INTEGER + 1;
|
||||
let group = RedisStreamGroup {
|
||||
name: text_blob("payments"),
|
||||
consumers: unsafe_value,
|
||||
pending: 2,
|
||||
last_delivered_id: "1714470000000-0".to_string(),
|
||||
entries_read: Some(unsafe_value),
|
||||
lag: None,
|
||||
};
|
||||
let consumer = RedisStreamConsumer {
|
||||
name: text_blob("worker-a"),
|
||||
pending: 2,
|
||||
idle_ms: unsafe_value,
|
||||
inactive_ms: Some(unsafe_value),
|
||||
};
|
||||
let entry = RedisStreamPendingEntry {
|
||||
id: "1714470000000-0".to_string(),
|
||||
consumer: text_blob("worker-a"),
|
||||
idle_ms: 2,
|
||||
deliveries: unsafe_value,
|
||||
};
|
||||
|
||||
let group_json = serde_json::to_value(group).unwrap();
|
||||
let consumer_json = serde_json::to_value(consumer).unwrap();
|
||||
let entry_json = serde_json::to_value(entry).unwrap();
|
||||
|
||||
assert_eq!(group_json["consumers"], unsafe_value.to_string());
|
||||
assert_eq!(group_json["pending"].as_u64(), Some(2));
|
||||
assert_eq!(group_json["entries_read"], unsafe_value.to_string());
|
||||
assert!(group_json.get("lag").is_none());
|
||||
assert_eq!(consumer_json["idle_ms"], unsafe_value.to_string());
|
||||
assert_eq!(consumer_json["inactive_ms"], unsafe_value.to_string());
|
||||
assert_eq!(entry_json["deliveries"], unsafe_value.to_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_monitoring_commands_are_read_only_and_pending_pagination_supports_redis_5() {
|
||||
let groups = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||
bulk("name"),
|
||||
bulk("payments"),
|
||||
bulk("consumers"),
|
||||
RedisRawValue::Int(1),
|
||||
bulk("pending"),
|
||||
RedisRawValue::Int(101),
|
||||
bulk("last-delivered-id"),
|
||||
bulk("1714470000000-0"),
|
||||
])]);
|
||||
let consumers = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||
bulk("name"),
|
||||
bulk("worker-a"),
|
||||
bulk("pending"),
|
||||
RedisRawValue::Int(101),
|
||||
bulk("idle"),
|
||||
RedisRawValue::Int(10),
|
||||
])]);
|
||||
let pending = RedisRawValue::Array(
|
||||
(17..=118)
|
||||
.map(|index| {
|
||||
RedisRawValue::Array(vec![
|
||||
bulk(&format!("1714470000000-{index}")),
|
||||
bulk("worker-a"),
|
||||
RedisRawValue::Int(index),
|
||||
RedisRawValue::Int(1),
|
||||
])
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let mut con = FakeRedisConnection::new(vec![groups, consumers, pending]);
|
||||
|
||||
let groups = super::get_stream_groups(&mut con, b"orders").await.unwrap();
|
||||
let consumers = super::get_stream_consumers(&mut con, b"orders", b"payments").await.unwrap();
|
||||
let page = super::get_stream_pending_page(&mut con, b"orders", b"payments", Some("1714470000000-17"), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(consumers.len(), 1);
|
||||
assert_eq!(page.entries.len(), 100);
|
||||
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-18"));
|
||||
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-117"));
|
||||
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-117"));
|
||||
assert_eq!(con.command_count("XINFO"), 2);
|
||||
assert_eq!(con.command_count("XPENDING"), 1);
|
||||
assert_eq!(con.command_count("XGROUP"), 0);
|
||||
assert_eq!(con.command_count("XACK"), 0);
|
||||
assert_eq!(con.command_count("XCLAIM"), 0);
|
||||
assert!(con.commands[0].contains("\r\nGROUPS\r\n"));
|
||||
assert!(con.commands[1].contains("\r\nCONSUMERS\r\n"));
|
||||
assert!(con.commands[2].contains("\r\n1714470000000-17\r\n"));
|
||||
assert!(!con.commands[2].contains("\r\n(1714470000000-17\r\n"));
|
||||
assert!(con.commands[2].contains("\r\n102\r\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_pending_pagination_keeps_the_first_entry_when_cursor_is_acknowledged() {
|
||||
let pending = RedisRawValue::Array(
|
||||
(18..=118)
|
||||
.map(|index| {
|
||||
RedisRawValue::Array(vec![
|
||||
bulk(&format!("1714470000000-{index}")),
|
||||
bulk("worker-a"),
|
||||
RedisRawValue::Int(index),
|
||||
RedisRawValue::Int(1),
|
||||
])
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let mut con = FakeRedisConnection::new(vec![pending]);
|
||||
|
||||
let page = super::get_stream_pending_page(&mut con, b"orders", b"payments", Some("1714470000000-17"), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(page.entries.len(), 100);
|
||||
assert_eq!(page.entries.first().map(|entry| entry.id.as_str()), Some("1714470000000-18"));
|
||||
assert_eq!(page.entries.last().map(|entry| entry.id.as_str()), Some("1714470000000-117"));
|
||||
assert_eq!(page.next_cursor.as_deref(), Some("1714470000000-117"));
|
||||
assert!(con.commands[0].contains("\r\n1714470000000-17\r\n"));
|
||||
assert!(!con.commands[0].contains("\r\n(1714470000000-17\r\n"));
|
||||
assert!(con.commands[0].contains("\r\n102\r\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_pending_filters_by_consumer_server_side() {
|
||||
let pending = RedisRawValue::Array(vec![RedisRawValue::Array(vec![
|
||||
bulk("1714470000000-0"),
|
||||
bulk("worker-a"),
|
||||
RedisRawValue::Int(2_400),
|
||||
RedisRawValue::Int(1),
|
||||
])]);
|
||||
let mut con = FakeRedisConnection::new(vec![pending]);
|
||||
|
||||
let page =
|
||||
super::get_stream_pending_page(&mut con, b"orders", b"payments", None, Some(b"worker-a")).await.unwrap();
|
||||
|
||||
assert_eq!(page.entries.len(), 1);
|
||||
assert_eq!(con.command_count("XPENDING"), 1);
|
||||
assert!(con.commands[0].contains("\r\n-\r\n"));
|
||||
assert!(con.commands[0].contains("\r\n+\r\n"));
|
||||
assert!(con.commands[0].contains("\r\n101\r\n"));
|
||||
assert!(con.commands[0].contains("\r\nworker-a\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_configured_database_count() {
|
||||
let value = RedisRawValue::Array(vec![
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::redis_driver::{
|
||||
self, RedisCollectionPage, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult, RedisValue,
|
||||
self, RedisCollectionPage, RedisCommandResult, RedisConnection, RedisDatabaseInfo, RedisScanResult,
|
||||
RedisStreamConsumer, RedisStreamGroup, RedisStreamPendingPage, RedisValue,
|
||||
};
|
||||
|
||||
async fn ensure_redis_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
|
||||
@@ -136,6 +137,100 @@ pub async fn redis_get_value_in_db_core(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_stream_groups_in_db_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
db: u32,
|
||||
key_raw: &str,
|
||||
) -> Result<Vec<RedisStreamGroup>, String> {
|
||||
ensure_redis_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(redis) => {
|
||||
let key = redis_driver::redis_key_raw_to_bytes(key_raw)?;
|
||||
match redis {
|
||||
RedisConnection::Direct(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::select_db(&mut *con, db).await?;
|
||||
redis_driver::get_stream_groups(&mut *con, &key).await
|
||||
}
|
||||
RedisConnection::Cluster(cluster) => {
|
||||
redis_driver::ensure_cluster_db(db)?;
|
||||
let mut con = redis_driver::cluster_key_connection(cluster, &key).await?;
|
||||
redis_driver::get_stream_groups(&mut con, &key).await
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_stream_consumers_in_db_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
db: u32,
|
||||
key_raw: &str,
|
||||
group_raw: &str,
|
||||
) -> Result<Vec<RedisStreamConsumer>, String> {
|
||||
ensure_redis_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(redis) => {
|
||||
let key = redis_driver::redis_key_raw_to_bytes(key_raw)?;
|
||||
let group = redis_driver::redis_key_raw_to_bytes(group_raw)?;
|
||||
match redis {
|
||||
RedisConnection::Direct(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::select_db(&mut *con, db).await?;
|
||||
redis_driver::get_stream_consumers(&mut *con, &key, &group).await
|
||||
}
|
||||
RedisConnection::Cluster(cluster) => {
|
||||
redis_driver::ensure_cluster_db(db)?;
|
||||
let mut con = redis_driver::cluster_key_connection(cluster, &key).await?;
|
||||
redis_driver::get_stream_consumers(&mut con, &key, &group).await
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_stream_pending_in_db_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
db: u32,
|
||||
key_raw: &str,
|
||||
group_raw: &str,
|
||||
cursor: Option<&str>,
|
||||
consumer_raw: Option<&str>,
|
||||
) -> Result<RedisStreamPendingPage, String> {
|
||||
ensure_redis_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
match pool {
|
||||
PoolKind::Redis(redis) => {
|
||||
let key = redis_driver::redis_key_raw_to_bytes(key_raw)?;
|
||||
let group = redis_driver::redis_key_raw_to_bytes(group_raw)?;
|
||||
let consumer = consumer_raw.map(redis_driver::redis_key_raw_to_bytes).transpose()?;
|
||||
match redis {
|
||||
RedisConnection::Direct(con) => {
|
||||
let mut con = con.lock().await;
|
||||
redis_driver::select_db(&mut *con, db).await?;
|
||||
redis_driver::get_stream_pending_page(&mut *con, &key, &group, cursor, consumer.as_deref()).await
|
||||
}
|
||||
RedisConnection::Cluster(cluster) => {
|
||||
redis_driver::ensure_cluster_db(db)?;
|
||||
let mut con = redis_driver::cluster_key_connection(cluster, &key).await?;
|
||||
redis_driver::get_stream_pending_page(&mut con, &key, &group, cursor, consumer.as_deref()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err("Not a Redis connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn redis_set_string_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
|
||||
Reference in new issue
Block a user