feat(transfer): show per-table failure details

This commit is contained in:
zipg authored and GitHub committed 2026-07-28 11:47:58 +08:00
1 parent 44cb4acbb7
commit 1e72d28ed3
20 files changed
+622 -65

No files matched your search

+1
View File
@@ -233,6 +233,7 @@ async fn main() {
password_hash: RwLock::new(password_hash),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
transfer_progress_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
nacos_imports: RwLock::new(HashMap::new()),
+1
View File
@@ -494,6 +494,7 @@ mod tests {
password_hash: RwLock::new(None),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
transfer_progress_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
nacos_imports: RwLock::new(HashMap::new()),
+1
View File
@@ -834,6 +834,7 @@ mod tests {
password_hash: RwLock::new(None),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
transfer_progress_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
nacos_imports: RwLock::new(HashMap::new()),
+1
View File
@@ -1540,6 +1540,7 @@ mod tests {
password_hash: RwLock::new(None),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
transfer_progress_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
nacos_imports: RwLock::new(HashMap::new()),
+82 -63
View File
@@ -6,10 +6,55 @@ use axum::Json;
use dbx_core::transfer::{self, TransferRequest, TransferStatus};
use futures::stream::Stream;
use serde::Deserialize;
use tokio::time::{sleep, Duration};
use crate::error::AppError;
use crate::sse::{TransferProgressChannel, TransferReplayEventKind};
use crate::state::WebState;
const COMPLETED_TRANSFER_CHANNEL_TTL: Duration = Duration::from_secs(60);
fn send_transfer_progress(channel: &TransferProgressChannel, progress: &transfer::TransferProgress) {
if let Ok(json) = serde_json::to_string(progress) {
let kind = if progress.terminal {
TransferReplayEventKind::Terminal
} else if matches!(&progress.status, TransferStatus::Error) {
TransferReplayEventKind::Failure
} else {
TransferReplayEventKind::Progress
};
channel.send(json, kind);
}
}
fn terminal_transfer_error(req: &TransferRequest, error: impl ToString) -> transfer::TransferProgress {
transfer::TransferProgress {
transfer_id: req.transfer_id.clone(),
table: String::new(),
table_index: 0,
total_tables: req.tables.len(),
rows_transferred: 0,
total_rows: None,
status: TransferStatus::Error,
error: Some(error.to_string()),
terminal: true,
}
}
async fn finish_transfer_channel(state: &Arc<WebState>, transfer_id: &str, channel: &Arc<TransferProgressChannel>) {
transfer::clear_cancelled(transfer_id).await;
let state = state.clone();
let transfer_id = transfer_id.to_string();
let channel = channel.clone();
tokio::spawn(async move {
sleep(COMPLETED_TRANSFER_CHANNEL_TTL).await;
let mut channels = state.transfer_progress_channels.write().await;
if channels.get(&transfer_id).is_some_and(|current| Arc::ptr_eq(current, &channel)) {
channels.remove(&transfer_id);
}
});
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StartTransferRequest {
@@ -45,9 +90,10 @@ pub async fn start_transfer(
let transfer_id = req.transfer_id.clone();
// Create a broadcast channel for progress
let (tx, _) = tokio::sync::broadcast::channel::<String>(256);
state.sse_channels.write().await.insert(transfer_id.clone(), tx.clone());
// Keep bounded replay state so a web EventSource opened after this POST
// still receives early table failures and the terminal result.
let progress_channel = Arc::new(TransferProgressChannel::new());
state.transfer_progress_channels.write().await.insert(transfer_id.clone(), progress_channel.clone());
let app = state.app.clone();
let state_clone = state.clone();
@@ -56,14 +102,16 @@ pub async fn start_transfer(
let source_db_type = match transfer::get_db_type(&app, &req.source_connection_id).await {
Ok(t) => t,
Err(e) => {
let _ = tx.send(serde_json::json!({"error": e}).to_string());
send_transfer_progress(&progress_channel, &terminal_transfer_error(&req, e));
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
};
let target_db_type = match transfer::get_db_type(&app, &req.target_connection_id).await {
Ok(t) => t,
Err(e) => {
let _ = tx.send(serde_json::json!({"error": e}).to_string());
send_transfer_progress(&progress_channel, &terminal_transfer_error(&req, e));
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
};
@@ -72,7 +120,8 @@ pub async fn start_transfer(
{
Ok(k) => k,
Err(e) => {
let _ = tx.send(serde_json::json!({"error": e}).to_string());
send_transfer_progress(&progress_channel, &terminal_transfer_error(&req, e));
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
};
@@ -80,7 +129,8 @@ pub async fn start_transfer(
{
Ok(k) => k,
Err(e) => {
let _ = tx.send(serde_json::json!({"error": e}).to_string());
send_transfer_progress(&progress_channel, &terminal_transfer_error(&req, e));
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
};
@@ -105,16 +155,14 @@ pub async fn start_transfer(
if matches!(source_db_type, dbx_core::models::connection::DatabaseType::Postgres)
&& matches!(target_db_type, dbx_core::models::connection::DatabaseType::Postgres)
{
let tx_clone = tx.clone();
let progress_channel_clone = progress_channel.clone();
match transfer::transfer_postgres_schema_dependencies(
&app,
&req,
&source_pool_key,
&target_pool_key,
|progress| {
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx_clone.send(json);
}
send_transfer_progress(&progress_channel_clone, &progress);
},
)
.await
@@ -132,11 +180,8 @@ pub async fn start_transfer(
error: None,
terminal: true,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
transfer::clear_cancelled(&req.transfer_id).await;
state_clone.remove_sse_channel(&req.transfer_id).await;
send_transfer_progress(&progress_channel, &progress);
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
Err(e) => {
@@ -151,11 +196,8 @@ pub async fn start_transfer(
error: Some(e),
terminal: true,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
transfer::clear_cancelled(&req.transfer_id).await;
state_clone.remove_sse_channel(&req.transfer_id).await;
send_transfer_progress(&progress_channel, &progress);
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
}
@@ -174,15 +216,12 @@ pub async fn start_transfer(
error: None,
terminal: true,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
transfer::clear_cancelled(&req.transfer_id).await;
state_clone.remove_sse_channel(&req.transfer_id).await;
send_transfer_progress(&progress_channel, &progress);
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
let tx_clone = tx.clone();
let progress_channel_clone = progress_channel.clone();
let mut last_rows_transferred = 0_u64;
let mut last_total_rows = None;
let result = transfer::transfer_table(
@@ -197,9 +236,7 @@ pub async fn start_transfer(
|progress| {
last_rows_transferred = progress.rows_transferred;
last_total_rows = progress.total_rows;
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx_clone.send(json);
}
send_transfer_progress(&progress_channel_clone, &progress);
},
)
.await;
@@ -217,9 +254,7 @@ pub async fn start_transfer(
error: None,
terminal: false,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
send_transfer_progress(&progress_channel, &progress);
}
Err(e) => {
if e == "Cancelled" {
@@ -234,11 +269,8 @@ pub async fn start_transfer(
error: None,
terminal: true,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
transfer::clear_cancelled(&req.transfer_id).await;
state_clone.remove_sse_channel(&req.transfer_id).await;
send_transfer_progress(&progress_channel, &progress);
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
failed_tables.push(table.clone());
@@ -253,9 +285,7 @@ pub async fn start_transfer(
error: Some(e),
terminal: false,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
send_transfer_progress(&progress_channel, &progress);
}
}
}
@@ -263,16 +293,14 @@ pub async fn start_transfer(
if matches!(source_db_type, dbx_core::models::connection::DatabaseType::Postgres)
&& matches!(target_db_type, dbx_core::models::connection::DatabaseType::Postgres)
{
let tx_clone = tx.clone();
let progress_channel_clone = progress_channel.clone();
match transfer::transfer_postgres_schema_objects(
&app,
&req,
&source_pool_key,
&target_pool_key,
|progress| {
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx_clone.send(json);
}
send_transfer_progress(&progress_channel_clone, &progress);
},
)
.await
@@ -290,11 +318,8 @@ pub async fn start_transfer(
error: None,
terminal: true,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
transfer::clear_cancelled(&req.transfer_id).await;
state_clone.remove_sse_channel(&req.transfer_id).await;
send_transfer_progress(&progress_channel, &progress);
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
return;
}
Err(e) => {
@@ -310,9 +335,7 @@ pub async fn start_transfer(
error: Some(e),
terminal: false,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
send_transfer_progress(&progress_channel, &progress);
}
}
}
@@ -337,12 +360,8 @@ pub async fn start_transfer(
},
terminal: true,
};
if let Ok(json) = serde_json::to_string(&done) {
let _ = tx.send(json);
}
transfer::clear_cancelled(&req.transfer_id).await;
state_clone.remove_sse_channel(&req.transfer_id).await;
send_transfer_progress(&progress_channel, &done);
finish_transfer_channel(&state_clone, &req.transfer_id, &progress_channel).await;
});
Ok(Json(serde_json::json!({ "transferId": transfer_id })))
@@ -383,11 +402,11 @@ pub async fn transfer_progress(
State(state): State<Arc<WebState>>,
Path(transfer_id): Path<String>,
) -> Result<Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>>, AppError> {
let channels = state.sse_channels.read().await;
let tx = channels.get(&transfer_id).ok_or_else(|| AppError::from("Transfer not found".to_string()))?;
let rx = tx.subscribe();
let channels = state.transfer_progress_channels.read().await;
let channel =
channels.get(&transfer_id).cloned().ok_or_else(|| AppError::from("Transfer not found".to_string()))?;
drop(channels);
Ok(crate::sse::sse_from_channel(rx))
Ok(crate::sse::sse_from_transfer_channel(channel))
}
pub async fn cancel_transfer(
+151
View File
@@ -1,8 +1,100 @@
use axum::response::sse::{Event, KeepAlive, Sse};
use futures::stream::Stream;
use serde_json::Value;
use std::collections::VecDeque;
use std::sync::{Arc, Mutex};
use tokio::sync::broadcast::{self, error::RecvError};
use tokio::sync::watch;
const TRANSFER_REPLAY_MAX_FAILURES: usize = 256;
const TRANSFER_REPLAY_MAX_BYTES: usize = 2 * 1024 * 1024;
#[derive(Clone, Copy)]
pub enum TransferReplayEventKind {
Progress,
Failure,
Terminal,
}
#[derive(Default)]
struct TransferReplayHistory {
failures: VecDeque<String>,
failure_bytes: usize,
latest: Option<String>,
omitted_failures: usize,
}
pub struct TransferProgressChannel {
tx: broadcast::Sender<String>,
history: Mutex<TransferReplayHistory>,
}
impl TransferProgressChannel {
pub fn new() -> Self {
let (tx, _) = broadcast::channel(256);
Self { tx, history: Mutex::new(TransferReplayHistory::default()) }
}
pub fn send(&self, mut data: String, kind: TransferReplayEventKind) {
let mut history = self.history.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
match kind {
TransferReplayEventKind::Progress => {
history.latest = Some(data.clone());
}
TransferReplayEventKind::Terminal => {
data = attach_omitted_failure_count(data, history.omitted_failures);
history.latest = Some(data.clone());
}
TransferReplayEventKind::Failure => {
// The failure itself is the current progress until a newer event arrives.
history.latest = None;
while !history.failures.is_empty()
&& (history.failures.len() >= TRANSFER_REPLAY_MAX_FAILURES
|| history.failure_bytes.saturating_add(data.len()) > TRANSFER_REPLAY_MAX_BYTES)
{
if let Some(removed) = history.failures.pop_front() {
history.failure_bytes = history.failure_bytes.saturating_sub(removed.len());
history.omitted_failures = history.omitted_failures.saturating_add(1);
}
}
if data.len() <= TRANSFER_REPLAY_MAX_BYTES {
history.failure_bytes += data.len();
history.failures.push_back(data.clone());
} else {
history.omitted_failures = history.omitted_failures.saturating_add(1);
}
}
}
let _ = self.tx.send(data);
}
fn subscribe(&self) -> (Vec<String>, broadcast::Receiver<String>) {
// Subscribe while holding the history lock so an event is either in the
// replay snapshot or in the live receiver, with no gap between them.
let history = self.history.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let rx = self.tx.subscribe();
let mut replay = history.failures.iter().cloned().collect::<Vec<_>>();
if let Some(latest) = &history.latest {
replay.push(latest.clone());
}
(replay, rx)
}
}
fn attach_omitted_failure_count(data: String, omitted_failures: usize) -> String {
if omitted_failures == 0 {
return data;
}
let Ok(mut value) = serde_json::from_str::<Value>(&data) else {
return data;
};
let Some(object) = value.as_object_mut() else {
return data;
};
object.insert("transferFailuresOmitted".to_string(), Value::from(omitted_failures as u64));
serde_json::to_string(&value).unwrap_or(data)
}
pub fn sse_from_channel(
rx: broadcast::Receiver<String>,
) -> Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>> {
@@ -51,3 +143,62 @@ pub fn sse_from_watch(
};
Sse::new(stream).keep_alive(KeepAlive::default())
}
pub fn sse_from_transfer_channel(
channel: Arc<TransferProgressChannel>,
) -> Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>> {
let (replay, mut rx) = channel.subscribe();
let stream = async_stream::stream! {
for data in replay {
yield Ok(Event::default().data(data));
}
loop {
match rx.recv().await {
Ok(data) => yield Ok(Event::default().data(data)),
Err(RecvError::Lagged(_)) => break,
Err(RecvError::Closed) => break,
}
}
};
Sse::new(stream).keep_alive(KeepAlive::default())
}
#[cfg(test)]
mod tests {
use super::*;
use axum::body::to_bytes;
use axum::response::IntoResponse;
#[tokio::test]
async fn delayed_transfer_subscription_replays_failure_and_terminal_events() {
let channel = Arc::new(TransferProgressChannel::new());
channel.send("early failure".to_string(), TransferReplayEventKind::Failure);
channel.send("terminal result".to_string(), TransferReplayEventKind::Terminal);
let response = sse_from_transfer_channel(channel).into_response();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let body = String::from_utf8(body.to_vec()).unwrap();
assert!(body.contains("data: early failure"));
assert!(body.contains("data: terminal result"));
assert!(body.find("early failure") < body.find("terminal result"));
}
#[tokio::test]
async fn delayed_transfer_subscription_reports_evicted_failure_count() {
let channel = Arc::new(TransferProgressChannel::new());
for index in 0..=TRANSFER_REPLAY_MAX_FAILURES {
channel.send(format!("failure-{index}"), TransferReplayEventKind::Failure);
}
channel.send(
r#"{"transferId":"transfer-1","status":"done","terminal":true}"#.to_string(),
TransferReplayEventKind::Terminal,
);
let response = sse_from_transfer_channel(channel).into_response();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let body = String::from_utf8(body.to_vec()).unwrap();
assert!(body.contains(r#""transferFailuresOmitted":1"#));
}
}
+3
View File
@@ -5,6 +5,8 @@ use std::sync::Arc;
use tokio::sync::{broadcast, watch, Mutex, RwLock};
use tokio_util::sync::CancellationToken;
use crate::sse::TransferProgressChannel;
pub struct LoginRateLimit {
pub fail_count: u32,
pub locked_until: Option<std::time::Instant>,
@@ -26,6 +28,7 @@ pub struct WebState {
pub password_hash: RwLock<Option<String>>,
pub sessions: RwLock<HashSet<String>>,
pub sse_channels: RwLock<HashMap<String, broadcast::Sender<String>>>,
pub transfer_progress_channels: RwLock<HashMap<String, Arc<TransferProgressChannel>>>,
pub table_import_channels: RwLock<HashMap<String, watch::Sender<String>>>,
pub sql_file_executions: RwLock<HashMap<String, CancellationToken>>,
pub nacos_imports: RwLock<HashMap<String, NacosImportContext>>,