fix(compare): cap per-table compare reads and flag truncation
This commit is contained in:
21 files changed
+471
-81
No files matched your search
@@ -512,6 +512,7 @@ mod tests {
|
||||
target_row_count: 2,
|
||||
source_truncated: false,
|
||||
target_truncated: false,
|
||||
row_budget: 100_000,
|
||||
degradation_level: Some("full".to_string()),
|
||||
sampling_rate: Some(1.0),
|
||||
confidence_score: Some(1.0),
|
||||
|
||||
@@ -88,6 +88,8 @@ pub struct DataCompareMissingTargetOptions {
|
||||
pub key_columns: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub fetch_batch_size: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub degradation_threshold: Option<DegradationThreshold>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -179,6 +181,12 @@ pub struct DataCompareFromTablesPreparation {
|
||||
pub target_row_count: u64,
|
||||
pub source_truncated: bool,
|
||||
pub target_truncated: bool,
|
||||
/// Rows per side this compare was allowed to read. When a side holds more rows than
|
||||
/// this, the diff only covers the first `row_budget` rows and the matching
|
||||
/// `*_truncated` flag is set, so the UI can warn instead of showing a partial plan
|
||||
/// as if it were complete.
|
||||
#[serde(default)]
|
||||
pub row_budget: usize,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub degradation_level: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -363,68 +371,77 @@ pub async fn prepare_data_compare_from_tables(
|
||||
let source_column_names = aligned_source_column_names(&options.columns, options.source_columns.as_ref());
|
||||
let source_key_columns = aligned_source_key_columns(&options.columns, &source_column_names, &options.key_columns);
|
||||
|
||||
let (source_rows, target_rows, sampling_rate, verification_method) = match °radation_level {
|
||||
DegradationLevel::Full => {
|
||||
let (src, tgt) = tokio::try_join!(
|
||||
fetch_compare_rows(
|
||||
state,
|
||||
&options.source_connection_id,
|
||||
&options.source_database,
|
||||
&options.source_schema,
|
||||
&options.source_table,
|
||||
&source_column_names,
|
||||
&source_key_columns,
|
||||
source_database_type,
|
||||
fetch_batch_size,
|
||||
),
|
||||
fetch_compare_rows(
|
||||
state,
|
||||
&options.target_connection_id,
|
||||
&options.target_database,
|
||||
&options.target_schema,
|
||||
&options.target_table,
|
||||
&options.columns,
|
||||
&options.key_columns,
|
||||
target_database_type,
|
||||
fetch_batch_size,
|
||||
)
|
||||
)?;
|
||||
(src, tgt, 1.0, "full_compare".to_string())
|
||||
}
|
||||
DegradationLevel::Sample => {
|
||||
let (src, tgt) = tokio::try_join!(
|
||||
fetch_sampled_compare_rows(
|
||||
state,
|
||||
&options.source_connection_id,
|
||||
&options.source_database,
|
||||
&options.source_schema,
|
||||
&options.source_table,
|
||||
&source_column_names,
|
||||
&source_key_columns,
|
||||
source_database_type,
|
||||
&sampling_strategy,
|
||||
threshold.sample_size,
|
||||
),
|
||||
fetch_sampled_compare_rows(
|
||||
state,
|
||||
&options.target_connection_id,
|
||||
&options.target_database,
|
||||
&options.target_schema,
|
||||
&options.target_table,
|
||||
&options.columns,
|
||||
&options.key_columns,
|
||||
target_database_type,
|
||||
&sampling_strategy,
|
||||
threshold.sample_size,
|
||||
)
|
||||
)?;
|
||||
let max_count = source_row_count.max(target_row_count);
|
||||
let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 };
|
||||
let method = "sampled".to_string();
|
||||
(src, tgt, sample_rate.min(1.0), method)
|
||||
}
|
||||
DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string()),
|
||||
};
|
||||
// Every row is read into memory and then shipped to the UI, so even the "full" level
|
||||
// is capped: the degradation thresholds decide the level, the budget bounds the fetch.
|
||||
let row_budget = rows_read_budget(°radation_level, &threshold);
|
||||
|
||||
let (source_rows, target_rows, sampling_rate, verification_method, source_truncated, target_truncated) =
|
||||
match °radation_level {
|
||||
DegradationLevel::Full => {
|
||||
let (src, tgt) = tokio::try_join!(
|
||||
fetch_compare_rows(
|
||||
state,
|
||||
&options.source_connection_id,
|
||||
&options.source_database,
|
||||
&options.source_schema,
|
||||
&options.source_table,
|
||||
&source_column_names,
|
||||
&source_key_columns,
|
||||
source_database_type,
|
||||
fetch_batch_size,
|
||||
row_budget,
|
||||
),
|
||||
fetch_compare_rows(
|
||||
state,
|
||||
&options.target_connection_id,
|
||||
&options.target_database,
|
||||
&options.target_schema,
|
||||
&options.target_table,
|
||||
&options.columns,
|
||||
&options.key_columns,
|
||||
target_database_type,
|
||||
fetch_batch_size,
|
||||
row_budget,
|
||||
)
|
||||
)?;
|
||||
let source_truncated = source_row_count > row_budget as u64;
|
||||
let target_truncated = target_row_count > row_budget as u64;
|
||||
(src, tgt, 1.0, "full_compare".to_string(), source_truncated, target_truncated)
|
||||
}
|
||||
DegradationLevel::Sample => {
|
||||
let (src, tgt) = tokio::try_join!(
|
||||
fetch_sampled_compare_rows(
|
||||
state,
|
||||
&options.source_connection_id,
|
||||
&options.source_database,
|
||||
&options.source_schema,
|
||||
&options.source_table,
|
||||
&source_column_names,
|
||||
&source_key_columns,
|
||||
source_database_type,
|
||||
&sampling_strategy,
|
||||
threshold.sample_size,
|
||||
),
|
||||
fetch_sampled_compare_rows(
|
||||
state,
|
||||
&options.target_connection_id,
|
||||
&options.target_database,
|
||||
&options.target_schema,
|
||||
&options.target_table,
|
||||
&options.columns,
|
||||
&options.key_columns,
|
||||
target_database_type,
|
||||
&sampling_strategy,
|
||||
threshold.sample_size,
|
||||
)
|
||||
)?;
|
||||
let max_count = source_row_count.max(target_row_count);
|
||||
let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 };
|
||||
let method = "sampled".to_string();
|
||||
(src, tgt, sample_rate.min(1.0), method, false, false)
|
||||
}
|
||||
DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string(), false, false),
|
||||
};
|
||||
|
||||
let target_columns = get_columns_core(
|
||||
state,
|
||||
@@ -468,8 +485,9 @@ pub async fn prepare_data_compare_from_tables(
|
||||
pre_sync_statements: Vec::new(),
|
||||
source_row_count,
|
||||
target_row_count,
|
||||
source_truncated: false,
|
||||
target_truncated: false,
|
||||
source_truncated,
|
||||
target_truncated,
|
||||
row_budget,
|
||||
degradation_level: Some(degradation_level.to_string()),
|
||||
sampling_rate: Some(sampling_rate),
|
||||
confidence_score: Some(confidence_score),
|
||||
@@ -509,6 +527,11 @@ pub async fn prepare_data_compare_missing_target(
|
||||
)
|
||||
.await?;
|
||||
let source_row_count = first_count(&source_count_result.rows)?;
|
||||
// A table that only exists on the source side has no count to compare against, but the
|
||||
// sync plan still embeds one statement per row: reading the whole table freezes the
|
||||
// client on large tables, so the plan covers at most one full-compare budget of rows.
|
||||
let threshold = options.degradation_threshold.clone().unwrap_or_default();
|
||||
let row_budget = threshold.full_compare_max_rows.max(1) as usize;
|
||||
let source_rows = fetch_compare_rows(
|
||||
state,
|
||||
&options.source_connection_id,
|
||||
@@ -519,8 +542,10 @@ pub async fn prepare_data_compare_missing_target(
|
||||
&options.key_columns,
|
||||
source_database_type,
|
||||
fetch_batch_size,
|
||||
row_budget,
|
||||
)
|
||||
.await?;
|
||||
let source_truncated = source_row_count > row_budget as u64;
|
||||
let result = missing_target_diff(&column_names, &options.key_columns, source_rows);
|
||||
let mut pre_sync_statements = Vec::new();
|
||||
pre_sync_statements.push(format!(
|
||||
@@ -567,12 +592,15 @@ pub async fn prepare_data_compare_missing_target(
|
||||
pre_sync_statements,
|
||||
source_row_count,
|
||||
target_row_count: 0,
|
||||
source_truncated: false,
|
||||
source_truncated,
|
||||
target_truncated: false,
|
||||
degradation_level: Some("full".to_string()),
|
||||
sampling_rate: Some(1.0),
|
||||
confidence_score: Some(1.0),
|
||||
verification_method: Some("missing_target_full".to_string()),
|
||||
row_budget,
|
||||
degradation_level: Some(should_degrade(source_row_count, 0, &threshold).to_string()),
|
||||
sampling_rate: Some(if source_truncated { row_budget as f64 / source_row_count.max(1) as f64 } else { 1.0 }),
|
||||
confidence_score: Some(if source_truncated { 0.5 } else { 1.0 }),
|
||||
verification_method: Some(
|
||||
if source_truncated { "missing_target_prefix" } else { "missing_target_full" }.to_string(),
|
||||
),
|
||||
source_checksums: None,
|
||||
target_checksums: None,
|
||||
})
|
||||
@@ -1190,6 +1218,12 @@ fn build_rownum_data_compare_select_sql(
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Fetches the rows a comparison needs, stopping after `row_budget` rows.
|
||||
///
|
||||
/// A compare reads both sides into memory and ships the diff to the UI, so an unbounded
|
||||
/// fetch turns a large table into an out-of-memory freeze on the client. Callers derive
|
||||
/// `source_truncated` / `target_truncated` from the table's row count versus `row_budget`
|
||||
/// so a partial diff is never mistaken for a complete one.
|
||||
async fn fetch_compare_rows(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
@@ -1200,20 +1234,15 @@ async fn fetch_compare_rows(
|
||||
key_columns: &[String],
|
||||
database_type: DatabaseType,
|
||||
fetch_batch_size: usize,
|
||||
row_budget: usize,
|
||||
) -> Result<Vec<Vec<Value>>, String> {
|
||||
let mut rows = Vec::new();
|
||||
let mut offset = 0usize;
|
||||
|
||||
loop {
|
||||
let sql = build_data_compare_select_sql(
|
||||
database_type,
|
||||
schema,
|
||||
table_name,
|
||||
columns,
|
||||
key_columns,
|
||||
fetch_batch_size,
|
||||
offset,
|
||||
);
|
||||
while rows.len() < row_budget {
|
||||
let batch_size = fetch_batch_size.min(row_budget - rows.len());
|
||||
let sql =
|
||||
build_data_compare_select_sql(database_type, schema, table_name, columns, key_columns, batch_size, offset);
|
||||
let result = execute_sql_statement_with_options(
|
||||
state,
|
||||
connection_id,
|
||||
@@ -1221,7 +1250,7 @@ async fn fetch_compare_rows(
|
||||
&sql,
|
||||
Some(schema),
|
||||
None,
|
||||
QueryExecutionOptions { max_rows: Some(fetch_batch_size), ..Default::default() },
|
||||
QueryExecutionOptions { max_rows: Some(batch_size), ..Default::default() },
|
||||
)
|
||||
.await?;
|
||||
let fetched = result.rows.len();
|
||||
@@ -1229,7 +1258,7 @@ async fn fetch_compare_rows(
|
||||
break;
|
||||
}
|
||||
rows.extend(result.rows);
|
||||
if fetched < fetch_batch_size {
|
||||
if fetched < batch_size {
|
||||
break;
|
||||
}
|
||||
offset += fetched;
|
||||
@@ -1295,6 +1324,15 @@ fn data_grid_column_info(column: crate::types::ColumnInfo) -> DataGridColumnInfo
|
||||
}
|
||||
}
|
||||
|
||||
/// Rows per side a compare may read for a given degradation level.
|
||||
fn rows_read_budget(level: &DegradationLevel, threshold: &DegradationThreshold) -> usize {
|
||||
match level {
|
||||
DegradationLevel::Full => threshold.full_compare_max_rows.max(1) as usize,
|
||||
DegradationLevel::Sample => threshold.sample_size,
|
||||
DegradationLevel::SkipWithRisk => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn should_degrade(source_row_count: u64, target_row_count: u64, threshold: &DegradationThreshold) -> DegradationLevel {
|
||||
let max_count = source_row_count.max(target_row_count);
|
||||
if max_count <= threshold.full_compare_max_rows {
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
use dbx_core::connection::AppState;
|
||||
use dbx_core::data_compare::{
|
||||
prepare_data_compare_from_tables, prepare_data_compare_missing_target, DataCompareFromTablesOptions,
|
||||
DataCompareMissingTargetOptions,
|
||||
};
|
||||
use dbx_core::db::mysql;
|
||||
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
fn live_mysql_config(id: &str) -> ConnectionConfig {
|
||||
let host = std::env::var("DBX_LIVE_MYSQL_TRANSFER_HOST").expect("DBX_LIVE_MYSQL_TRANSFER_HOST");
|
||||
let port =
|
||||
std::env::var("DBX_LIVE_MYSQL_TRANSFER_PORT").ok().and_then(|value| value.parse::<u16>().ok()).unwrap_or(3306);
|
||||
let username = std::env::var("DBX_LIVE_MYSQL_TRANSFER_USER").unwrap_or_else(|_| "root".to_string());
|
||||
let password = std::env::var("DBX_LIVE_MYSQL_TRANSFER_PASSWORD").expect("DBX_LIVE_MYSQL_TRANSFER_PASSWORD");
|
||||
|
||||
serde_json::from_value(json!({
|
||||
"id": id,
|
||||
"name": id,
|
||||
"db_type": DatabaseType::Mysql,
|
||||
"host": host,
|
||||
"port": port,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"database": null,
|
||||
"connect_timeout_secs": 10,
|
||||
"query_timeout_secs": 300,
|
||||
"idle_timeout_secs": 60,
|
||||
"keepalive_interval_secs": 0
|
||||
}))
|
||||
.expect("live MySQL compare config should deserialize")
|
||||
}
|
||||
|
||||
fn mysql_url(config: &ConnectionConfig) -> String {
|
||||
format!("mysql://{}:{}@{}:{}", config.username, config.password, config.host, config.port)
|
||||
}
|
||||
|
||||
async fn live_state() -> Arc<AppState> {
|
||||
let task_tmp = std::path::PathBuf::from(
|
||||
std::env::var("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR").expect("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR"),
|
||||
);
|
||||
let dir = task_tmp.join(format!("data-compare-{}", uuid::Uuid::new_v4().simple()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
Arc::new(AppState::new(storage))
|
||||
}
|
||||
|
||||
fn big_source_table_ddl(database: &str, table: &str, rows: u64) -> String {
|
||||
format!(
|
||||
"CREATE DATABASE IF NOT EXISTS {database};\
|
||||
SET SESSION cte_max_recursion_depth={}; \
|
||||
CREATE TABLE {database}.{table} (\
|
||||
id INT NOT NULL PRIMARY KEY, payload VARCHAR(120), amount DECIMAL(12,2)\
|
||||
);\
|
||||
INSERT INTO {database}.{table} (id, payload, amount)\
|
||||
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < {rows})\
|
||||
SELECT n, CONCAT('row-', n), n * 1.5 FROM seq",
|
||||
rows + 1
|
||||
)
|
||||
}
|
||||
|
||||
/// A source-only table must not drag the whole table into memory (and across the IPC
|
||||
/// boundary) when the user compares it against a database that has no such table.
|
||||
#[test]
|
||||
#[ignore = "requires a disposable MySQL endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||
fn live_mysql_data_compare_missing_target_is_bounded() {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(run_live_mysql_data_compare_missing_target_is_bounded());
|
||||
}
|
||||
|
||||
async fn run_live_mysql_data_compare_missing_target_is_bounded() {
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("compare-conn-{}", &suffix[..12]);
|
||||
let source_database = format!("dbx_cmp_src_{}", &suffix[..12]);
|
||||
let target_database = format!("dbx_cmp_dst_{}", &suffix[..12]);
|
||||
let config = live_mysql_config(&connection_id);
|
||||
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
|
||||
|
||||
let rows: u64 = 300_000;
|
||||
mysql::execute_query(&setup_pool, &big_source_table_ddl(&source_database, "big_source_only", rows), false)
|
||||
.await
|
||||
.unwrap();
|
||||
mysql::execute_query(&setup_pool, &format!("CREATE DATABASE {target_database}"), false).await.unwrap();
|
||||
|
||||
let state = live_state().await;
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
|
||||
let started = Instant::now();
|
||||
let preparation = prepare_data_compare_missing_target(
|
||||
&state,
|
||||
DataCompareMissingTargetOptions {
|
||||
source_connection_id: connection_id.clone(),
|
||||
source_database: source_database.clone(),
|
||||
source_schema: source_database.clone(),
|
||||
source_table: "big_source_only".to_string(),
|
||||
target_connection_id: connection_id.clone(),
|
||||
target_database: target_database.clone(),
|
||||
target_schema: target_database.clone(),
|
||||
target_table: "big_source_only".to_string(),
|
||||
key_columns: vec!["id".to_string()],
|
||||
fetch_batch_size: None,
|
||||
degradation_threshold: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
println!(
|
||||
"missing-target compare: rows={} added={} truncated={} level={:?} sql_bytes={} elapsed={:?}",
|
||||
preparation.source_row_count,
|
||||
preparation.result.added.len(),
|
||||
preparation.source_truncated,
|
||||
preparation.degradation_level,
|
||||
preparation.sync_sql.len(),
|
||||
elapsed
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
preparation.result.added.len(),
|
||||
100_000,
|
||||
"compare must stop at the full-compare row budget instead of materialising all {rows} rows"
|
||||
);
|
||||
assert!(preparation.source_truncated, "capped compare must flag source_truncated");
|
||||
assert!(preparation.result.added.len() < rows as usize, "the capped plan must be smaller than the table");
|
||||
assert!(elapsed < Duration::from_secs(30), "capped compare took {elapsed:?}");
|
||||
}
|
||||
|
||||
/// The user-facing scenario: two databases on one connection, small tables, with real
|
||||
/// differences in both directions.
|
||||
#[test]
|
||||
#[ignore = "requires a disposable MySQL endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||
fn live_mysql_data_compare_two_databases_with_differences() {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(run_live_mysql_data_compare_two_databases_with_differences());
|
||||
}
|
||||
|
||||
async fn run_live_mysql_data_compare_two_databases_with_differences() {
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("compare-conn-{}", &suffix[..12]);
|
||||
let source_database = format!("dbx_cmp2_src_{}", &suffix[..12]);
|
||||
let target_database = format!("dbx_cmp2_dst_{}", &suffix[..12]);
|
||||
let config = live_mysql_config(&connection_id);
|
||||
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
|
||||
|
||||
mysql::execute_query(
|
||||
&setup_pool,
|
||||
&format!(
|
||||
"CREATE DATABASE {source_database};\
|
||||
CREATE DATABASE {target_database};\
|
||||
CREATE TABLE {source_database}.items (id INT NOT NULL PRIMARY KEY, name VARCHAR(60), qty INT);\
|
||||
CREATE TABLE {target_database}.items (id INT NOT NULL PRIMARY KEY, name VARCHAR(60), qty INT);\
|
||||
SET SESSION cte_max_recursion_depth=4000;\
|
||||
INSERT INTO {source_database}.items (id, name, qty)\
|
||||
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < 2000)\
|
||||
SELECT n, CONCAT('name-', n), n FROM seq;\
|
||||
INSERT INTO {target_database}.items (id, name, qty)\
|
||||
WITH RECURSIVE seq(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n < 1500)\
|
||||
SELECT n, CONCAT('name-', n * 2), n FROM seq"
|
||||
),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let state = live_state().await;
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
|
||||
let started = Instant::now();
|
||||
let preparation = prepare_data_compare_from_tables(
|
||||
&state,
|
||||
DataCompareFromTablesOptions {
|
||||
source_connection_id: connection_id.clone(),
|
||||
source_database: source_database.clone(),
|
||||
source_schema: source_database.clone(),
|
||||
source_table: "items".to_string(),
|
||||
target_connection_id: connection_id.clone(),
|
||||
target_database: target_database.clone(),
|
||||
target_schema: target_database.clone(),
|
||||
target_table: "items".to_string(),
|
||||
columns: vec!["id".to_string(), "name".to_string(), "qty".to_string()],
|
||||
key_columns: vec!["id".to_string()],
|
||||
source_columns: None,
|
||||
fetch_batch_size: None,
|
||||
degradation_threshold: None,
|
||||
sampling_strategy: None,
|
||||
enable_checksum: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
println!(
|
||||
"two-database compare: source_rows={} target_rows={} added={} removed={} modified={} statements={} sql_bytes={} elapsed={:?}",
|
||||
preparation.source_row_count,
|
||||
preparation.target_row_count,
|
||||
preparation.result.added.len(),
|
||||
preparation.result.removed.len(),
|
||||
preparation.result.modified.len(),
|
||||
preparation.sync_statements.len(),
|
||||
preparation.sync_sql.len(),
|
||||
elapsed
|
||||
);
|
||||
|
||||
assert_eq!(preparation.source_row_count, 2000);
|
||||
assert_eq!(preparation.target_row_count, 1500);
|
||||
assert_eq!(preparation.result.added.len(), 500, "rows 1501..2000 exist only on the source");
|
||||
assert_eq!(preparation.result.removed.len(), 0, "the target has no extra rows");
|
||||
assert!(!preparation.source_truncated && !preparation.target_truncated, "small compares are not truncated");
|
||||
assert_eq!(preparation.result.modified.len(), 1500, "every shared row changed its name");
|
||||
assert!(elapsed < Duration::from_secs(30), "compare took {elapsed:?}");
|
||||
}
|
||||
@@ -239,6 +239,7 @@ fn data_compare_from_tables_preparation_optional_fields_default() {
|
||||
target_row_count: 0,
|
||||
source_truncated: false,
|
||||
target_truncated: false,
|
||||
row_budget: 100_000,
|
||||
degradation_level: None,
|
||||
sampling_rate: None,
|
||||
confidence_score: None,
|
||||
@@ -252,6 +253,8 @@ fn data_compare_from_tables_preparation_optional_fields_default() {
|
||||
// Core fields must be present
|
||||
assert!(json.get("result").is_some(), "result must be present");
|
||||
assert!(json.get("syncSql").is_some(), "syncSql must be present");
|
||||
// The row budget travels with the preparation so the UI can flag a partial compare.
|
||||
assert_eq!(json.get("rowBudget").and_then(|value| value.as_u64()), Some(100_000));
|
||||
|
||||
// New optional fields should be absent when None
|
||||
assert!(json.get("degradationLevel").is_none(), "degradationLevel should be absent");
|
||||
|
||||
Reference in new issue
Block a user