From 9572a9e52d7f8281d8f96f63ca3b73deeb3c9eb6 Mon Sep 17 00:00:00 2001 From: miracle Date: Thu, 25 Jun 2026 22:14:08 +0800 Subject: [PATCH] fix: inject deterministic ORDER BY for dedup query pagination (Doris/MySQL) Problem: When exporting DISTINCT/GROUP BY query results from distributed databases like Doris 2.1, LIMIT/OFFSET pagination without ORDER BY causes duplicate rows across pages because tablet scan order varies between independent query executions. Fix: Detect dedup queries (DISTINCT, GROUP BY, HAVING) that lack an ORDER BY clause and automatically inject a positional ORDER BY (1, 2, ..., N) before applying LIMIT/OFFSET. Positional references work correctly with aliases, expressions, CASE WHEN, subqueries, and aggregate functions. Changes: - Add dedup_projection_count_without_order_by() using sqlparser to detect dedup queries and count projection columns - Add ormat_positional_order_by() to generate ORDER BY 1, 2, ..., N - Modify dd_standard_limit() to accept dedup_projection_count and inject ORDER BY; wrap in subquery when original query already has LIMIT - Update dd_outer_standard_limit() to accept an order_sql parameter - Add 20 unit tests covering Doris, MySQL, PostgreSQL, aliases, expressions, subqueries, GROUP BY, UNION, and existing ORDER BY scenarios All 1203 tests pass (61 in query_result_sql, 11 in query_result_export). --- crates/dbx-core/src/query_result_sql.rs | 314 +++++++++++++++++++++++- 1 file changed, 308 insertions(+), 6 deletions(-) diff --git a/crates/dbx-core/src/query_result_sql.rs b/crates/dbx-core/src/query_result_sql.rs index 477bcf8f7a..c7196a01ca 100644 --- a/crates/dbx-core/src/query_result_sql.rs +++ b/crates/dbx-core/src/query_result_sql.rs @@ -3,6 +3,9 @@ use serde::{Deserialize, Serialize}; use crate::models::connection::DatabaseType; use crate::sql::find_statement_at_cursor; use crate::sql_dialect::{pagination_strategy, quote_table_identifier, PaginationContext, TablePaginationStrategy}; +use sqlparser::ast::{GroupByExpr, SelectItem, SetExpr, Statement}; +use sqlparser::dialect::GenericDialect; +use sqlparser::parser::Parser; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -191,7 +194,8 @@ pub fn build_paginated_query_sql(options: PaginatedQuerySqlOptions) -> QuerySqlB TablePaginationStrategy::AgentMaxRows | TablePaginationStrategy::Unbounded => ok(format!("{statement};")), TablePaginationStrategy::IrisTop => ok(add_iris_top_limit(&statement, safe_limit)), TablePaginationStrategy::LimitOffset => { - ok(add_standard_limit(&statement, options.database_type, safe_limit, safe_offset)) + let dedup_count = dedup_projection_count_without_order_by(&options.original_sql); + ok(add_standard_limit(&statement, options.database_type, safe_limit, safe_offset, dedup_count)) } } } @@ -488,7 +492,7 @@ fn add_iris_top_limit(statement: &str, limit: usize) -> String { fn add_questdb_limit(statement: &str, limit: usize, offset: usize) -> String { if has_top_level_limit(statement) { if offset > 0 { - return add_outer_standard_limit(statement, Some(DatabaseType::Questdb), limit, offset); + return add_outer_standard_limit(statement, Some(DatabaseType::Questdb), limit, offset, ""); } return format!("{statement};"); } @@ -562,15 +566,31 @@ fn add_rownum_limit(statement: &str, limit: usize, offset: usize) -> String { ) } -fn add_standard_limit(statement: &str, database_type: Option, limit: usize, offset: usize) -> String { +fn add_standard_limit( + statement: &str, + database_type: Option, + limit: usize, + offset: usize, + dedup_projection_count: Option, +) -> String { + let order_sql = dedup_projection_count.map_or(String::new(), |count| format_positional_order_by(count)); + if has_top_level_limit(statement) { + if !order_sql.is_empty() { + // For dedup queries (DISTINCT / GROUP BY) without user ORDER BY, + // wrap the query to guarantee deterministic LIMIT/OFFSET pagination. + // The inner query preserves DISTINCT semantics; the outer query + // adds ORDER BY on positional columns to ensure consistent row + // ordering across pages in distributed databases like Doris. + return add_outer_standard_limit(statement, database_type, limit, offset, &order_sql); + } if offset > 0 { - return add_outer_standard_limit(statement, database_type, limit, offset); + return add_outer_standard_limit(statement, database_type, limit, offset, ""); } return format!("{statement};"); } let offset_sql = if offset > 0 { format!(" OFFSET {offset}") } else { String::new() }; - format!("{statement} LIMIT {limit}{offset_sql};") + format!("{statement}{order_sql} LIMIT {limit}{offset_sql};") } fn add_outer_standard_limit( @@ -578,9 +598,59 @@ fn add_outer_standard_limit( database_type: Option, limit: usize, offset: usize, + order_sql: &str, ) -> String { let alias = quote_table_identifier(database_type, "dbx_page"); - format!("SELECT * FROM ({statement}) {alias} LIMIT {limit} OFFSET {offset};") + format!("SELECT * FROM ({statement}) {alias}{order_sql} LIMIT {limit} OFFSET {offset};") +} + +/// For dedup queries (DISTINCT / GROUP BY) without an ORDER BY clause, generate +/// a positional `ORDER BY 1, 2, ..., N` clause so that LIMIT/OFFSET pagination +/// returns deterministic results across pages. This is especially important for +/// distributed databases (e.g. Doris, StarRocks) where tablet scan order varies +/// between independent query executions. +fn format_positional_order_by(column_count: usize) -> String { + if column_count == 0 { + return String::new(); + } + let cols: Vec = (1..=column_count).map(|i| i.to_string()).collect(); + format!(" ORDER BY {}", cols.join(", ")) +} + +/// Detect dedup queries (SELECT DISTINCT, GROUP BY, HAVING) that lack a +/// top-level ORDER BY clause. Returns the number of projection items so that +/// a positional ORDER BY can be injected for deterministic pagination. +/// +/// Returns `None` for: +/// - Non-SELECT queries +/// - Queries without dedup semantics +/// - Queries that already specify ORDER BY +/// - Wildcard projections (`SELECT *`) +/// - Parse failures +fn dedup_projection_count_without_order_by(sql: &str) -> Option { + let dialect = GenericDialect {}; + let statements = Parser::parse_sql(&dialect, sql).ok()?; + let [Statement::Query(query)] = statements.as_slice() else { + return None; + }; + // Reject if the query already has an ORDER BY clause. + if query.order_by.is_some() { + return None; + } + let SetExpr::Select(select) = query.body.as_ref() else { + return None; + }; + let has_distinct = select.distinct.is_some(); + let has_group_by = !matches!(&select.group_by, GroupByExpr::Expressions(exprs, _) if exprs.is_empty()); + let has_having = select.having.is_some(); + if !has_distinct && !has_group_by && !has_having { + return None; + } + // Wildcard projections cannot be used with positional ORDER BY. + if select.projection.len() == 1 && matches!(select.projection.first(), Some(SelectItem::Wildcard(_))) { + return None; + } + Some(select.projection.len()) } fn find_top_level_trailing_order_by(sql: &str) -> Option { @@ -1418,4 +1488,236 @@ WHERE u.id = picked.id; assert_eq!(result, err("with")); } + + // ----------------------------------------------------------------------- + // Dedup query ORDER BY injection (DISTINCT / GROUP BY) + // ----------------------------------------------------------------------- + + #[test] + fn dedup_count_detects_distinct_query_without_order_by() { + assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT a, b, c FROM t"), Some(3)); + } + + #[test] + fn dedup_count_detects_group_by_query() { + assert_eq!(dedup_projection_count_without_order_by("SELECT city, COUNT(*) FROM users GROUP BY city"), Some(2)); + } + + #[test] + fn dedup_count_returns_none_for_plain_select() { + assert_eq!(dedup_projection_count_without_order_by("SELECT a, b FROM t"), None); + } + + #[test] + fn dedup_count_returns_none_when_order_by_exists() { + assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT a, b FROM t ORDER BY a"), None); + } + + #[test] + fn dedup_count_returns_none_for_wildcard() { + assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT * FROM t"), None); + } + + #[test] + fn doris_distinct_query_first_page_gets_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT DISTINCT city, name FROM users".to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city, name FROM users ORDER BY 1, 2 LIMIT 100;"); + } + + #[test] + fn doris_distinct_query_second_page_wraps_with_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT DISTINCT city, name FROM users".to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 100, + }); + + assert!(result.ok); + // No top-level LIMIT in the original query, so ORDER BY + LIMIT + OFFSET + // are appended directly to the statement (no wrapping needed). + assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city, name FROM users ORDER BY 1, 2 LIMIT 100 OFFSET 100;"); + } + + #[test] + fn doris_group_by_query_first_page_gets_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT dept, SUM(salary) FROM employees GROUP BY dept".to_string(), + database_type: Some(DatabaseType::Doris), + limit: 50, + offset: 0, + }); + + assert!(result.ok); + assert_eq!( + result.sql.unwrap(), + "SELECT dept, SUM(salary) FROM employees GROUP BY dept ORDER BY 1, 2 LIMIT 50;" + ); + } + + #[test] + fn doris_plain_query_without_distinct_is_unaffected() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT id, name FROM users".to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + assert_eq!(result.sql.unwrap(), "SELECT id, name FROM users LIMIT 100;"); + } + + #[test] + fn doris_distinct_with_existing_limit_first_page_keeps_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT DISTINCT a, b, c FROM t LIMIT 500".to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + // The dedup query has a LIMIT, so it must be wrapped. + // `add_outer_standard_limit` always includes OFFSET clause. + assert_eq!( + result.sql.unwrap(), + "SELECT * FROM (SELECT DISTINCT a, b, c FROM t LIMIT 500) \"dbx_page\" ORDER BY 1, 2, 3 LIMIT 100 OFFSET 0;" + ); + } + + #[test] + fn mysql_distinct_query_first_page_gets_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT DISTINCT city FROM users".to_string(), + database_type: Some(DatabaseType::Mysql), + limit: 200, + offset: 0, + }); + + assert!(result.ok); + assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city FROM users ORDER BY 1 LIMIT 200;"); + } + + #[test] + fn postgres_distinct_query_first_page_gets_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT DISTINCT city, name FROM users".to_string(), + database_type: Some(DatabaseType::Postgres), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city, name FROM users ORDER BY 1, 2 LIMIT 100;"); + } + + #[test] + fn distinct_query_with_existing_order_by_is_not_modified() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT DISTINCT a, b FROM t ORDER BY a".to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + // Already has ORDER BY, so only LIMIT is appended. + assert_eq!(result.sql.unwrap(), "SELECT DISTINCT a, b FROM t ORDER BY a LIMIT 100;"); + } + + // ----------------------------------------------------------------------- + // Complex queries with aliases, expressions, subqueries + // ----------------------------------------------------------------------- + + #[test] + fn dedup_count_handles_aliases() { + assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT a AS x, b AS y, c AS z FROM t"), Some(3)); + } + + #[test] + fn dedup_count_handles_expressions() { + assert_eq!( + dedup_projection_count_without_order_by( + "SELECT DISTINCT a + b AS sum_col, CASE WHEN c > 0 THEN 'Y' ELSE 'N' END AS flag FROM t" + ), + Some(2) + ); + } + + #[test] + fn dedup_count_handles_aggregate_with_alias() { + assert_eq!( + dedup_projection_count_without_order_by( + "SELECT city, COUNT(*) AS cnt, AVG(salary) AS avg_sal FROM users GROUP BY city" + ), + Some(3) + ); + } + + #[test] + fn doris_distinct_with_alias_and_expression_gets_positional_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: + "SELECT DISTINCT a AS col1, b + c AS col2, CASE WHEN d > 0 THEN 'Y' ELSE 'N' END AS col3 FROM t" + .to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + // Positional ORDER BY 1, 2, 3 works regardless of aliases or expressions. + assert_eq!( + result.sql.unwrap(), + "SELECT DISTINCT a AS col1, b + c AS col2, CASE WHEN d > 0 THEN 'Y' ELSE 'N' END AS col3 FROM t ORDER BY 1, 2, 3 LIMIT 100;" + ); + } + + #[test] + fn doris_group_by_with_aggregate_alias_gets_positional_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: "SELECT dept, SUM(salary) AS total, COUNT(*) AS head_count FROM emp GROUP BY dept" + .to_string(), + database_type: Some(DatabaseType::Doris), + limit: 50, + offset: 100, + }); + + assert!(result.ok); + assert_eq!( + result.sql.unwrap(), + "SELECT dept, SUM(salary) AS total, COUNT(*) AS head_count FROM emp GROUP BY dept ORDER BY 1, 2, 3 LIMIT 50 OFFSET 100;" + ); + } + + #[test] + fn doris_distinct_with_subquery_column_gets_positional_order_by() { + let result = build_paginated_query_sql(PaginatedQuerySqlOptions { + original_sql: + "SELECT DISTINCT name, (SELECT MAX(score) FROM scores s WHERE s.uid = u.id) AS max_score FROM users u" + .to_string(), + database_type: Some(DatabaseType::Doris), + limit: 100, + offset: 0, + }); + + assert!(result.ok); + assert_eq!( + result.sql.unwrap(), + "SELECT DISTINCT name, (SELECT MAX(score) FROM scores s WHERE s.uid = u.id) AS max_score FROM users u ORDER BY 1, 2 LIMIT 100;" + ); + } + + #[test] + fn union_query_is_not_treated_as_dedup() { + assert_eq!(dedup_projection_count_without_order_by("SELECT a FROM t1 UNION SELECT b FROM t2"), None); + } }