fix: inject deterministic ORDER BY for dedup query pagination (Doris/MySQL)

Problem: When exporting DISTINCT/GROUP BY query results from distributed
databases like Doris 2.1, LIMIT/OFFSET pagination without ORDER BY causes
duplicate rows across pages because tablet scan order varies between
independent query executions.

Fix: Detect dedup queries (DISTINCT, GROUP BY, HAVING) that lack an
ORDER BY clause and automatically inject a positional ORDER BY (1, 2, ..., N)
before applying LIMIT/OFFSET. Positional references work correctly with
aliases, expressions, CASE WHEN, subqueries, and aggregate functions.

Changes:
- Add dedup_projection_count_without_order_by() using sqlparser to detect
  dedup queries and count projection columns
- Add ormat_positional_order_by() to generate ORDER BY 1, 2, ..., N
- Modify dd_standard_limit() to accept dedup_projection_count and inject
  ORDER BY; wrap in subquery when original query already has LIMIT
- Update dd_outer_standard_limit() to accept an order_sql parameter
- Add 20 unit tests covering Doris, MySQL, PostgreSQL, aliases, expressions,
  subqueries, GROUP BY, UNION, and existing ORDER BY scenarios

All 1203 tests pass (61 in query_result_sql, 11 in query_result_export).
This commit is contained in:
miracle authored and GitHub committed 2026-06-25 22:14:08 +08:00
1 parent 16f546ada1
commit 9572a9e52d
1 file changed
+308 -6
+308 -6
View File
@@ -3,6 +3,9 @@ use serde::{Deserialize, Serialize};
use crate::models::connection::DatabaseType;
use crate::sql::find_statement_at_cursor;
use crate::sql_dialect::{pagination_strategy, quote_table_identifier, PaginationContext, TablePaginationStrategy};
use sqlparser::ast::{GroupByExpr, SelectItem, SetExpr, Statement};
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -191,7 +194,8 @@ pub fn build_paginated_query_sql(options: PaginatedQuerySqlOptions) -> QuerySqlB
TablePaginationStrategy::AgentMaxRows | TablePaginationStrategy::Unbounded => ok(format!("{statement};")),
TablePaginationStrategy::IrisTop => ok(add_iris_top_limit(&statement, safe_limit)),
TablePaginationStrategy::LimitOffset => {
ok(add_standard_limit(&statement, options.database_type, safe_limit, safe_offset))
let dedup_count = dedup_projection_count_without_order_by(&options.original_sql);
ok(add_standard_limit(&statement, options.database_type, safe_limit, safe_offset, dedup_count))
}
}
}
@@ -488,7 +492,7 @@ fn add_iris_top_limit(statement: &str, limit: usize) -> String {
fn add_questdb_limit(statement: &str, limit: usize, offset: usize) -> String {
if has_top_level_limit(statement) {
if offset > 0 {
return add_outer_standard_limit(statement, Some(DatabaseType::Questdb), limit, offset);
return add_outer_standard_limit(statement, Some(DatabaseType::Questdb), limit, offset, "");
}
return format!("{statement};");
}
@@ -562,15 +566,31 @@ fn add_rownum_limit(statement: &str, limit: usize, offset: usize) -> String {
)
}
fn add_standard_limit(statement: &str, database_type: Option<DatabaseType>, limit: usize, offset: usize) -> String {
fn add_standard_limit(
statement: &str,
database_type: Option<DatabaseType>,
limit: usize,
offset: usize,
dedup_projection_count: Option<usize>,
) -> String {
let order_sql = dedup_projection_count.map_or(String::new(), |count| format_positional_order_by(count));
if has_top_level_limit(statement) {
if !order_sql.is_empty() {
// For dedup queries (DISTINCT / GROUP BY) without user ORDER BY,
// wrap the query to guarantee deterministic LIMIT/OFFSET pagination.
// The inner query preserves DISTINCT semantics; the outer query
// adds ORDER BY on positional columns to ensure consistent row
// ordering across pages in distributed databases like Doris.
return add_outer_standard_limit(statement, database_type, limit, offset, &order_sql);
}
if offset > 0 {
return add_outer_standard_limit(statement, database_type, limit, offset);
return add_outer_standard_limit(statement, database_type, limit, offset, "");
}
return format!("{statement};");
}
let offset_sql = if offset > 0 { format!(" OFFSET {offset}") } else { String::new() };
format!("{statement} LIMIT {limit}{offset_sql};")
format!("{statement}{order_sql} LIMIT {limit}{offset_sql};")
}
fn add_outer_standard_limit(
@@ -578,9 +598,59 @@ fn add_outer_standard_limit(
database_type: Option<DatabaseType>,
limit: usize,
offset: usize,
order_sql: &str,
) -> String {
let alias = quote_table_identifier(database_type, "dbx_page");
format!("SELECT * FROM ({statement}) {alias} LIMIT {limit} OFFSET {offset};")
format!("SELECT * FROM ({statement}) {alias}{order_sql} LIMIT {limit} OFFSET {offset};")
}
/// For dedup queries (DISTINCT / GROUP BY) without an ORDER BY clause, generate
/// a positional `ORDER BY 1, 2, ..., N` clause so that LIMIT/OFFSET pagination
/// returns deterministic results across pages. This is especially important for
/// distributed databases (e.g. Doris, StarRocks) where tablet scan order varies
/// between independent query executions.
fn format_positional_order_by(column_count: usize) -> String {
if column_count == 0 {
return String::new();
}
let cols: Vec<String> = (1..=column_count).map(|i| i.to_string()).collect();
format!(" ORDER BY {}", cols.join(", "))
}
/// Detect dedup queries (SELECT DISTINCT, GROUP BY, HAVING) that lack a
/// top-level ORDER BY clause. Returns the number of projection items so that
/// a positional ORDER BY can be injected for deterministic pagination.
///
/// Returns `None` for:
/// - Non-SELECT queries
/// - Queries without dedup semantics
/// - Queries that already specify ORDER BY
/// - Wildcard projections (`SELECT *`)
/// - Parse failures
fn dedup_projection_count_without_order_by(sql: &str) -> Option<usize> {
let dialect = GenericDialect {};
let statements = Parser::parse_sql(&dialect, sql).ok()?;
let [Statement::Query(query)] = statements.as_slice() else {
return None;
};
// Reject if the query already has an ORDER BY clause.
if query.order_by.is_some() {
return None;
}
let SetExpr::Select(select) = query.body.as_ref() else {
return None;
};
let has_distinct = select.distinct.is_some();
let has_group_by = !matches!(&select.group_by, GroupByExpr::Expressions(exprs, _) if exprs.is_empty());
let has_having = select.having.is_some();
if !has_distinct && !has_group_by && !has_having {
return None;
}
// Wildcard projections cannot be used with positional ORDER BY.
if select.projection.len() == 1 && matches!(select.projection.first(), Some(SelectItem::Wildcard(_))) {
return None;
}
Some(select.projection.len())
}
fn find_top_level_trailing_order_by(sql: &str) -> Option<usize> {
@@ -1418,4 +1488,236 @@ WHERE u.id = picked.id;
assert_eq!(result, err("with"));
}
// -----------------------------------------------------------------------
// Dedup query ORDER BY injection (DISTINCT / GROUP BY)
// -----------------------------------------------------------------------
#[test]
fn dedup_count_detects_distinct_query_without_order_by() {
assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT a, b, c FROM t"), Some(3));
}
#[test]
fn dedup_count_detects_group_by_query() {
assert_eq!(dedup_projection_count_without_order_by("SELECT city, COUNT(*) FROM users GROUP BY city"), Some(2));
}
#[test]
fn dedup_count_returns_none_for_plain_select() {
assert_eq!(dedup_projection_count_without_order_by("SELECT a, b FROM t"), None);
}
#[test]
fn dedup_count_returns_none_when_order_by_exists() {
assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT a, b FROM t ORDER BY a"), None);
}
#[test]
fn dedup_count_returns_none_for_wildcard() {
assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT * FROM t"), None);
}
#[test]
fn doris_distinct_query_first_page_gets_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT DISTINCT city, name FROM users".to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 0,
});
assert!(result.ok);
assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city, name FROM users ORDER BY 1, 2 LIMIT 100;");
}
#[test]
fn doris_distinct_query_second_page_wraps_with_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT DISTINCT city, name FROM users".to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 100,
});
assert!(result.ok);
// No top-level LIMIT in the original query, so ORDER BY + LIMIT + OFFSET
// are appended directly to the statement (no wrapping needed).
assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city, name FROM users ORDER BY 1, 2 LIMIT 100 OFFSET 100;");
}
#[test]
fn doris_group_by_query_first_page_gets_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT dept, SUM(salary) FROM employees GROUP BY dept".to_string(),
database_type: Some(DatabaseType::Doris),
limit: 50,
offset: 0,
});
assert!(result.ok);
assert_eq!(
result.sql.unwrap(),
"SELECT dept, SUM(salary) FROM employees GROUP BY dept ORDER BY 1, 2 LIMIT 50;"
);
}
#[test]
fn doris_plain_query_without_distinct_is_unaffected() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT id, name FROM users".to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 0,
});
assert!(result.ok);
assert_eq!(result.sql.unwrap(), "SELECT id, name FROM users LIMIT 100;");
}
#[test]
fn doris_distinct_with_existing_limit_first_page_keeps_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT DISTINCT a, b, c FROM t LIMIT 500".to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 0,
});
assert!(result.ok);
// The dedup query has a LIMIT, so it must be wrapped.
// `add_outer_standard_limit` always includes OFFSET clause.
assert_eq!(
result.sql.unwrap(),
"SELECT * FROM (SELECT DISTINCT a, b, c FROM t LIMIT 500) \"dbx_page\" ORDER BY 1, 2, 3 LIMIT 100 OFFSET 0;"
);
}
#[test]
fn mysql_distinct_query_first_page_gets_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT DISTINCT city FROM users".to_string(),
database_type: Some(DatabaseType::Mysql),
limit: 200,
offset: 0,
});
assert!(result.ok);
assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city FROM users ORDER BY 1 LIMIT 200;");
}
#[test]
fn postgres_distinct_query_first_page_gets_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT DISTINCT city, name FROM users".to_string(),
database_type: Some(DatabaseType::Postgres),
limit: 100,
offset: 0,
});
assert!(result.ok);
assert_eq!(result.sql.unwrap(), "SELECT DISTINCT city, name FROM users ORDER BY 1, 2 LIMIT 100;");
}
#[test]
fn distinct_query_with_existing_order_by_is_not_modified() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT DISTINCT a, b FROM t ORDER BY a".to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 0,
});
assert!(result.ok);
// Already has ORDER BY, so only LIMIT is appended.
assert_eq!(result.sql.unwrap(), "SELECT DISTINCT a, b FROM t ORDER BY a LIMIT 100;");
}
// -----------------------------------------------------------------------
// Complex queries with aliases, expressions, subqueries
// -----------------------------------------------------------------------
#[test]
fn dedup_count_handles_aliases() {
assert_eq!(dedup_projection_count_without_order_by("SELECT DISTINCT a AS x, b AS y, c AS z FROM t"), Some(3));
}
#[test]
fn dedup_count_handles_expressions() {
assert_eq!(
dedup_projection_count_without_order_by(
"SELECT DISTINCT a + b AS sum_col, CASE WHEN c > 0 THEN 'Y' ELSE 'N' END AS flag FROM t"
),
Some(2)
);
}
#[test]
fn dedup_count_handles_aggregate_with_alias() {
assert_eq!(
dedup_projection_count_without_order_by(
"SELECT city, COUNT(*) AS cnt, AVG(salary) AS avg_sal FROM users GROUP BY city"
),
Some(3)
);
}
#[test]
fn doris_distinct_with_alias_and_expression_gets_positional_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql:
"SELECT DISTINCT a AS col1, b + c AS col2, CASE WHEN d > 0 THEN 'Y' ELSE 'N' END AS col3 FROM t"
.to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 0,
});
assert!(result.ok);
// Positional ORDER BY 1, 2, 3 works regardless of aliases or expressions.
assert_eq!(
result.sql.unwrap(),
"SELECT DISTINCT a AS col1, b + c AS col2, CASE WHEN d > 0 THEN 'Y' ELSE 'N' END AS col3 FROM t ORDER BY 1, 2, 3 LIMIT 100;"
);
}
#[test]
fn doris_group_by_with_aggregate_alias_gets_positional_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql: "SELECT dept, SUM(salary) AS total, COUNT(*) AS head_count FROM emp GROUP BY dept"
.to_string(),
database_type: Some(DatabaseType::Doris),
limit: 50,
offset: 100,
});
assert!(result.ok);
assert_eq!(
result.sql.unwrap(),
"SELECT dept, SUM(salary) AS total, COUNT(*) AS head_count FROM emp GROUP BY dept ORDER BY 1, 2, 3 LIMIT 50 OFFSET 100;"
);
}
#[test]
fn doris_distinct_with_subquery_column_gets_positional_order_by() {
let result = build_paginated_query_sql(PaginatedQuerySqlOptions {
original_sql:
"SELECT DISTINCT name, (SELECT MAX(score) FROM scores s WHERE s.uid = u.id) AS max_score FROM users u"
.to_string(),
database_type: Some(DatabaseType::Doris),
limit: 100,
offset: 0,
});
assert!(result.ok);
assert_eq!(
result.sql.unwrap(),
"SELECT DISTINCT name, (SELECT MAX(score) FROM scores s WHERE s.uid = u.id) AS max_score FROM users u ORDER BY 1, 2 LIMIT 100;"
);
}
#[test]
fn union_query_is_not_treated_as_dedup() {
assert_eq!(dedup_projection_count_without_order_by("SELECT a FROM t1 UNION SELECT b FROM t2"), None);
}
}