1 parent
4eabeb6a15
commit
eccbc5a315
4 files changed
+375
-5
No files matched your search
@@ -696,3 +696,106 @@ fn clickhouse_strictness_first_left_joins_do_not_raise_syntax_errors() {
|
||||
assert_eq!(tables, vec![("events", Some("a")), ("wallets", Some("b"))]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_datasource_create_table_supports_iceberg_clauses() {
|
||||
let sql = r#"CREATE TABLE account_flow (
|
||||
id STRING,
|
||||
databasename STRING,
|
||||
created TIMESTAMP
|
||||
)
|
||||
USING iceberg
|
||||
PARTITIONED BY (databasename, truncate(created, 7))
|
||||
COMMENT '账户流水表'
|
||||
TBLPROPERTIES (
|
||||
'format-version' = '2',
|
||||
'snapshot.base.keep.minutes' = '1440',
|
||||
'self-optimizing.group' = 'supbig',
|
||||
'write.metadata.delete-after-commit.enabled' = 'true',
|
||||
'write.metadata.previous-versions-max' = '3',
|
||||
'clean-orphan-file.enabled' = 'true',
|
||||
'clean-orphan-file.min-existing-time-minutes' = '1440',
|
||||
'primary-key' = 'id,databasename',
|
||||
'table.drop-base-path.enabled' = 'true'
|
||||
);"#;
|
||||
|
||||
let analysis = analyze_sql_references(sql, Some("spark"))
|
||||
.unwrap_or_else(|error| panic!("Spark datasource CREATE TABLE should analyze: {error}"));
|
||||
|
||||
assert!(analysis.tables.is_empty());
|
||||
assert!(analysis.columns.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_datasource_ctas_preserves_query_references() {
|
||||
let sql = "CREATE TABLE account_flow USING iceberg PARTITIONED BY (id) TBLPROPERTIES ('format-version' = '2') AS SELECT s.id FROM source_flow s";
|
||||
let analysis = analyze_sql_references(sql, Some("spark")).expect("Spark datasource CTAS should analyze");
|
||||
|
||||
let tables: Vec<_> = analysis.tables.iter().map(|table| (table.name.as_str(), table.alias.as_deref())).collect();
|
||||
assert_eq!(tables, vec![("source_flow", Some("s"))]);
|
||||
assert_eq!(analysis.tables[0].span.start_column, sql.find("source_flow").expect("source table") + 1);
|
||||
assert_eq!(analysis.columns.iter().map(|column| column.name.as_str()).collect::<Vec<_>>(), vec!["id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_datasource_create_table_validates_options_and_properties() {
|
||||
let sql = "CREATE TABLE account_flow (id STRING) USING iceberg TBLPROPERTIES ('format-version' = '2') COMMENT 'account flow' PARTITIONED BY (id) OPTIONS ('merge-schema' = 'true')";
|
||||
let analysis =
|
||||
analyze_sql_references(sql, Some("spark")).expect("reordered Spark datasource clauses should analyze");
|
||||
|
||||
assert!(analysis.tables.is_empty());
|
||||
assert!(analysis.columns.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_datasource_create_table_rejects_duplicate_clauses() {
|
||||
let duplicate_sql = [
|
||||
"CREATE TABLE broken (id STRING) USING iceberg OPTIONS ('a' = '1') OPTIONS ('b' = '2')",
|
||||
"CREATE TABLE broken (id STRING) USING iceberg PARTITIONED BY (id) PARTITIONED BY (id)",
|
||||
"CREATE TABLE broken (id STRING) USING iceberg COMMENT 'first' COMMENT 'second'",
|
||||
"CREATE TABLE broken (id STRING) USING iceberg TBLPROPERTIES ('a' = '1') TBLPROPERTIES ('b' = '2')",
|
||||
];
|
||||
|
||||
let accepted: Vec<_> =
|
||||
duplicate_sql.iter().copied().filter(|sql| analyze_sql_references(sql, Some("spark")).is_ok()).collect();
|
||||
assert!(accepted.is_empty(), "duplicate Spark datasource clauses were accepted: {accepted:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_datasource_create_table_keeps_syntax_errors() {
|
||||
for sql in [
|
||||
"CREATE TABLE broken (id STRING) USING",
|
||||
"CREATE TABLE broken (id STRING) USING 'iceberg'",
|
||||
"CREATE TABLE broken (id STRING USING iceberg",
|
||||
"CREATE TABLE broken (id STRING, created TIMESTAMP) USING iceberg PARTITIONED BY (truncate(created 7))",
|
||||
"CREATE TABLE broken (id STRING, created TIMESTAMP) USING iceberg PARTITIONED BY (created) UNKNOWN CLAUSE",
|
||||
"CREATE TABLE broken (id STRING) USING iceberg OPTIONS ('merge-schema')",
|
||||
"CREATE TABLE broken (id STRING) USING iceberg TBLPROPERTIES ('format-version')",
|
||||
] {
|
||||
let error = analyze_sql_references(sql, Some("spark"))
|
||||
.expect_err(&format!("malformed Spark datasource CREATE TABLE must keep its parser error: {sql}"));
|
||||
assert!(!error.is_empty());
|
||||
}
|
||||
|
||||
let provider_error = analyze_sql_references("CREATE TABLE broken (id STRING) USING 'iceberg'", Some("spark"))
|
||||
.expect_err("quoted Spark datasource provider must remain invalid");
|
||||
assert!(provider_error.contains("Line: 1, Column:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spark_selects_still_report_query_references() {
|
||||
let analysis = analyze_sql_references("SELECT s.id FROM source_flow s", Some("spark"))
|
||||
.expect("ordinary Spark SELECT should analyze");
|
||||
|
||||
assert_eq!(analysis.tables[0].name, "source_flow");
|
||||
assert_eq!(analysis.tables[0].alias.as_deref(), Some("s"));
|
||||
assert_eq!(analysis.columns[0].name, "id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_dialect_still_rejects_spark_datasource_clauses() {
|
||||
let error = analyze_sql_references("CREATE TABLE account_flow (id STRING) USING iceberg", Some("generic"))
|
||||
.expect_err("Spark datasource clauses must remain dialect-specific");
|
||||
|
||||
assert!(error.contains("USING"));
|
||||
}
|
||||
Reference in new issue
Block a user