feat(d1): add Cloudflare D1 database support

This commit is contained in:
Anubis authored and GitHub committed 2026-07-12 22:03:57 +08:00
1 parent 977e7f78ca
commit 2a92746dc0
61 files changed
+1732 -69

No files matched your search

@@ -116,6 +116,35 @@
"driverManagement": false
}
},
{
"dbType": "cloudflare-d1",
"label": "Cloudflare D1",
"runtimeMode": "native",
"mcpMode": "bridge",
"singleConnectionPool": true,
"metadataConnectionScoped": false,
"skipTcpProbe": true,
"defaultPort": 443,
"supportLevel": "operate",
"capabilities": {
"queryExecution": true,
"metadataBrowse": true,
"objectBrowser": true,
"objectSource": true,
"schemaSearch": true,
"diagram": true,
"tableDataEdit": true,
"tableStructureEdit": false,
"tableImport": true,
"dataTransfer": true,
"sqlFileExecution": true,
"databaseCreate": false,
"fieldLineage": true,
"sqlExplain": false,
"userAdmin": false,
"driverManagement": false
}
},
{
"dbType": "redis",
"label": "Redis",
+57 -5
View File
@@ -86,6 +86,7 @@ pub enum PoolKind {
Sqlite(db::sqlite::SqliteHandle),
Rqlite(db::rqlite_driver::RqliteClient),
Turso(db::turso_driver::TursoClient),
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
Redis(db::redis_driver::RedisConnection),
DuckDb(DuckDbHandle),
DuckDbWorker(DuckDbWorkerHandle),
@@ -244,7 +245,11 @@ pub fn database_connection_config(config: &ConnectionConfig, database: Option<&s
if let Some(db) = database {
if !matches!(
db_config.db_type,
DatabaseType::Oracle | DatabaseType::Dameng | DatabaseType::MongoDb | DatabaseType::OceanbaseOracle
DatabaseType::Oracle
| DatabaseType::Dameng
| DatabaseType::MongoDb
| DatabaseType::OceanbaseOracle
| DatabaseType::CloudflareD1
) {
db_config.database = Some(db.to_string());
}
@@ -1083,6 +1088,9 @@ impl AppState {
db::turso_driver::test_connection(&client, connect_timeout).await?;
PoolKind::Turso(client)
}
DatabaseType::CloudflareD1 => {
PoolKind::CloudflareD1(db::cloudflare_d1_driver::connect(&db_config, connect_timeout).await?)
}
DatabaseType::Redis => {
let con = if db_config.uses_redis_cluster() {
db::redis_driver::RedisConnection::Cluster(
@@ -1848,6 +1856,18 @@ impl AppState {
}
}
}
PoolKind::CloudflareD1(client) => {
let client = client.clone();
drop(connections);
let timeout = crate::db::connection_timeout();
match db::cloudflare_d1_driver::test_connection(&client, timeout).await {
Ok(()) => false,
Err(err) => {
log::warn!("Cloudflare D1 connection pool '{pool_key}' is stale: {err}");
true
}
}
}
PoolKind::Agent(client) => {
let client = client.clone();
drop(connections);
@@ -2360,6 +2380,15 @@ impl AppState {
false
}
},
PoolKind::CloudflareD1(client) => {
match db::cloudflare_d1_driver::test_connection(client, timeout).await {
Ok(()) => true,
Err(e) => {
log::warn!("Cloudflare D1 connection pool '{key}' is unhealthy: {e}");
false
}
}
}
PoolKind::Agent(client) => {
let mut agent = client.lock().await;
match agent.test_connection(serde_json::json!({})).await {
@@ -2769,12 +2798,13 @@ fn session_scoped_pool_key_for(
client_session_id: Option<&str>,
) -> String {
let shares_base_pool = config.is_some_and(|config| {
config.db_type == DatabaseType::DuckDb
matches!(config.db_type, DatabaseType::DuckDb | DatabaseType::CloudflareD1)
|| (config.db_type == DatabaseType::Sqlite && db::sqlite::is_memory_database_path(&config.host))
});
if shares_base_pool {
// In-memory SQLite databases only exist inside one connection. A session-scoped
// handle would silently point query/data tabs at a different empty database.
// DuckDB and D1 already use connection-scoped handles. In-memory SQLite databases
// only exist inside one connection, so a session-scoped handle would point tabs at
// a different empty database.
return base_pool_key;
}
session_scoped_pool_key(base_pool_key, client_session_id)
@@ -2787,6 +2817,7 @@ fn clone_pool_kind(pool: &PoolKind) -> PoolKind {
PoolKind::Sqlite(p) => PoolKind::Sqlite(p.clone()),
PoolKind::Rqlite(client) => PoolKind::Rqlite(client.clone()),
PoolKind::Turso(client) => PoolKind::Turso(client.clone()),
PoolKind::CloudflareD1(client) => PoolKind::CloudflareD1(client.clone()),
#[cfg(feature = "duckdb-bundled")]
PoolKind::DuckDb(con) => PoolKind::DuckDb(con.clone()),
#[cfg(feature = "duckdb-bundled")]
@@ -2821,6 +2852,7 @@ pub async fn close_pool_kind(pool: PoolKind) {
PoolKind::Sqlite(_) => {}
PoolKind::Rqlite(_) => {}
PoolKind::Turso(_) => {}
PoolKind::CloudflareD1(_) => {}
PoolKind::Redis(conn) => {
drop(conn);
}
@@ -3939,6 +3971,20 @@ mod tests {
assert_eq!(scoped.database.as_deref(), Some("admin"));
}
#[test]
fn cloudflare_d1_query_namespace_does_not_replace_database_id() {
let mut config = mysql_config(Some("database-uuid"));
config.db_type = DatabaseType::CloudflareD1;
let scoped = database_connection_config(&config, Some("main"));
assert_eq!(scoped.database.as_deref(), Some("database-uuid"));
assert_eq!(
super::base_pool_key_for(Some(DatabaseType::CloudflareD1), "d1-conn", Some("main"), false),
"d1-conn"
);
}
#[test]
fn oracle_database_connection_ignores_requested_database() {
let mut config = mysql_config(Some("ORCL"));
@@ -4001,7 +4047,6 @@ mod tests {
super::session_scoped_pool_key_for(Some(&duckdb), "duckdb-conn".to_string(), Some("tab-1")),
"duckdb-conn"
);
let mut sqlite_memory = mysql_config(None);
sqlite_memory.db_type = DatabaseType::Sqlite;
sqlite_memory.host = " :MeMoRy: ".to_string();
@@ -4015,6 +4060,13 @@ mod tests {
super::session_scoped_pool_key_for(Some(&sqlite_memory), "sqlite-file".to_string(), Some("tab-1")),
"sqlite-file:session:tab-1"
);
let mut cloudflare_d1 = mysql_config(None);
cloudflare_d1.db_type = DatabaseType::CloudflareD1;
assert_eq!(
super::session_scoped_pool_key_for(Some(&cloudflare_d1), "d1-conn".to_string(), Some("tab-1")),
"d1-conn"
);
}
#[test]
+1
View File
@@ -2088,6 +2088,7 @@ fn uses_keyless_row_predicate(database_type: Option<DatabaseType>) -> bool {
| DatabaseType::Sqlite
| DatabaseType::Rqlite
| DatabaseType::Turso
| DatabaseType::CloudflareD1
| DatabaseType::DuckDb
| DatabaseType::SqlServer
| DatabaseType::Oracle
@@ -16,6 +16,7 @@ pub fn is_single_connection_pool(db_type: &DatabaseType) -> bool {
| DatabaseType::DuckDb
| DatabaseType::Rqlite
| DatabaseType::Turso
| DatabaseType::CloudflareD1
| DatabaseType::MongoDb
| DatabaseType::Oracle
| DatabaseType::Dameng
@@ -43,6 +44,7 @@ pub fn skips_tcp_probe(db_type: &DatabaseType) -> bool {
DatabaseType::Sqlite
| DatabaseType::DuckDb
| DatabaseType::Turso
| DatabaseType::CloudflareD1
| DatabaseType::Jdbc
| DatabaseType::MessageQueue
) || is_agent_type(db_type)
@@ -75,6 +77,13 @@ mod tests {
assert!(!is_local_file_db_type(&DatabaseType::Redis));
assert!(!is_local_file_db_type(&DatabaseType::MongoDb));
assert!(!is_local_file_db_type(&DatabaseType::Turso));
assert!(!is_local_file_db_type(&DatabaseType::CloudflareD1));
assert!(!is_local_file_db_type(&DatabaseType::Rqlite));
}
#[test]
fn cloudflare_d1_uses_a_single_http_pool_without_tcp_probe() {
assert!(is_single_connection_pool(&DatabaseType::CloudflareD1));
assert!(skips_tcp_probe(&DatabaseType::CloudflareD1));
}
}
@@ -0,0 +1,94 @@
use crate::models::connection::DatabaseType;
use crate::table_import::{mapping_indexes_for_columns, ImportSqlBatch, TableImportColumnMapping};
use crate::transfer::generate_insert_typed;
use super::sql_limits::build_sql_batches;
pub(crate) fn build_import_insert_batches(
rows: &[Vec<serde_json::Value>],
source_columns: &[String],
mappings: &[TableImportColumnMapping],
target_column_types: &[(String, String)],
table: &str,
schema: &str,
max_rows: usize,
) -> Result<Vec<ImportSqlBatch>, String> {
let mapped = mapping_indexes_for_columns(source_columns, mappings)?;
let columns = mapped.iter().map(|(_, target)| target.clone()).collect::<Vec<_>>();
let column_types = columns
.iter()
.map(|column| {
target_column_types
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case(column))
.map(|(_, data_type)| data_type.clone())
})
.collect::<Vec<_>>();
let rows = rows
.iter()
.map(|row| {
mapped
.iter()
.map(|(source_index, _)| row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null))
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
build_sql_batches(rows.len(), max_rows, "import row", |range| {
generate_insert_typed(&columns, &column_types, &rows[range], table, schema, &DatabaseType::CloudflareD1)
})
.map(|batches| {
batches.into_iter().map(|batch| ImportSqlBatch { sql: batch.sql, row_count: batch.item_count }).collect()
})
}
pub(crate) fn build_streaming_import_insert_batch(
rows: &[Vec<serde_json::Value>],
source_columns: &[String],
mappings: &[TableImportColumnMapping],
target_column_types: &[(String, String)],
table: &str,
schema: &str,
max_rows: usize,
) -> Result<Option<ImportSqlBatch>, String> {
let batches =
build_import_insert_batches(rows, source_columns, mappings, target_column_types, table, schema, max_rows)?;
if batches.is_empty() {
return Ok(None);
}
let row_count = batches.iter().map(|batch| batch.row_count).sum();
let sql = batches.into_iter().map(|batch| batch.sql).collect::<Vec<_>>().join(";\n");
Ok(Some(ImportSqlBatch { sql, row_count }))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn streaming_import_joins_individually_size_limited_statements() {
let rows = vec![vec![serde_json::json!("a".repeat(60_000))], vec![serde_json::json!("b".repeat(60_000))]];
let columns = vec!["value".to_string()];
let mappings = vec![TableImportColumnMapping {
source_column: "value".to_string(),
target_column: "value".to_string(),
target_data_type: None,
}];
let batch = crate::table_import::build_import_insert_batch_from_rows(
&rows,
&columns,
&mappings,
&[],
"events",
"main",
&DatabaseType::CloudflareD1,
)
.unwrap()
.unwrap();
assert_eq!(batch.row_count, 2);
assert_eq!(batch.sql.split(";\n").count(), 2);
assert!(batch.sql.split(";\n").all(|statement| statement.len() <= super::super::MAX_SQL_STATEMENT_BYTES));
}
}
+628
View File
@@ -0,0 +1,628 @@
use reqwest::Client as HttpClient;
use serde::Deserialize;
use std::time::{Duration, Instant};
use super::{http_client_builder, json_value_for_js, with_connection_timeout};
use crate::models::connection::ConnectionConfig;
use crate::types::{
ColumnInfo, DatabaseInfo, ForeignKeyInfo, IndexInfo, ObjectSource, ObjectSourceKind, QueryResult, TableInfo,
TriggerInfo,
};
mod import;
mod sql_guard;
mod sql_lexer;
mod sql_limits;
#[cfg(test)]
mod transfer_tests;
mod trigger;
pub(crate) use import::{build_import_insert_batches, build_streaming_import_insert_batch};
pub use sql_limits::MAX_SQL_STATEMENT_BYTES;
const CLOUDFLARE_API_BASE_URL: &str = "https://api.cloudflare.com/client/v4";
#[derive(Clone)]
pub struct CloudflareD1Client {
http: HttpClient,
endpoint: String,
api_token: String,
}
impl CloudflareD1Client {
pub fn new(account_id: &str, database_id: &str, api_token: &str, timeout: Duration) -> Result<Self, String> {
let account_id = required_identifier(account_id, "Cloudflare Account ID")?;
let database_id = required_identifier(database_id, "Cloudflare D1 Database ID")?;
let endpoint = format!("{CLOUDFLARE_API_BASE_URL}/accounts/{account_id}/d1/database/{database_id}/raw");
Self::with_endpoint(endpoint, api_token, timeout)
}
fn with_endpoint(endpoint: String, api_token: &str, timeout: Duration) -> Result<Self, String> {
let api_token = required_field(api_token, "Cloudflare API Token")?;
let http = http_client_builder(timeout)
.build()
.map_err(|error| format!("Failed to configure Cloudflare D1 HTTP client: {error}"))?;
Ok(Self { http, endpoint, api_token: api_token.to_string() })
}
fn post_raw(&self, sql: &str) -> reqwest::RequestBuilder {
self.http
.post(&self.endpoint)
.bearer_auth(&self.api_token)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.json(&serde_json::json!({ "sql": sql }))
}
}
#[derive(Debug, Deserialize)]
struct D1ApiResponse {
#[serde(default)]
success: Option<bool>,
#[serde(default)]
errors: Vec<D1ApiError>,
#[serde(default)]
result: Vec<D1StatementResult>,
}
#[derive(Debug, Deserialize)]
struct D1ApiError {
#[serde(default)]
code: Option<u64>,
message: String,
}
#[derive(Debug, Default, Deserialize)]
struct D1StatementResult {
#[serde(default)]
success: Option<bool>,
#[serde(default)]
results: D1RawResult,
#[serde(default)]
meta: D1Meta,
}
#[derive(Debug, Default, Deserialize)]
struct D1RawResult {
#[serde(default)]
columns: Vec<String>,
#[serde(default)]
rows: Vec<Vec<serde_json::Value>>,
}
#[derive(Debug, Default, Deserialize)]
struct D1Meta {
#[serde(default)]
changes: Option<u64>,
}
pub async fn test_connection(client: &CloudflareD1Client, timeout: Duration) -> Result<(), String> {
with_connection_timeout("Cloudflare D1", timeout, async { send_raw(client, "SELECT 1").await.map(|_| ()) }).await
}
pub async fn connect(config: &ConnectionConfig, timeout: Duration) -> Result<CloudflareD1Client, String> {
let client = CloudflareD1Client::new(
&config.host,
config.database.as_deref().unwrap_or_default(),
&config.password,
timeout,
)?;
test_connection(&client, timeout).await?;
Ok(client)
}
pub async fn list_databases(_client: &CloudflareD1Client) -> Result<Vec<DatabaseInfo>, String> {
Ok(vec![DatabaseInfo { name: "main".to_string() }])
}
pub async fn list_tables(client: &CloudflareD1Client, _schema: &str) -> Result<Vec<TableInfo>, String> {
let result = query_inner(
client,
"SELECT name, type FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' AND name <> '_cf_KV' ORDER BY name",
)
.await?;
Ok(result
.rows
.into_iter()
.map(|row| {
let table_type = value_as_string(row.get(1)).unwrap_or_else(|| "table".to_string());
TableInfo {
name: value_as_string(row.first()).unwrap_or_default(),
table_type: if table_type.eq_ignore_ascii_case("view") { "VIEW" } else { "BASE TABLE" }.to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}
})
.collect())
}
pub async fn get_columns(client: &CloudflareD1Client, _schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
// table_info omits generated columns; table_xinfo exposes them and marks
// virtual/stored generated columns with hidden values 2 and 3.
let result = query_inner(client, &format!("PRAGMA table_xinfo({})", sqlite_ident(table))).await?;
Ok(result
.rows
.into_iter()
.map(|row| {
let hidden = value_by_column(&result.columns, &row, "hidden")
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(0);
ColumnInfo {
name: value_by_column(&result.columns, &row, "name").unwrap_or_default(),
data_type: value_by_column(&result.columns, &row, "type").unwrap_or_default(),
is_nullable: value_by_column(&result.columns, &row, "notnull")
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(0)
== 0,
column_default: value_by_column(&result.columns, &row, "dflt_value"),
is_primary_key: value_by_column(&result.columns, &row, "pk")
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(0)
> 0,
extra: match hidden {
2 => Some("generated always as virtual".to_string()),
3 => Some("generated always as stored".to_string()),
_ => None,
},
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
..Default::default()
}
})
.collect())
}
pub async fn list_indexes(client: &CloudflareD1Client, _schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
let result = query_inner(client, &format!("PRAGMA index_list({})", sqlite_ident(table))).await?;
let mut indexes = Vec::new();
for row in result.rows {
let name = value_by_column(&result.columns, &row, "name").unwrap_or_default();
if name.is_empty() {
continue;
}
let is_unique =
value_by_column(&result.columns, &row, "unique").and_then(|value| value.parse::<i64>().ok()).unwrap_or(0)
!= 0;
let origin = value_by_column(&result.columns, &row, "origin").unwrap_or_default();
let column_result = query_inner(client, &format!("PRAGMA index_info({})", sqlite_ident(&name))).await?;
let columns =
column_result.rows.iter().filter_map(|row| value_by_column(&column_result.columns, row, "name")).collect();
indexes.push(IndexInfo {
name,
columns,
is_unique,
is_primary: origin == "pk",
filter: None,
index_type: None,
included_columns: None,
comment: None,
});
}
Ok(indexes)
}
pub async fn list_foreign_keys(
client: &CloudflareD1Client,
_schema: &str,
table: &str,
) -> Result<Vec<ForeignKeyInfo>, String> {
let result = query_inner(client, &format!("PRAGMA foreign_key_list({})", sqlite_ident(table))).await?;
Ok(result
.rows
.into_iter()
.map(|row| ForeignKeyInfo {
name: format!("fk_{}", value_by_column(&result.columns, &row, "id").unwrap_or_else(|| "0".to_string())),
column: value_by_column(&result.columns, &row, "from").unwrap_or_default(),
ref_schema: None,
ref_table: value_by_column(&result.columns, &row, "table").unwrap_or_default(),
ref_column: value_by_column(&result.columns, &row, "to").unwrap_or_default(),
on_update: value_by_column(&result.columns, &row, "on_update"),
on_delete: value_by_column(&result.columns, &row, "on_delete"),
})
.collect())
}
pub async fn list_triggers(
client: &CloudflareD1Client,
_schema: &str,
table: &str,
) -> Result<Vec<TriggerInfo>, String> {
let result = query_inner(
client,
&format!(
"SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND tbl_name = {} ORDER BY name",
sqlite_string(table)
),
)
.await?;
Ok(result
.rows
.into_iter()
.map(|row| {
let statement = value_as_string(row.get(1));
let metadata = trigger::metadata_from_sql(statement.as_deref().unwrap_or_default());
TriggerInfo {
name: value_as_string(row.first()).unwrap_or_default(),
event: metadata.event.to_string(),
timing: metadata.timing.to_string(),
statement,
}
})
.collect())
}
pub async fn table_ddl(client: &CloudflareD1Client, table: &str) -> Result<String, String> {
first_string_cell(
query_inner(
client,
&format!("SELECT sql FROM sqlite_master WHERE type='table' AND name={}", sqlite_string(table)),
)
.await?,
)
}
pub async fn object_source(
client: &CloudflareD1Client,
name: &str,
object_type: &ObjectSourceKind,
) -> Result<ObjectSource, String> {
let kind = match object_type {
ObjectSourceKind::View => "view",
_ => return Err("Object source is not supported for this Cloudflare D1 object type".to_string()),
};
let source = first_string_cell(
query_inner(
client,
&format!(
"SELECT sql FROM sqlite_master WHERE type={} AND name={}",
sqlite_string(kind),
sqlite_string(name)
),
)
.await?,
)?;
Ok(ObjectSource { name: name.to_string(), object_type: object_type.clone(), schema: None, source, editable: None })
}
pub async fn execute_query_with_max_rows(
client: &CloudflareD1Client,
sql: &str,
max_rows: Option<usize>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let sql = sql.trim();
if sql.is_empty() {
return Ok(empty_query_result(start.elapsed().as_millis()));
}
sql_guard::validate_sql(sql)?;
sql_limits::validate_statement_sizes(sql)?;
let mut statements = send_raw(client, sql).await?;
let affected_rows: u64 = statements.iter().filter_map(|statement| statement.meta.changes).sum();
let display = statements.pop().unwrap_or_default();
Ok(query_result(display.results, affected_rows, start.elapsed().as_millis(), max_rows))
}
async fn query_inner(client: &CloudflareD1Client, sql: &str) -> Result<D1RawResult, String> {
send_raw(client, sql)
.await?
.into_iter()
.next()
.map(|statement| statement.results)
.ok_or_else(|| "Cloudflare D1 returned no result".to_string())
}
async fn send_raw(client: &CloudflareD1Client, sql: &str) -> Result<Vec<D1StatementResult>, String> {
let response =
client.post_raw(sql).send().await.map_err(|error| format!("Cloudflare D1 request failed: {error}"))?;
let status = response.status();
let body = response.text().await.map_err(|error| format!("Cloudflare D1 response read failed: {error}"))?;
let parsed: D1ApiResponse = serde_json::from_str(&body)
.map_err(|error| format!("Cloudflare D1 response parse failed: {error}; body: {}", response_excerpt(&body)))?;
if !status.is_success() || parsed.success == Some(false) {
return Err(format_d1_error(status.as_u16(), &parsed.errors, &body));
}
if parsed.result.iter().any(|result| result.success == Some(false)) {
return Err(format_d1_error(status.as_u16(), &parsed.errors, &body));
}
if parsed.result.is_empty() {
return Err("Cloudflare D1 returned no statement results".to_string());
}
Ok(parsed.result)
}
fn query_result(
mut result: D1RawResult,
affected_rows: u64,
execution_time_ms: u128,
max_rows: Option<usize>,
) -> QueryResult {
result.rows = result.rows.into_iter().map(|row| row.into_iter().map(json_value_for_js).collect()).collect();
let row_limit = max_rows.unwrap_or(crate::query::MAX_ROWS).max(1);
let truncated = result.rows.len() > row_limit;
if truncated {
result.rows.truncate(row_limit);
}
QueryResult {
columns: result.columns,
column_types: Vec::new(),
column_sortables: vec![],
rows: result.rows,
affected_rows,
execution_time_ms,
truncated,
session_id: None,
has_more: false,
}
}
fn empty_query_result(execution_time_ms: u128) -> QueryResult {
query_result(D1RawResult::default(), 0, execution_time_ms, None)
}
fn format_d1_error(status: u16, errors: &[D1ApiError], body: &str) -> String {
let details = errors
.iter()
.map(|error| match error.code {
Some(code) => format!("{} ({code})", error.message),
None => error.message.clone(),
})
.collect::<Vec<_>>()
.join("; ");
if details.is_empty() {
format!("Cloudflare D1 API error ({status}): {}", response_excerpt(body))
} else {
format!("Cloudflare D1 API error ({status}): {details}")
}
}
fn response_excerpt(body: &str) -> String {
const MAX_CHARS: usize = 1_000;
let mut excerpt = body.chars().take(MAX_CHARS).collect::<String>();
if body.chars().count() > MAX_CHARS {
excerpt.push('…');
}
excerpt
}
fn required_field<'a>(value: &'a str, label: &str) -> Result<&'a str, String> {
let value = value.trim();
if value.is_empty() {
Err(format!("{label} is required"))
} else {
Ok(value)
}
}
fn required_identifier<'a>(value: &'a str, label: &str) -> Result<&'a str, String> {
let value = required_field(value, label)?;
if value.chars().all(|character| character.is_ascii_alphanumeric() || character == '-' || character == '_') {
Ok(value)
} else {
Err(format!("{label} contains invalid characters"))
}
}
fn first_string_cell(result: D1RawResult) -> Result<String, String> {
result
.rows
.first()
.and_then(|row| row.first())
.and_then(|value| value_as_string(Some(value)))
.ok_or_else(|| "Object not found".to_string())
}
fn value_by_column(columns: &[String], row: &[serde_json::Value], name: &str) -> Option<String> {
columns
.iter()
.position(|column| column.eq_ignore_ascii_case(name))
.and_then(|index| row.get(index))
.and_then(|value| value_as_string(Some(value)))
}
fn value_as_string(value: Option<&serde_json::Value>) -> Option<String> {
match value? {
serde_json::Value::Null => None,
serde_json::Value::String(value) => Some(value.clone()),
serde_json::Value::Number(value) => Some(value.to_string()),
serde_json::Value::Bool(value) => Some(value.to_string()),
other => Some(other.to_string()),
}
}
fn sqlite_ident(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
fn sqlite_string(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[test]
fn validates_required_connection_fields() {
let timeout = Duration::from_secs(1);
assert!(CloudflareD1Client::new("", "database", "token", timeout).err().unwrap().contains("Account ID"));
assert!(CloudflareD1Client::new("account", "", "token", timeout).err().unwrap().contains("Database ID"));
assert!(CloudflareD1Client::new("account", "database", "", timeout).err().unwrap().contains("API Token"));
assert!(CloudflareD1Client::new("account/path", "database", "token", timeout)
.err()
.unwrap()
.contains("invalid characters"));
}
#[test]
fn parses_raw_api_response() {
let response: D1ApiResponse = serde_json::from_value(serde_json::json!({
"success": true,
"errors": [],
"result": [{
"success": true,
"results": { "columns": ["id", "name"], "rows": [[1, "Ada"]] },
"meta": { "changes": 0 }
}]
}))
.unwrap();
assert_eq!(response.success, Some(true));
assert_eq!(response.result[0].results.columns, ["id", "name"]);
assert_eq!(response.result[0].results.rows[0], [serde_json::json!(1), serde_json::json!("Ada")]);
}
#[test]
fn converts_and_truncates_query_result() {
let raw = D1RawResult {
columns: vec!["id".to_string()],
rows: vec![vec![serde_json::json!(1)], vec![serde_json::json!(2)]],
};
let result = query_result(raw, 0, 5, Some(1));
assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]);
assert!(result.truncated);
}
#[test]
fn limits_error_response_excerpts() {
let excerpt = response_excerpt(&"x".repeat(1_100));
assert_eq!(excerpt.chars().count(), 1_001);
assert!(excerpt.ends_with('…'));
}
#[tokio::test]
async fn sends_bearer_authenticated_raw_query_and_maps_response() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("POST /raw HTTP/1.1"));
assert!(request.to_ascii_lowercase().contains("authorization: bearer test-token"));
assert!(request.contains("\"sql\":\"SELECT 1\""));
let body = r#"{"success":true,"errors":[],"result":[{"success":true,"results":{"columns":["1"],"rows":[[1]]},"meta":{"changes":0}}]}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client =
CloudflareD1Client::with_endpoint(format!("http://{address}/raw"), "test-token", Duration::from_secs(2))
.unwrap();
let result = execute_query_with_max_rows(&client, "SELECT 1", None).await.unwrap();
assert_eq!(result.columns, ["1"]);
assert_eq!(result.rows, vec![vec![serde_json::json!(1)]]);
server.await.unwrap();
}
#[tokio::test]
async fn lists_tables_and_views_from_sqlite_master() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.contains("sqlite_master"));
assert!(request.contains("name <> '_cf_KV'"));
assert!(!request.contains("d1_%"));
let body = r#"{"success":true,"errors":[],"result":[{"success":true,"results":{"columns":["name","type"],"rows":[["users","table"],["active_users","view"]]},"meta":{"changes":0}}]}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client =
CloudflareD1Client::with_endpoint(format!("http://{address}/raw"), "test-token", Duration::from_secs(2))
.unwrap();
let tables = list_tables(&client, "main").await.unwrap();
assert_eq!(tables.len(), 2);
assert_eq!(tables[0].name, "users");
assert_eq!(tables[0].table_type, "BASE TABLE");
assert_eq!(tables[1].name, "active_users");
assert_eq!(tables[1].table_type, "VIEW");
server.await.unwrap();
}
#[tokio::test]
async fn lists_generated_columns_from_table_xinfo() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.contains("PRAGMA table_xinfo"));
let body = r#"{"success":true,"errors":[],"result":[{"success":true,"results":{"columns":["cid","name","type","notnull","dflt_value","pk","hidden"],"rows":[[0,"price","INTEGER",1,null,0,0],[1,"tax","INTEGER",0,null,0,2],[2,"total","INTEGER",0,null,0,3]]},"meta":{"changes":0}}]}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client =
CloudflareD1Client::with_endpoint(format!("http://{address}/raw"), "test-token", Duration::from_secs(2))
.unwrap();
let columns = get_columns(&client, "main", "orders").await.unwrap();
assert_eq!(columns.len(), 3);
assert_eq!(columns[0].extra, None);
assert_eq!(columns[1].extra.as_deref(), Some("generated always as virtual"));
assert_eq!(columns[2].extra.as_deref(), Some("generated always as stored"));
server.await.unwrap();
}
#[test]
fn rejects_oversized_statements_but_allows_large_multi_statement_batches() {
let oversized = format!("SELECT '{}'", "x".repeat(MAX_SQL_STATEMENT_BYTES));
let error = sql_limits::validate_statement_sizes(&oversized).unwrap_err();
assert!(error.contains("statement 1"));
assert!(error.contains("100000 bytes"));
let first = format!("SELECT '{}'", "a".repeat(60_000));
let second = format!("SELECT '{}'", "b".repeat(60_000));
assert!(sql_limits::validate_statement_sizes(&format!("{first}; {second}")).is_ok());
let trigger = format!(
"CREATE TRIGGER audit AFTER INSERT ON users BEGIN INSERT INTO logs VALUES ('{}'); INSERT INTO logs VALUES ('{}'); END",
"a".repeat(60_000),
"b".repeat(60_000)
);
assert!(sql_limits::validate_statement_sizes(&trigger).is_err());
}
async fn read_http_request(socket: &mut tokio::net::TcpStream) -> String {
let mut bytes = Vec::new();
let mut buffer = [0_u8; 2048];
loop {
let read = socket.read(&mut buffer).await.unwrap();
if read == 0 {
break;
}
bytes.extend_from_slice(&buffer[..read]);
let Some(header_end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") else {
continue;
};
let headers = String::from_utf8_lossy(&bytes[..header_end]);
let content_length = headers
.lines()
.find_map(|line| line.split_once(':').filter(|(name, _)| name.eq_ignore_ascii_case("content-length")))
.and_then(|(_, value)| value.trim().parse::<usize>().ok())
.unwrap_or(0);
if bytes.len() >= header_end + 4 + content_length {
break;
}
}
String::from_utf8(bytes).unwrap()
}
}
@@ -0,0 +1,168 @@
use super::sql_lexer::{complete_sql_statements, lex_sql, SqlLexeme};
const SUPPORTED_PRAGMAS: &[&str] = &[
"case_sensitive_like",
"defer_foreign_keys",
"foreign_key_check",
"foreign_key_list",
"foreign_keys",
"ignore_check_constraints",
"index_info",
"index_list",
"index_xinfo",
"legacy_alter_table",
"optimize",
"quick_check",
"recursive_triggers",
"reverse_unordered_selects",
"table_info",
"table_list",
"table_xinfo",
];
pub(super) fn validate_sql(sql: &str) -> Result<(), String> {
for statement in complete_sql_statements(sql) {
validate_statement(statement)?;
}
Ok(())
}
fn validate_statement(statement: &str) -> Result<(), String> {
let lexemes = lex_sql(statement);
let words = lexemes
.iter()
.filter_map(|lexeme| match lexeme {
SqlLexeme::Word(word) => Some(*word),
SqlLexeme::Number(_) | SqlLexeme::Symbol(_) | SqlLexeme::Semicolon(_) => None,
})
.collect::<Vec<_>>();
let Some(first) = words.first() else {
return Ok(());
};
if ["BEGIN", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", "START"]
.iter()
.any(|keyword| first.eq_ignore_ascii_case(keyword))
{
return Err(
"Cloudflare D1 does not support explicit transaction or savepoint statements. Submit the SQL statements together without BEGIN, COMMIT, ROLLBACK, SAVEPOINT, or RELEASE."
.to_string(),
);
}
if first.eq_ignore_ascii_case("ATTACH") || first.eq_ignore_ascii_case("DETACH") {
return Err(
"Cloudflare D1 does not support ATTACH or DETACH. Each connection targets one provider-managed D1 database."
.to_string(),
);
}
if first.eq_ignore_ascii_case("CREATE") {
validate_create_statement(&words)?;
}
if first.eq_ignore_ascii_case("PRAGMA") {
validate_pragma(&words, &lexemes)?;
}
Ok(())
}
fn validate_create_statement(words: &[&str]) -> Result<(), String> {
if words.get(1).is_some_and(|word| word.eq_ignore_ascii_case("TEMP") || word.eq_ignore_ascii_case("TEMPORARY")) {
return Err(
"Cloudflare D1 does not support temporary tables, indexes, views, or triggers. Create a persistent object instead."
.to_string(),
);
}
let creates_virtual_table = words
.get(1..3)
.is_some_and(|prefix| prefix[0].eq_ignore_ascii_case("VIRTUAL") && prefix[1].eq_ignore_ascii_case("TABLE"));
if !creates_virtual_table {
return Ok(());
}
let module =
words.iter().position(|word| word.eq_ignore_ascii_case("USING")).and_then(|index| words.get(index + 1));
if module.is_some_and(|module| module.eq_ignore_ascii_case("fts5") || module.eq_ignore_ascii_case("fts5vocab")) {
return Ok(());
}
Err("Cloudflare D1 only supports FTS5 and fts5vocab virtual table modules.".to_string())
}
fn validate_pragma(words: &[&str], lexemes: &[SqlLexeme<'_>]) -> Result<(), String> {
let mut pragma = words.get(1).copied();
if pragma.is_some_and(|word| word.eq_ignore_ascii_case("main") || word.eq_ignore_ascii_case("temp")) {
pragma = words.get(2).copied();
}
let Some(pragma) = pragma else {
return Err("Cloudflare D1 requires a supported PRAGMA name.".to_string());
};
if !SUPPORTED_PRAGMAS.iter().any(|supported| pragma.eq_ignore_ascii_case(supported)) {
return Err(format!(
"Cloudflare D1 does not support PRAGMA {pragma}. Use one of the documented D1-compatible PRAGMA statements."
));
}
if pragma.eq_ignore_ascii_case("optimize") && contains_negative_one(lexemes) {
return Err("Cloudflare D1 does not support PRAGMA optimize(-1); use PRAGMA optimize instead.".to_string());
}
Ok(())
}
fn contains_negative_one(lexemes: &[SqlLexeme<'_>]) -> bool {
lexemes
.windows(2)
.any(|pair| matches!(pair[0], SqlLexeme::Symbol(b'-')) && matches!(pair[1], SqlLexeme::Number("1")))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn accepts_supported_d1_sql_features() {
assert!(validate_sql(
"CREATE TRIGGER audit AFTER INSERT ON users BEGIN INSERT INTO logs VALUES (NEW.id); END;"
)
.is_ok());
assert!(
validate_sql("CREATE VIRTUAL TABLE docs USING fts5(title, body); SELECT json_extract('{}', '$');").is_ok()
);
assert!(validate_sql("PRAGMA table_info(users); PRAGMA optimize;").is_ok());
}
#[test]
fn rejects_explicit_transaction_control() {
for sql in [
"BEGIN; INSERT INTO users VALUES (1); COMMIT;",
"SAVEPOINT before_update",
"ROLLBACK TO before_update",
"RELEASE before_update",
] {
let error = validate_sql(sql).unwrap_err();
assert!(error.contains("does not support explicit transaction"));
}
}
#[test]
fn rejects_attached_temporary_and_unknown_virtual_databases() {
assert!(validate_sql("ATTACH DATABASE 'other.db' AS other").unwrap_err().contains("ATTACH"));
assert!(validate_sql("CREATE TEMP TABLE scratch(id INTEGER)").unwrap_err().contains("temporary"));
assert!(validate_sql("CREATE VIRTUAL TABLE places USING rtree(id, min_x, max_x)")
.unwrap_err()
.contains("only supports FTS5"));
}
#[test]
fn rejects_unsupported_pragma_forms() {
assert!(validate_sql("PRAGMA journal_mode=WAL").unwrap_err().contains("PRAGMA journal_mode"));
assert!(validate_sql("PRAGMA data_version").unwrap_err().contains("PRAGMA data_version"));
assert!(validate_sql("PRAGMA optimize(-1)").unwrap_err().contains("optimize(-1)"));
assert!(validate_sql("PRAGMA optimize /* -1 is only a comment */").is_ok());
}
}
@@ -0,0 +1,180 @@
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum SqlLexeme<'a> {
Word(&'a str),
Number(&'a str),
Symbol(u8),
Semicolon(usize),
}
pub(super) fn lex_sql(sql: &str) -> Vec<SqlLexeme<'_>> {
let bytes = sql.as_bytes();
let mut lexemes = Vec::new();
let mut index = 0;
while index < bytes.len() {
match bytes[index] {
b'-' if bytes.get(index + 1) == Some(&b'-') => {
index += 2;
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
}
b'/' if bytes.get(index + 1) == Some(&b'*') => {
index += 2;
while index + 1 < bytes.len() && !(bytes[index] == b'*' && bytes[index + 1] == b'/') {
index += 1;
}
index = (index + 2).min(bytes.len());
}
quote @ (b'\'' | b'"' | b'`') => {
index += 1;
while index < bytes.len() {
if bytes[index] != quote {
index += 1;
continue;
}
if bytes.get(index + 1) == Some(&quote) {
index += 2;
} else {
index += 1;
break;
}
}
}
b'[' => {
index += 1;
while index < bytes.len() {
if bytes[index] != b']' {
index += 1;
continue;
}
if bytes.get(index + 1) == Some(&b']') {
index += 2;
} else {
index += 1;
break;
}
}
}
b';' => {
lexemes.push(SqlLexeme::Semicolon(index));
index += 1;
}
byte if byte.is_ascii_digit() => {
let start = index;
index += 1;
while index < bytes.len() && bytes[index].is_ascii_digit() {
index += 1;
}
lexemes.push(SqlLexeme::Number(&sql[start..index]));
}
byte if byte.is_ascii_alphabetic() || byte == b'_' => {
let start = index;
index += 1;
while index < bytes.len()
&& (bytes[index].is_ascii_alphanumeric() || matches!(bytes[index], b'_' | b'$'))
{
index += 1;
}
lexemes.push(SqlLexeme::Word(&sql[start..index]));
}
byte if matches!(byte, b'(' | b')' | b'=' | b'.' | b',' | b'-' | b'+') => {
lexemes.push(SqlLexeme::Symbol(byte));
index += 1;
}
_ => index += 1,
}
}
lexemes
}
pub(super) fn complete_sql_statements(sql: &str) -> Vec<&str> {
let mut statements = Vec::new();
let mut statement_start = 0;
let mut prefix = Vec::with_capacity(3);
let mut is_trigger = false;
let mut trigger_body_started = false;
let mut trigger_depth = 0_usize;
for lexeme in lex_sql(sql) {
match lexeme {
SqlLexeme::Word(word) => {
if prefix.len() < 3 {
prefix.push(word);
is_trigger = is_create_trigger_prefix(&prefix);
}
if !is_trigger {
continue;
}
if !trigger_body_started && word.eq_ignore_ascii_case("BEGIN") {
trigger_body_started = true;
trigger_depth = 1;
} else if trigger_body_started
&& (word.eq_ignore_ascii_case("BEGIN") || word.eq_ignore_ascii_case("CASE"))
{
trigger_depth += 1;
} else if trigger_body_started && word.eq_ignore_ascii_case("END") {
trigger_depth = trigger_depth.saturating_sub(1);
}
}
SqlLexeme::Semicolon(position) => {
if is_trigger && trigger_body_started && trigger_depth > 0 {
continue;
}
push_statement(sql, statement_start, position, &mut statements);
statement_start = position + 1;
prefix.clear();
is_trigger = false;
trigger_body_started = false;
trigger_depth = 0;
}
SqlLexeme::Number(_) | SqlLexeme::Symbol(_) => {}
}
}
push_statement(sql, statement_start, sql.len(), &mut statements);
statements
}
fn is_create_trigger_prefix(words: &[&str]) -> bool {
words.first().is_some_and(|word| word.eq_ignore_ascii_case("CREATE"))
&& (words.get(1).is_some_and(|word| word.eq_ignore_ascii_case("TRIGGER"))
|| (words
.get(1)
.is_some_and(|word| word.eq_ignore_ascii_case("TEMP") || word.eq_ignore_ascii_case("TEMPORARY"))
&& words.get(2).is_some_and(|word| word.eq_ignore_ascii_case("TRIGGER"))))
}
fn push_statement<'a>(sql: &'a str, start: usize, end: usize, statements: &mut Vec<&'a str>) {
let statement = sql[start..end].trim();
if !statement.is_empty() {
statements.push(statement);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keeps_trigger_body_semicolons_in_one_statement() {
let sql = "CREATE TRIGGER audit AFTER UPDATE ON users BEGIN INSERT INTO logs VALUES ('a;b'); UPDATE stats SET value = CASE WHEN value > 0 THEN value ELSE 0 END; END; SELECT 1;";
let statements = complete_sql_statements(sql);
assert_eq!(statements.len(), 2);
assert!(statements[0].starts_with("CREATE TRIGGER"));
assert!(statements[0].ends_with("END"));
assert_eq!(statements[1], "SELECT 1");
}
#[test]
fn ignores_trigger_words_in_comments_and_strings() {
let sql = "SELECT 'CREATE TRIGGER fake; BEGIN END'; -- CREATE TRIGGER ignored\nSELECT 2;";
assert_eq!(
complete_sql_statements(sql),
["SELECT 'CREATE TRIGGER fake; BEGIN END'", "-- CREATE TRIGGER ignored\nSELECT 2"]
);
}
}
@@ -0,0 +1,115 @@
use std::ops::Range;
pub const MAX_SQL_STATEMENT_BYTES: usize = 100_000;
#[derive(Debug, PartialEq, Eq)]
pub(super) struct SqlBatch {
pub sql: String,
pub item_count: usize,
}
pub(super) fn build_sql_batches<F>(
item_count: usize,
max_items: usize,
item_label: &str,
render: F,
) -> Result<Vec<SqlBatch>, String>
where
F: Fn(Range<usize>) -> String,
{
let mut batches = Vec::new();
let mut start = 0;
let max_items = max_items.max(1);
while start < item_count {
let mut end = start + 1;
let mut accepted = render(start..end);
if accepted.len() > MAX_SQL_STATEMENT_BYTES {
return Err(format!(
"Cloudflare D1 {item_label} {} generates a {}-byte SQL statement; the maximum is {MAX_SQL_STATEMENT_BYTES} bytes. Reduce the value size because a single item cannot be split safely.",
start + 1,
accepted.len()
));
}
while end < item_count && end - start < max_items {
let candidate = render(start..end + 1);
if candidate.len() > MAX_SQL_STATEMENT_BYTES {
break;
}
accepted = candidate;
end += 1;
}
if !accepted.trim().is_empty() {
batches.push(SqlBatch { sql: accepted, item_count: end - start });
}
start = end;
}
Ok(batches)
}
pub(super) fn validate_statement_sizes(sql: &str) -> Result<(), String> {
let statements = super::sql_lexer::complete_sql_statements(sql);
for (index, statement) in statements.iter().enumerate() {
if statement.len() > MAX_SQL_STATEMENT_BYTES {
return Err(statement_size_error(index + 1, statement.len()));
}
}
Ok(())
}
fn statement_size_error(index: usize, byte_len: usize) -> String {
format!(
"Cloudflare D1 SQL statement {index} is {byte_len} bytes; the maximum is {MAX_SQL_STATEMENT_BYTES} bytes. Split the statement or reduce the batch/value size."
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn splits_utf8_sql_batches_by_bytes() {
let rows = ["你".repeat(20_000), "好".repeat(20_000)];
let batches = build_sql_batches(rows.len(), 100, "import row", |range| {
format!("INSERT INTO events VALUES ('{}')", rows[range].join("'), ('"))
})
.unwrap();
assert_eq!(batches.len(), 2);
assert!(batches.iter().all(|batch| batch.sql.len() <= MAX_SQL_STATEMENT_BYTES));
}
#[test]
fn rejects_a_single_oversized_item() {
let error = build_sql_batches(1, 100, "import row", |_| {
format!("INSERT INTO events VALUES ('{}')", "x".repeat(MAX_SQL_STATEMENT_BYTES))
})
.unwrap_err();
assert!(error.contains("import row 1"));
assert!(error.contains("maximum is 100000 bytes"));
}
#[test]
fn validates_trigger_and_other_batch_statements_individually() {
let trigger = "CREATE TRIGGER audit AFTER UPDATE ON users BEGIN INSERT INTO logs VALUES (NEW.id); END";
let first = format!("SELECT '{}'", "a".repeat(60_000));
let second = format!("SELECT '{}'", "b".repeat(60_000));
assert!(validate_statement_sizes(&format!("{trigger}; {first}; {second}")).is_ok());
}
#[test]
fn rejects_an_oversized_complete_trigger_statement() {
let trigger = format!(
"CREATE TRIGGER audit AFTER UPDATE ON users BEGIN INSERT INTO logs VALUES ('{}'); INSERT INTO logs VALUES ('{}'); END",
"a".repeat(60_000),
"b".repeat(60_000)
);
assert!(validate_statement_sizes(&trigger).is_err());
}
}
@@ -0,0 +1,19 @@
use serde_json::json;
use crate::models::connection::DatabaseType;
use crate::transfer::generate_upsert_typed;
#[test]
fn uses_sqlite_upsert_syntax() {
let sql = generate_upsert_typed(
&[String::from("id"), String::from("name")],
&[Some(String::from("integer")), Some(String::from("text"))],
&[vec![json!(1), json!("Ada")]],
"users",
"main",
&DatabaseType::CloudflareD1,
&[String::from("id")],
);
assert!(sql.contains("ON CONFLICT (\"id\") DO UPDATE SET"));
}
@@ -0,0 +1,71 @@
use super::sql_lexer::{lex_sql, SqlLexeme};
pub(super) struct TriggerMetadata {
pub timing: &'static str,
pub event: &'static str,
}
pub(super) fn metadata_from_sql(sql: &str) -> TriggerMetadata {
let words = lex_sql(sql)
.into_iter()
.filter_map(|lexeme| match lexeme {
SqlLexeme::Word(word) => Some(word),
SqlLexeme::Number(_) | SqlLexeme::Symbol(_) | SqlLexeme::Semicolon(_) => None,
})
.collect::<Vec<_>>();
let declaration_start =
words.iter().position(|word| word.eq_ignore_ascii_case("TRIGGER")).map_or(0, |index| index + 1);
let declaration_end = words[declaration_start..]
.iter()
.position(|word| word.eq_ignore_ascii_case("ON") || word.eq_ignore_ascii_case("BEGIN"))
.map_or(words.len(), |index| declaration_start + index);
let declaration = &words[declaration_start..declaration_end];
let timing = if declaration
.windows(2)
.any(|words| words[0].eq_ignore_ascii_case("INSTEAD") && words[1].eq_ignore_ascii_case("OF"))
{
"INSTEAD OF"
} else if declaration.iter().any(|word| word.eq_ignore_ascii_case("AFTER")) {
"AFTER"
} else {
// SQLite defaults to BEFORE when no timing keyword is present.
"BEFORE"
};
let event = declaration
.iter()
.find_map(|word| ["DELETE", "INSERT", "UPDATE"].into_iter().find(|event| word.eq_ignore_ascii_case(event)))
.unwrap_or("UNKNOWN");
TriggerMetadata { timing, event }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_event_from_declaration_not_trigger_body() {
let metadata = metadata_from_sql(
"CREATE TRIGGER audit AFTER UPDATE ON users BEGIN INSERT INTO audit_log VALUES (NEW.id); END",
);
assert_eq!(metadata.timing, "AFTER");
assert_eq!(metadata.event, "UPDATE");
}
#[test]
fn supports_default_and_instead_of_timing() {
let default_timing = metadata_from_sql(
"CREATE TRIGGER update_customer UPDATE OF name ON customers BEGIN DELETE FROM cache; END",
);
let instead_of = metadata_from_sql(
"CREATE TRIGGER [insert] INSTEAD OF INSERT ON customer_view BEGIN UPDATE customers SET name = NEW.name; END",
);
assert_eq!(default_timing.timing, "BEFORE");
assert_eq!(default_timing.event, "UPDATE");
assert_eq!(instead_of.timing, "INSTEAD OF");
assert_eq!(instead_of.event, "INSERT");
}
}
+2
View File
@@ -1,5 +1,7 @@
pub mod agent_driver;
pub mod clickhouse_driver;
pub mod cloudflare_d1;
pub use cloudflare_d1 as cloudflare_d1_driver;
pub mod duckdb_driver;
#[cfg(feature = "duckdb-bundled")]
pub mod duckdb_worker_process;
+27 -1
View File
@@ -439,6 +439,8 @@ pub enum DatabaseType {
Iris,
#[serde(rename = "turso")]
Turso,
#[serde(rename = "cloudflare-d1")]
CloudflareD1,
#[serde(rename = "influxdb")]
InfluxDb,
#[serde(rename = "questdb")]
@@ -761,7 +763,7 @@ impl ConnectionConfig {
},
DatabaseType::Redshift => Some("dev"),
DatabaseType::ClickHouse => Some("default"),
DatabaseType::Rqlite | DatabaseType::Turso => Some("main"),
DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1 => Some("main"),
DatabaseType::Gaussdb | DatabaseType::OpenGauss => Some("postgres"),
DatabaseType::Kwdb => Some("defaultdb"),
DatabaseType::Vastbase => Some("postgres"),
@@ -872,6 +874,7 @@ impl ConnectionConfig {
DatabaseType::ClickHouse => clickhouse_http_url(self, raw_host, port),
DatabaseType::Rqlite => rqlite_http_url(self, raw_host, port),
DatabaseType::Turso => turso_http_url(self, raw_host, port),
DatabaseType::CloudflareD1 => cloudflare_d1_api_url(self),
DatabaseType::SqlServer => {
format!("server=tcp:{host},{port};database={}", self.database.as_deref().unwrap_or("master"))
}
@@ -1009,6 +1012,7 @@ impl ConnectionConfig {
DatabaseType::ClickHouse => clickhouse_http_url(self, raw_host, port),
DatabaseType::Rqlite => rqlite_http_url(self, raw_host, port),
DatabaseType::Turso => turso_http_url(self, raw_host, port),
DatabaseType::CloudflareD1 => cloudflare_d1_api_url(self),
DatabaseType::SqlServer => format!(
"server=tcp:{host},{port};user={};password={};database={}",
self.username,
@@ -1695,6 +1699,12 @@ fn turso_http_url(config: &ConnectionConfig, host: &str, port: u16) -> String {
format!("{scheme}://{}:{port}", bracket_ipv6(trimmed))
}
fn cloudflare_d1_api_url(config: &ConnectionConfig) -> String {
let account_id = config.host.trim();
let database_id = config.database.as_deref().unwrap_or_default().trim();
format!("https://api.cloudflare.com/client/v4/accounts/{account_id}/d1/database/{database_id}")
}
fn trim_http_host_port(value: &str, default_port: u16) -> String {
let authority = value.trim_end_matches('/').split('/').next().unwrap_or(value).split('?').next().unwrap_or(value);
if authority.starts_with('[') && !authority.contains("]:") {
@@ -1935,6 +1945,22 @@ mod tests {
assert_eq!(serde_json::from_str::<DatabaseType>("\"zookeeper\"").unwrap(), DatabaseType::ZooKeeper);
}
#[test]
fn cloudflare_d1_database_type_and_api_url_are_stable() {
assert_eq!(serde_json::to_string(&DatabaseType::CloudflareD1).unwrap(), "\"cloudflare-d1\"");
assert_eq!(serde_json::from_str::<DatabaseType>("\"cloudflare-d1\"").unwrap(), DatabaseType::CloudflareD1);
let mut config = mysql_config("", "secret-token", Some("database-id"));
config.db_type = DatabaseType::CloudflareD1;
config.host = "account-id".to_string();
config.port = 443;
assert_eq!(
config.connection_url(),
"https://api.cloudflare.com/client/v4/accounts/account-id/d1/database/database-id"
);
assert!(!config.connection_url().contains("secret-token"));
}
#[test]
fn connection_config_defaults_missing_agent_java_options_to_empty() {
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
+30 -4
View File
@@ -862,6 +862,7 @@ fn should_discard_pool_after_query_timeout(db_type: Option<DatabaseType>) -> boo
| DatabaseType::SqlServer
| DatabaseType::Rqlite
| DatabaseType::Turso
| DatabaseType::CloudflareD1
| DatabaseType::Elasticsearch
| DatabaseType::Qdrant
| DatabaseType::Milvus
@@ -1285,6 +1286,17 @@ pub async fn do_execute(
)
.await
}
PoolKind::CloudflareD1(client) => {
let client = client.clone();
let max_rows = options.max_rows;
drop(connections);
wait_for_query_opt(
cancel_token,
query_timeout,
db::cloudflare_d1_driver::execute_query_with_max_rows(&client, sql, max_rows),
)
.await
}
PoolKind::ClickHouse(client) => {
let client = client.clone();
let database = pool_key.split(':').nth(1).unwrap_or("default").to_string();
@@ -1821,13 +1833,16 @@ pub async fn execute_multi_core_with_options_for_client(
.map(|results| results.into_iter().map(Into::into).collect());
}
let is_turso = {
let is_http_sqlite = {
let configs = state.configs.read().await;
configs.get(connection_id).is_some_and(|c| c.db_type == DatabaseType::Turso)
configs
.get(connection_id)
.is_some_and(|c| matches!(c.db_type, DatabaseType::Turso | DatabaseType::CloudflareD1))
};
// Turso sends all statements in one HTTP pipeline for transactional integrity.
if is_turso {
// HTTP SQLite providers send all statements in one request so the provider
// can preserve batch ordering and atomicity.
if is_http_sqlite {
let result =
execute_sql_statement_with_options(state, connection_id, database, sql, schema, cancel_token, options)
.await?;
@@ -2313,6 +2328,7 @@ pub async fn execute_statements_in_transaction(
PoolKind::Postgres(pg) => TxPath::Pg(pg.clone()),
PoolKind::Mysql(mp, _mode) => TxPath::Mysql(mp.clone(), false),
PoolKind::Sqlite(sq) => TxPath::Sqlite(sq.clone()),
PoolKind::CloudflareD1(client) => TxPath::CloudflareD1(client.clone()),
PoolKind::ClickHouse(_)
| PoolKind::Rqlite(_)
| PoolKind::Turso(_)
@@ -2351,6 +2367,15 @@ pub async fn execute_statements_in_transaction(
exec_tx_mysql_inner(state, &pool_key, pool, statements, start, operation_budget.clone()).await
}
Some(TxPath::Sqlite(pool)) => exec_tx_sqlite_inner(pool, statements, start).await,
Some(TxPath::CloudflareD1(client)) => {
let sql = statements.join(";\n");
wait_for_query_opt(
None,
operation_budget.query_timeout,
db::cloudflare_d1_driver::execute_query_with_max_rows(&client, &sql, None),
)
.await
}
Some(TxPath::Explicit) => {
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
exec_tx_explicit_inner(state, &pool_key, mysql_dialect, Some(database), statements, schema, start).await
@@ -2376,6 +2401,7 @@ enum TxPath {
Pg(deadpool_postgres::Pool),
Mysql(mysql_async::Pool, bool),
Sqlite(db::sqlite::SqliteHandle),
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
Explicit,
None,
}
+14
View File
@@ -808,6 +808,7 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
drop(connections);
client.list_databases().await
}
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_databases(client).await,
_ => Ok(vec![]),
}
}
@@ -1908,6 +1909,9 @@ async fn list_tables_once(
.await
.map(|infos| collection_names_to_tables(infos.into_iter().map(|i| i.name).collect(), "COLLECTION"))
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types)),
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_tables(client, schema)
.await
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types)),
_ => Ok(vec![]),
}
}
@@ -4197,6 +4201,9 @@ pub async fn get_columns_core(
PoolKind::Rqlite(client) => {
db::rqlite_driver::get_columns(client, schema, table).await.map(deduplicate_column_infos)
}
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::get_columns(client, schema, table)
.await
.map(deduplicate_column_infos),
_ => Ok(vec![]),
}
})
@@ -4293,6 +4300,7 @@ pub async fn list_indexes_core(
PoolKind::Sqlite(p) => db::sqlite::list_indexes(p, schema, table).await,
PoolKind::Rqlite(client) => db::rqlite_driver::list_indexes(client, schema, table).await,
PoolKind::MongoDb(client) => db::mongo_driver::list_indexes(client, database, table).await,
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_indexes(client, schema, table).await,
_ => Ok(vec![]),
}
})
@@ -4339,6 +4347,7 @@ pub async fn list_foreign_keys_core(
PoolKind::Postgres(p) => db::postgres::list_foreign_keys(p, schema, table).await,
PoolKind::Sqlite(p) => db::sqlite::list_foreign_keys(p, schema, table).await,
PoolKind::Rqlite(client) => db::rqlite_driver::list_foreign_keys(client, schema, table).await,
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_foreign_keys(client, schema, table).await,
_ => Ok(vec![]),
}
})
@@ -4383,6 +4392,7 @@ pub async fn list_triggers_core(
PoolKind::Postgres(p) => db::postgres::list_triggers(p, schema, table).await,
PoolKind::Sqlite(p) => db::sqlite::list_triggers(p, schema, table).await,
PoolKind::Rqlite(client) => db::rqlite_driver::list_triggers(client, schema, table).await,
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_triggers(client, schema, table).await,
_ => Ok(vec![]),
}
})
@@ -4655,6 +4665,7 @@ pub async fn get_table_ddl_core(
PoolKind::Postgres(p) => pg_ddl(p, schema, table).await,
PoolKind::Sqlite(p) => sqlite_ddl(p, table).await,
PoolKind::Rqlite(client) => db::rqlite_driver::table_ddl(client, table).await,
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::table_ddl(client, table).await,
_ => Err("DDL not supported for this database type".to_string()),
}
}
@@ -5074,6 +5085,9 @@ async fn get_object_source_once(
.await?;
first_string_cell(result)?
}
PoolKind::CloudflareD1(client) => {
return db::cloudflare_d1_driver::object_source(client, name, &object_type).await;
}
_ => return Err("Object source is not supported for this database type".to_string()),
}
}
+29 -6
View File
@@ -744,6 +744,17 @@ pub fn build_import_insert_batch_from_rows(
if rows.is_empty() {
return Ok(None);
}
if *db_type == DatabaseType::CloudflareD1 {
return crate::db::cloudflare_d1::build_streaming_import_insert_batch(
rows,
columns,
mappings,
target_column_types,
table,
schema,
rows.len(),
);
}
let mapped = mapping_indexes_for_columns(columns, mappings)?;
let target_columns = mapped.iter().map(|(_, target)| target.clone()).collect::<Vec<_>>();
let column_types = target_columns
@@ -781,6 +792,17 @@ pub fn build_import_insert_batches(
db_type: &DatabaseType,
batch_size: usize,
) -> Result<Vec<ImportSqlBatch>, String> {
if *db_type == DatabaseType::CloudflareD1 {
return crate::db::cloudflare_d1::build_import_insert_batches(
&data.rows,
&data.columns,
mappings,
target_column_types,
table,
schema,
batch_size.clamp(1, 100),
);
}
let mapped = mapping_indexes(data, mappings)?;
let columns = mapped.iter().map(|(_, target)| target.clone()).collect::<Vec<_>>();
let column_types = columns
@@ -819,7 +841,7 @@ pub fn build_import_insert_batches(
pub fn truncate_sql(table: &str, schema: &str, db_type: &DatabaseType) -> String {
let full_table = qualified_table(table, schema, db_type);
match db_type {
DatabaseType::Sqlite => format!("DELETE FROM {full_table}"),
DatabaseType::Sqlite | DatabaseType::CloudflareD1 => format!("DELETE FROM {full_table}"),
_ => format!("TRUNCATE TABLE {full_table}"),
}
}
@@ -937,7 +959,7 @@ fn text_data_type(db_type: &DatabaseType) -> &'static str {
fn integer_data_type(db_type: &DatabaseType) -> &'static str {
match db_type {
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => "INTEGER",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1 => "INTEGER",
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => "NUMBER(19)",
DatabaseType::ClickHouse => "Int64",
_ => "BIGINT",
@@ -954,7 +976,7 @@ fn decimal_data_type(db_type: &DatabaseType) -> &'static str {
| DatabaseType::Highgo
| DatabaseType::Kwdb
| DatabaseType::Vastbase => "DOUBLE PRECISION",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => "REAL",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1 => "REAL",
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => "BINARY_DOUBLE",
DatabaseType::ClickHouse => "Float64",
_ => "DOUBLE",
@@ -970,7 +992,7 @@ fn boolean_data_type(db_type: &DatabaseType) -> &'static str {
| DatabaseType::Sundb
| DatabaseType::Databend => "TINYINT(1)",
DatabaseType::SqlServer => "BIT",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => "INTEGER",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1 => "INTEGER",
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => "NUMBER(1)",
DatabaseType::ClickHouse => "UInt8",
_ => "BOOLEAN",
@@ -979,7 +1001,7 @@ fn boolean_data_type(db_type: &DatabaseType) -> &'static str {
fn date_data_type(db_type: &DatabaseType) -> &'static str {
match db_type {
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => "TEXT",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1 => "TEXT",
DatabaseType::ClickHouse => "Date",
_ => "DATE",
}
@@ -994,7 +1016,7 @@ fn timestamp_data_type(db_type: &DatabaseType) -> &'static str {
| DatabaseType::Sundb
| DatabaseType::Databend => "DATETIME",
DatabaseType::SqlServer => "DATETIME2",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => "TEXT",
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1 => "TEXT",
DatabaseType::ClickHouse => "DateTime64",
_ => "TIMESTAMP",
}
@@ -1382,6 +1404,7 @@ where
};
let effective_batch_size = match db_type {
DatabaseType::Oracle | DatabaseType::OceanbaseOracle => 1,
DatabaseType::CloudflareD1 => batch_size.clamp(1, 100),
_ => batch_size.max(1),
};
let mut rows_imported = 0;
+16 -4
View File
@@ -1020,6 +1020,7 @@ pub fn escape_value_typed(val: &serde_json::Value, db_type: &DatabaseType, colum
serde_json::Value::Bool(b) => match db_type {
DatabaseType::Mysql
| DatabaseType::Sqlite
| DatabaseType::CloudflareD1
| DatabaseType::DuckDb
| DatabaseType::Doris
| DatabaseType::StarRocks => {
@@ -1881,7 +1882,10 @@ pub fn generate_upsert_typed(
}
match db_type {
db_type if is_postgres_transfer_dialect(db_type) => {
db_type
if is_postgres_transfer_dialect(db_type)
|| matches!(db_type, DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb) =>
{
let pk_list = pk_columns.iter().map(|c| quote_identifier(c, db_type)).collect::<Vec<_>>().join(", ");
let mut sql = format!("INSERT INTO {full_table} ({col_list}) VALUES\n{}", value_rows.join(",\n"));
if non_pk_columns.is_empty() {
@@ -2069,6 +2073,10 @@ fn generate_transfer_write_sql_batches(
}
let max_rows = max_transfer_write_rows(db_type, mode);
let max_sql_bytes = match db_type {
DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES,
_ => MAX_TRANSFER_WRITE_SQL_BYTES,
};
let mut statements = Vec::new();
let mut start = 0;
@@ -2096,7 +2104,7 @@ fn generate_transfer_write_sql_batches(
db_type,
pk_columns,
);
if candidate.len() > MAX_TRANSFER_WRITE_SQL_BYTES && !accepted.is_empty() {
if candidate.len() > max_sql_bytes && !accepted.is_empty() {
break;
}
accepted = candidate;
@@ -3896,7 +3904,9 @@ where
if request.mode == TransferMode::Overwrite {
let full_table = qualified_table(&target_table, &request.target_schema, target_db_type);
let truncate_sql = match target_db_type {
DatabaseType::Sqlite | DatabaseType::DuckDb => format!("DELETE FROM {full_table}"),
DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb => {
format!("DELETE FROM {full_table}")
}
_ => format!("TRUNCATE TABLE {full_table}"),
};
execute_on_pool(state, target_pool_key, &truncate_sql)
@@ -4174,7 +4184,9 @@ where
if request.mode == TransferMode::Overwrite {
let full_table = qualified_table(&target_table, &request.target_schema, target_db_type);
let truncate_sql = match target_db_type {
DatabaseType::Sqlite | DatabaseType::DuckDb => format!("DELETE FROM {full_table}"),
DatabaseType::Sqlite | DatabaseType::CloudflareD1 | DatabaseType::DuckDb => {
format!("DELETE FROM {full_table}")
}
_ => format!("TRUNCATE TABLE {full_table}"),
};
execute_on_pool(state, target_pool_key, &truncate_sql).await.map_err(|e| format!("Failed to truncate: {e}"))?;