feat(data-grid): add multi-format copy extractors
This commit is contained in:
1 parent
166027600f
commit
0b7a886cfd
60 files changed
+4615
-1766
No files matched your search
@@ -36,11 +36,14 @@ default = ["duckdb-bundled", "mq-admin", "sqlite-sqlcipher", "system-fonts"]
|
||||
duckdb-bundled = ["duckdb/bundled", "duckdb/parquet"]
|
||||
mq-admin = []
|
||||
sqlite-sqlcipher = ["rusqlite/bundled-sqlcipher-vendored-openssl"]
|
||||
openapi = ["dep:utoipa"]
|
||||
system-fonts = ["font-kit"]
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] }
|
||||
utoipa = { version = "5.5", optional = true }
|
||||
unicode-width = "0.2"
|
||||
serde_yaml_ng = "0.10"
|
||||
json5 = "0.4"
|
||||
regex = "1"
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
use crate::data_grid_sql::{DataGridCopyInsertMode, DataGridTableMeta};
|
||||
use crate::models::connection::DatabaseType;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::fmt;
|
||||
|
||||
pub const DATA_GRID_EXTRACTOR_CONTRACT_VERSION: u16 = 1;
|
||||
|
||||
const DEFAULT_DSV_COLUMN_SEPARATOR: &str = ",";
|
||||
const DEFAULT_DSV_ROW_SEPARATOR: &str = "\n";
|
||||
const DEFAULT_DSV_NULL_TEXT: &str = "NULL";
|
||||
const DEFAULT_DSV_QUOTE: char = '"';
|
||||
const DEFAULT_SQL_SKIP_COMPUTED_COLUMNS: bool = true;
|
||||
const DEFAULT_SQL_SKIP_GENERATED_COLUMNS: bool = true;
|
||||
const DEFAULT_JSON_PRETTY: bool = true;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum DataGridExtractorId {
|
||||
Tsv,
|
||||
TsvWithHeaders,
|
||||
Csv,
|
||||
CsvWithHeaders,
|
||||
PipeSeparated,
|
||||
Dsv,
|
||||
Json,
|
||||
JsonLines,
|
||||
OneRow,
|
||||
SqlInList,
|
||||
SqlInserts,
|
||||
SqlUpdates,
|
||||
WhereClause,
|
||||
Markdown,
|
||||
Html,
|
||||
Xml,
|
||||
Pretty,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum DataGridSelectionKind {
|
||||
#[default]
|
||||
Cells,
|
||||
Rows,
|
||||
Columns,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum DataGridQuotePolicy {
|
||||
Always,
|
||||
#[default]
|
||||
Minimal,
|
||||
Never,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
|
||||
pub struct DataGridDsvOptions {
|
||||
pub column_separator: String,
|
||||
pub row_separator: String,
|
||||
pub null_text: String,
|
||||
pub quote: char,
|
||||
pub quote_policy: DataGridQuotePolicy,
|
||||
pub include_column_header: bool,
|
||||
pub include_row_header: bool,
|
||||
}
|
||||
|
||||
impl Default for DataGridDsvOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
column_separator: DEFAULT_DSV_COLUMN_SEPARATOR.to_owned(),
|
||||
row_separator: DEFAULT_DSV_ROW_SEPARATOR.to_owned(),
|
||||
null_text: DEFAULT_DSV_NULL_TEXT.to_owned(),
|
||||
quote: DEFAULT_DSV_QUOTE,
|
||||
quote_policy: DataGridQuotePolicy::Minimal,
|
||||
include_column_header: false,
|
||||
include_row_header: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
|
||||
pub struct DataGridSqlExtractorOptions {
|
||||
pub skip_computed_columns: bool,
|
||||
pub skip_generated_columns: bool,
|
||||
pub insert_mode: DataGridCopyInsertMode,
|
||||
pub exclude_primary_keys_from_insert: bool,
|
||||
}
|
||||
|
||||
impl Default for DataGridSqlExtractorOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
skip_computed_columns: DEFAULT_SQL_SKIP_COMPUTED_COLUMNS,
|
||||
skip_generated_columns: DEFAULT_SQL_SKIP_GENERATED_COLUMNS,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
exclude_primary_keys_from_insert: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
|
||||
pub struct DataGridJsonExtractorOptions {
|
||||
pub pretty: bool,
|
||||
}
|
||||
|
||||
impl Default for DataGridJsonExtractorOptions {
|
||||
fn default() -> Self {
|
||||
Self { pretty: DEFAULT_JSON_PRETTY }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase", default, deny_unknown_fields)]
|
||||
pub struct DataGridExtractorOptions {
|
||||
pub dsv: DataGridDsvOptions,
|
||||
pub sql: DataGridSqlExtractorOptions,
|
||||
pub json: DataGridJsonExtractorOptions,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DataGridExtractColumn {
|
||||
pub display_name: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source_name: Option<String>,
|
||||
pub source_index: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DataGridExtractRequest {
|
||||
pub version: u16,
|
||||
pub extractor: DataGridExtractorId,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub database_type: Option<DatabaseType>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub table_meta: Option<DataGridTableMeta>,
|
||||
#[serde(default)]
|
||||
pub columns: Vec<DataGridExtractColumn>,
|
||||
#[serde(default)]
|
||||
pub selected_column_indexes: Vec<usize>,
|
||||
#[serde(default)]
|
||||
pub rows: Vec<Vec<Value>>,
|
||||
#[serde(default)]
|
||||
pub selection_kind: DataGridSelectionKind,
|
||||
#[serde(default)]
|
||||
pub options: DataGridExtractorOptions,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum DataGridExtractWarningCode {
|
||||
OmittedColumns,
|
||||
DuplicateJsonColumnNames,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DataGridExtractWarning {
|
||||
pub code: DataGridExtractWarningCode,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DataGridExtractResult {
|
||||
pub text: String,
|
||||
pub mime_type: String,
|
||||
pub file_extension: String,
|
||||
pub row_count: usize,
|
||||
pub column_count: usize,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub omitted_columns: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub warnings: Vec<DataGridExtractWarning>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum DataGridExtractErrorCode {
|
||||
UnsupportedVersion,
|
||||
EmptySelection,
|
||||
InvalidColumnIndex,
|
||||
InvalidColumnMapping,
|
||||
MissingTableMetadata,
|
||||
MissingPrimaryKey,
|
||||
NullPrimaryKey,
|
||||
NoWritableColumns,
|
||||
UnsupportedDatabase,
|
||||
InvalidDsvConfiguration,
|
||||
InputTooLarge,
|
||||
OutputTooLarge,
|
||||
EncodingFailed,
|
||||
ExecutionFailed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DataGridExtractError {
|
||||
pub code: DataGridExtractErrorCode,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl DataGridExtractError {
|
||||
pub fn new(code: DataGridExtractErrorCode, message: impl Into<String>) -> Self {
|
||||
Self { code, message: message.into() }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for DataGridExtractError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for DataGridExtractError {}
|
||||
@@ -0,0 +1,232 @@
|
||||
use super::{
|
||||
value_text, write_bytes, DataGridDsvOptions, DataGridExtractError, DataGridExtractErrorCode, DataGridQuotePolicy,
|
||||
ExtractContext,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
use std::io::Write;
|
||||
|
||||
const MAX_SEPARATOR_CHARACTERS: usize = 8;
|
||||
const MAX_NULL_TEXT_CHARACTERS: usize = 64;
|
||||
|
||||
/// A single DSV data cell. NULL is carried as structured state (not pre-flattened
|
||||
/// into the `null_text` string) so the sentinel can be encoded distinctly from a
|
||||
/// string value that happens to equal it.
|
||||
enum DsvCell<'a> {
|
||||
Null,
|
||||
Text(Cow<'a, str>),
|
||||
}
|
||||
|
||||
pub(super) fn write_dsv(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
options: &DataGridDsvOptions,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
validate_dsv_options(options, true)?;
|
||||
let mut first_row = true;
|
||||
if options.include_column_header {
|
||||
write_dsv_row(
|
||||
output,
|
||||
context.selected_columns.iter().map(|column| Cow::Borrowed(column.display_name.as_str())),
|
||||
options,
|
||||
options.include_row_header.then(|| "#".to_string()),
|
||||
)?;
|
||||
first_row = false;
|
||||
}
|
||||
for (row_index, row) in context.request.rows.iter().enumerate() {
|
||||
if !first_row {
|
||||
write_bytes(output, options.row_separator.as_bytes())?;
|
||||
}
|
||||
write_dsv_data_row(
|
||||
output,
|
||||
context.selected_source_indexes.iter().map(|source_index| {
|
||||
let value = &row[*source_index];
|
||||
if value.is_null() {
|
||||
DsvCell::Null
|
||||
} else {
|
||||
DsvCell::Text(value_text(value))
|
||||
}
|
||||
}),
|
||||
options,
|
||||
options.include_row_header.then(|| (row_index + 1).to_string()),
|
||||
)?;
|
||||
first_row = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes a row of header strings (column names / row-number header). Headers
|
||||
/// are never NULL, so they use the plain string field writer.
|
||||
fn write_dsv_row<'a>(
|
||||
output: &mut dyn Write,
|
||||
values: impl Iterator<Item = Cow<'a, str>>,
|
||||
options: &DataGridDsvOptions,
|
||||
row_header: Option<String>,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
let mut first = true;
|
||||
if let Some(header) = row_header {
|
||||
write_dsv_field(output, &header, options)?;
|
||||
first = false;
|
||||
}
|
||||
for value in values {
|
||||
if !first {
|
||||
write_bytes(output, options.column_separator.as_bytes())?;
|
||||
}
|
||||
write_dsv_field(output, &value, options)?;
|
||||
first = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes a row of data cells, preserving NULL as a distinct sentinel.
|
||||
fn write_dsv_data_row<'a>(
|
||||
output: &mut dyn Write,
|
||||
cells: impl Iterator<Item = DsvCell<'a>>,
|
||||
options: &DataGridDsvOptions,
|
||||
row_header: Option<String>,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
let mut first = true;
|
||||
if let Some(header) = row_header {
|
||||
write_dsv_field(output, &header, options)?;
|
||||
first = false;
|
||||
}
|
||||
for cell in cells {
|
||||
if !first {
|
||||
write_bytes(output, options.column_separator.as_bytes())?;
|
||||
}
|
||||
write_dsv_data_field(output, &cell, options)?;
|
||||
first = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dsv_needs_quote(value: &str, options: &DataGridDsvOptions) -> bool {
|
||||
value.contains(&options.column_separator)
|
||||
|| value.contains(&options.row_separator)
|
||||
|| value.contains(options.quote)
|
||||
|| value.contains('\r')
|
||||
|| value.contains('\n')
|
||||
}
|
||||
|
||||
fn write_quoted_dsv_field(
|
||||
output: &mut dyn Write,
|
||||
value: &str,
|
||||
options: &DataGridDsvOptions,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
let quote = options.quote;
|
||||
let mut quote_buffer = [0; 4];
|
||||
let quote_text = quote.encode_utf8(&mut quote_buffer);
|
||||
write_bytes(output, quote_text.as_bytes())?;
|
||||
let mut segments = value.split(quote);
|
||||
if let Some(first) = segments.next() {
|
||||
write_bytes(output, first.as_bytes())?;
|
||||
}
|
||||
for segment in segments {
|
||||
write_bytes(output, quote_text.as_bytes())?;
|
||||
write_bytes(output, quote_text.as_bytes())?;
|
||||
write_bytes(output, segment.as_bytes())?;
|
||||
}
|
||||
write_bytes(output, quote_text.as_bytes())
|
||||
}
|
||||
|
||||
/// Encodes a header (non-data) string field. Quotes per policy; a header equal
|
||||
/// to the NULL sentinel is NOT special-cased because headers are not data.
|
||||
fn write_dsv_field(
|
||||
output: &mut dyn Write,
|
||||
value: &str,
|
||||
options: &DataGridDsvOptions,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
let should_quote = match options.quote_policy {
|
||||
DataGridQuotePolicy::Always => true,
|
||||
DataGridQuotePolicy::Never => false,
|
||||
DataGridQuotePolicy::Minimal => dsv_needs_quote(value, options),
|
||||
};
|
||||
if !should_quote {
|
||||
return write_bytes(output, value.as_bytes());
|
||||
}
|
||||
write_quoted_dsv_field(output, value, options)
|
||||
}
|
||||
|
||||
/// Encodes a data cell. NULL is always written as the bare sentinel (never
|
||||
/// quoted/escaped); a string equal to the sentinel is force-quoted in Minimal
|
||||
/// mode so it stays distinguishable from a real NULL. Under Always the string is
|
||||
/// quoted and the bare NULL still differs; under Never the two are inherently
|
||||
/// indistinguishable (documented limitation).
|
||||
fn write_dsv_data_field(
|
||||
output: &mut dyn Write,
|
||||
cell: &DsvCell<'_>,
|
||||
options: &DataGridDsvOptions,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
match cell {
|
||||
DsvCell::Null => write_bytes(output, options.null_text.as_bytes()),
|
||||
DsvCell::Text(value) => {
|
||||
let should_quote = match options.quote_policy {
|
||||
DataGridQuotePolicy::Always => true,
|
||||
DataGridQuotePolicy::Never => false,
|
||||
DataGridQuotePolicy::Minimal => {
|
||||
dsv_needs_quote(value, options) || value.as_ref() == options.null_text.as_str()
|
||||
}
|
||||
};
|
||||
if !should_quote {
|
||||
return write_bytes(output, value.as_bytes());
|
||||
}
|
||||
write_quoted_dsv_field(output, value, options)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn write_one_row(context: &ExtractContext<'_>, output: &mut dyn Write) -> Result<(), DataGridExtractError> {
|
||||
let options = DataGridDsvOptions { column_separator: ",".to_string(), ..context.request.options.dsv.clone() };
|
||||
validate_dsv_options(&options, false)?;
|
||||
let values = context
|
||||
.request
|
||||
.rows
|
||||
.iter()
|
||||
.flat_map(|row| context.selected_source_indexes.iter().map(move |source_index| &row[*source_index]));
|
||||
let mut first = true;
|
||||
for value in values {
|
||||
if !first {
|
||||
write_bytes(output, options.column_separator.as_bytes())?;
|
||||
}
|
||||
let cell = if value.is_null() { DsvCell::Null } else { DsvCell::Text(value_text(value)) };
|
||||
write_dsv_data_field(output, &cell, &options)?;
|
||||
first = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_dsv_options(options: &DataGridDsvOptions, uses_row_separator: bool) -> Result<(), DataGridExtractError> {
|
||||
if options.column_separator.is_empty() || (uses_row_separator && options.row_separator.is_empty()) {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration,
|
||||
"DSV column and row separators must not be empty.",
|
||||
));
|
||||
}
|
||||
if options.column_separator.chars().count() > MAX_SEPARATOR_CHARACTERS
|
||||
|| options.row_separator.chars().count() > MAX_SEPARATOR_CHARACTERS
|
||||
|| options.null_text.chars().count() > MAX_NULL_TEXT_CHARACTERS
|
||||
{
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration,
|
||||
"DSV separators may contain at most 8 characters and NULL text at most 64 characters.",
|
||||
));
|
||||
}
|
||||
if uses_row_separator
|
||||
&& (options.column_separator.contains(&options.row_separator)
|
||||
|| options.row_separator.contains(&options.column_separator))
|
||||
{
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration,
|
||||
"DSV column and row separators must not overlap.",
|
||||
));
|
||||
}
|
||||
if options.quote.is_control()
|
||||
|| options.column_separator.contains(options.quote)
|
||||
|| (uses_row_separator && options.row_separator.contains(options.quote))
|
||||
{
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration,
|
||||
"DSV quote must be a non-control character that is not part of a separator.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
use super::{value_text, write_bytes, DataGridExtractError, ExtractContext};
|
||||
use std::io::Write;
|
||||
use unicode_width::UnicodeWidthStr;
|
||||
|
||||
pub(super) fn write_markdown(context: &ExtractContext<'_>, output: &mut dyn Write) -> Result<(), DataGridExtractError> {
|
||||
let headers = context.selected_columns.iter().map(|column| markdown_text(&column.display_name));
|
||||
write_bytes(output, format!("| {} |\n", headers.collect::<Vec<_>>().join(" | ")).as_bytes())?;
|
||||
write_bytes(
|
||||
output,
|
||||
format!("| {} |", context.selected_columns.iter().map(|_| "---").collect::<Vec<_>>().join(" | ")).as_bytes(),
|
||||
)?;
|
||||
for row in &context.request.rows {
|
||||
write_bytes(output, b"\n")?;
|
||||
let values =
|
||||
context.selected_source_indexes.iter().map(|source_index| markdown_text(&value_text(&row[*source_index])));
|
||||
write_bytes(output, format!("| {} |", values.collect::<Vec<_>>().join(" | ")).as_bytes())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn write_html(context: &ExtractContext<'_>, output: &mut dyn Write) -> Result<(), DataGridExtractError> {
|
||||
write_bytes(output, b"<table>\n <thead><tr>")?;
|
||||
for column in &context.selected_columns {
|
||||
write_bytes(output, format!("<th>{}</th>", html_text(&column.display_name)).as_bytes())?;
|
||||
}
|
||||
write_bytes(output, b"</tr></thead>\n <tbody>")?;
|
||||
for row in &context.request.rows {
|
||||
write_bytes(output, b"\n <tr>")?;
|
||||
for source_index in &context.selected_source_indexes {
|
||||
let value = &row[*source_index];
|
||||
write_bytes(output, format!("<td>{}</td>", html_text(&value_text(value))).as_bytes())?;
|
||||
}
|
||||
write_bytes(output, b"</tr>")?;
|
||||
}
|
||||
write_bytes(output, b"\n </tbody>\n</table>")
|
||||
}
|
||||
|
||||
pub(super) fn write_xml(context: &ExtractContext<'_>, output: &mut dyn Write) -> Result<(), DataGridExtractError> {
|
||||
write_bytes(output, b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rows>")?;
|
||||
for row in &context.request.rows {
|
||||
write_bytes(output, b"\n <row>")?;
|
||||
for (column, source_index) in context.selected_columns.iter().zip(&context.selected_source_indexes) {
|
||||
let value = &row[*source_index];
|
||||
let name = html_text(&column.display_name);
|
||||
if value.is_null() {
|
||||
write_bytes(output, format!("\n <column name=\"{name}\" null=\"true\" />").as_bytes())?;
|
||||
} else {
|
||||
write_bytes(
|
||||
output,
|
||||
format!("\n <column name=\"{name}\">{}</column>", html_text(&value_text(value))).as_bytes(),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
write_bytes(output, b"\n </row>")?;
|
||||
}
|
||||
write_bytes(output, b"\n</rows>")
|
||||
}
|
||||
|
||||
pub(super) fn write_pretty(context: &ExtractContext<'_>, output: &mut dyn Write) -> Result<(), DataGridExtractError> {
|
||||
let widths = context
|
||||
.selected_columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, column)| {
|
||||
context
|
||||
.request
|
||||
.rows
|
||||
.iter()
|
||||
.map(|row| value_text(&row[context.selected_source_indexes[index]]).replace(['\r', '\n'], " ").width())
|
||||
.fold(column.display_name.width(), usize::max)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
write_pretty_border(output, &widths)?;
|
||||
write_pretty_row(output, context.selected_columns.iter().map(|column| column.display_name.as_str()), &widths)?;
|
||||
write_pretty_border(output, &widths)?;
|
||||
for row in &context.request.rows {
|
||||
let values = context
|
||||
.selected_source_indexes
|
||||
.iter()
|
||||
.map(|source_index| value_text(&row[*source_index]).replace(['\r', '\n'], " "));
|
||||
write_pretty_owned_row(output, values, &widths)?;
|
||||
}
|
||||
write_pretty_border(output, &widths)
|
||||
}
|
||||
|
||||
fn write_pretty_owned_row(
|
||||
output: &mut dyn Write,
|
||||
values: impl Iterator<Item = String>,
|
||||
widths: &[usize],
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
write_bytes(output, b"|")?;
|
||||
for (index, value) in values.enumerate() {
|
||||
let padding = widths.get(index).copied().unwrap_or_default().saturating_sub(value.width());
|
||||
write_bytes(output, b" ")?;
|
||||
write_bytes(output, value.as_bytes())?;
|
||||
write_repeated_byte(output, b' ', padding)?;
|
||||
write_bytes(output, b" |")?;
|
||||
}
|
||||
write_bytes(output, b"\n")
|
||||
}
|
||||
|
||||
fn write_pretty_border(output: &mut dyn Write, widths: &[usize]) -> Result<(), DataGridExtractError> {
|
||||
write_bytes(output, b"+")?;
|
||||
for width in widths {
|
||||
write_repeated_byte(output, b'-', width.saturating_add(2))?;
|
||||
write_bytes(output, b"+")?;
|
||||
}
|
||||
write_bytes(output, b"\n")
|
||||
}
|
||||
|
||||
fn write_pretty_row<'a>(
|
||||
output: &mut dyn Write,
|
||||
values: impl Iterator<Item = &'a str>,
|
||||
widths: &[usize],
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
write_bytes(output, b"|")?;
|
||||
for (index, value) in values.enumerate() {
|
||||
let padding = widths.get(index).copied().unwrap_or_default().saturating_sub(value.width());
|
||||
write_bytes(output, b" ")?;
|
||||
write_bytes(output, value.as_bytes())?;
|
||||
write_repeated_byte(output, b' ', padding)?;
|
||||
write_bytes(output, b" |")?;
|
||||
}
|
||||
write_bytes(output, b"\n")
|
||||
}
|
||||
|
||||
fn write_repeated_byte(output: &mut dyn Write, byte: u8, mut count: usize) -> Result<(), DataGridExtractError> {
|
||||
let mut buffer = [0u8; 1024];
|
||||
buffer.fill(byte);
|
||||
while count > 0 {
|
||||
let chunk_size = count.min(buffer.len());
|
||||
write_bytes(output, &buffer[..chunk_size])?;
|
||||
count -= chunk_size;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn markdown_text(value: &str) -> String {
|
||||
value.replace('\\', "\\\\").replace('|', "\\|").replace(['\r', '\n'], "<br>")
|
||||
}
|
||||
|
||||
fn html_text(value: &str) -> String {
|
||||
value.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'")
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
use super::{
|
||||
write_bytes, DataGridExtractColumn, DataGridExtractError, DataGridExtractErrorCode, DataGridExtractWarning,
|
||||
DataGridExtractWarningCode, ExtractContext,
|
||||
};
|
||||
use serde::ser::{SerializeMap, SerializeSeq};
|
||||
use serde::{Serialize, Serializer};
|
||||
use std::io::Write;
|
||||
|
||||
pub(super) fn write_json(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<Vec<DataGridExtractWarning>, DataGridExtractError> {
|
||||
let (names, warnings) = unique_json_names(&context.selected_columns);
|
||||
let rows =
|
||||
JsonRows { names: &names, source_indexes: &context.selected_source_indexes, rows: &context.request.rows };
|
||||
let result = if context.request.options.json.pretty {
|
||||
serde_json::to_writer_pretty(output, &rows)
|
||||
} else {
|
||||
serde_json::to_writer(output, &rows)
|
||||
};
|
||||
result.map_err(|error| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::EncodingFailed,
|
||||
format!("Failed to encode JSON extractor output: {error}"),
|
||||
)
|
||||
})?;
|
||||
Ok(warnings)
|
||||
}
|
||||
|
||||
pub(super) fn write_json_lines(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<Vec<DataGridExtractWarning>, DataGridExtractError> {
|
||||
let (names, warnings) = unique_json_names(&context.selected_columns);
|
||||
for (index, row) in context.request.rows.iter().enumerate() {
|
||||
if index > 0 {
|
||||
write_bytes(output, b"\n")?;
|
||||
}
|
||||
serde_json::to_writer(
|
||||
&mut *output,
|
||||
&JsonRow { names: &names, source_indexes: &context.selected_source_indexes, values: row },
|
||||
)
|
||||
.map_err(|error| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::EncodingFailed,
|
||||
format!("Failed to encode JSON Lines extractor output: {error}"),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
Ok(warnings)
|
||||
}
|
||||
|
||||
struct JsonRows<'a, 'value> {
|
||||
names: &'a [String],
|
||||
source_indexes: &'a [usize],
|
||||
rows: &'value [Vec<serde_json::Value>],
|
||||
}
|
||||
|
||||
impl Serialize for JsonRows<'_, '_> {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let mut sequence = serializer.serialize_seq(Some(self.rows.len()))?;
|
||||
for row in self.rows {
|
||||
sequence.serialize_element(&JsonRow {
|
||||
names: self.names,
|
||||
source_indexes: self.source_indexes,
|
||||
values: row,
|
||||
})?;
|
||||
}
|
||||
sequence.end()
|
||||
}
|
||||
}
|
||||
|
||||
struct JsonRow<'a, 'value> {
|
||||
names: &'a [String],
|
||||
source_indexes: &'a [usize],
|
||||
values: &'value [serde_json::Value],
|
||||
}
|
||||
|
||||
impl Serialize for JsonRow<'_, '_> {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let mut map = serializer.serialize_map(Some(self.names.len()))?;
|
||||
for (name, source_index) in self.names.iter().zip(self.source_indexes) {
|
||||
map.serialize_entry(name, &self.values[*source_index])?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
}
|
||||
|
||||
fn unique_json_names(columns: &[&DataGridExtractColumn]) -> (Vec<String>, Vec<DataGridExtractWarning>) {
|
||||
let reserved = columns.iter().map(|column| column.display_name.as_str()).collect::<std::collections::HashSet<_>>();
|
||||
let mut emitted = std::collections::HashSet::<String>::new();
|
||||
let mut suffixes = std::collections::HashMap::<String, usize>::new();
|
||||
let mut duplicate = false;
|
||||
let names = columns
|
||||
.iter()
|
||||
.map(|column| {
|
||||
let base = column.display_name.clone();
|
||||
if emitted.insert(base.clone()) {
|
||||
return base;
|
||||
}
|
||||
duplicate = true;
|
||||
let suffix = suffixes.entry(base.clone()).or_insert(2);
|
||||
loop {
|
||||
let candidate = format!("{base}_{suffix}");
|
||||
*suffix += 1;
|
||||
if !reserved.contains(candidate.as_str()) && emitted.insert(candidate.clone()) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let warnings = if duplicate {
|
||||
vec![DataGridExtractWarning {
|
||||
code: DataGridExtractWarningCode::DuplicateJsonColumnNames,
|
||||
message: "Duplicate JSON column names were suffixed to prevent data loss.".to_string(),
|
||||
}]
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
(names, warnings)
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
use serde_json::Value;
|
||||
use std::borrow::Cow;
|
||||
use std::io::{self, Write};
|
||||
|
||||
mod contract;
|
||||
mod delimited;
|
||||
mod document;
|
||||
mod json;
|
||||
mod sql;
|
||||
|
||||
pub use contract::*;
|
||||
use delimited::{write_dsv, write_one_row};
|
||||
use document::{write_html, write_markdown, write_pretty, write_xml};
|
||||
use json::{write_json, write_json_lines};
|
||||
use sql::{write_sql_in_list, write_sql_inserts, write_sql_updates, write_where_clause};
|
||||
|
||||
const DATA_GRID_EXTRACTOR_MAX_OUTPUT_BYTES: usize = 32 * 1024 * 1024;
|
||||
const DATA_GRID_EXTRACTOR_MAX_INPUT_BYTES: usize = 64 * 1024 * 1024;
|
||||
const DATA_GRID_EXTRACTOR_MAX_ROWS: usize = 100_000;
|
||||
const DATA_GRID_EXTRACTOR_MAX_COLUMNS: usize = 10_000;
|
||||
const DATA_GRID_EXTRACTOR_MAX_CELLS: usize = 2_000_000;
|
||||
|
||||
struct ExtractContext<'a> {
|
||||
request: &'a DataGridExtractRequest,
|
||||
selected_columns: Vec<&'a DataGridExtractColumn>,
|
||||
selected_source_indexes: Vec<usize>,
|
||||
selected_column_info: Vec<Option<&'a crate::data_grid_sql::DataGridColumnInfo>>,
|
||||
}
|
||||
|
||||
pub fn extract_data_grid_selection(
|
||||
request: DataGridExtractRequest,
|
||||
) -> Result<DataGridExtractResult, DataGridExtractError> {
|
||||
if request.version != DATA_GRID_EXTRACTOR_CONTRACT_VERSION {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::UnsupportedVersion,
|
||||
format!("Unsupported data grid extractor contract version: {}", request.version),
|
||||
));
|
||||
}
|
||||
let context = build_context(&request)?;
|
||||
let mut output = BoundedOutput::new(DATA_GRID_EXTRACTOR_MAX_OUTPUT_BYTES);
|
||||
let metadata = write_extraction(&context, &mut output);
|
||||
if output.exceeded_limit() {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::OutputTooLarge,
|
||||
"Extracted clipboard output exceeds 32 MiB; export the data to a file instead.",
|
||||
));
|
||||
}
|
||||
let metadata = metadata?;
|
||||
let text = String::from_utf8(output.into_bytes()).map_err(|error| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::EncodingFailed,
|
||||
format!("Extractor produced invalid UTF-8: {error}"),
|
||||
)
|
||||
})?;
|
||||
Ok(DataGridExtractResult {
|
||||
text,
|
||||
mime_type: metadata.mime_type.to_owned(),
|
||||
file_extension: metadata.file_extension.to_owned(),
|
||||
row_count: request.rows.len(),
|
||||
column_count: context.selected_columns.len(),
|
||||
omitted_columns: metadata.omitted_columns,
|
||||
warnings: metadata.warnings,
|
||||
})
|
||||
}
|
||||
|
||||
struct BoundedOutput {
|
||||
bytes: Vec<u8>,
|
||||
max_bytes: usize,
|
||||
exceeded_limit: bool,
|
||||
}
|
||||
|
||||
impl BoundedOutput {
|
||||
fn new(max_bytes: usize) -> Self {
|
||||
Self { bytes: Vec::new(), max_bytes, exceeded_limit: false }
|
||||
}
|
||||
|
||||
fn exceeded_limit(&self) -> bool {
|
||||
self.exceeded_limit
|
||||
}
|
||||
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for BoundedOutput {
|
||||
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
|
||||
if buffer.len() > self.max_bytes.saturating_sub(self.bytes.len()) {
|
||||
self.exceeded_limit = true;
|
||||
return Err(io::Error::other("data grid extractor output limit exceeded"));
|
||||
}
|
||||
self.bytes.extend_from_slice(buffer);
|
||||
Ok(buffer.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn build_context(request: &DataGridExtractRequest) -> Result<ExtractContext<'_>, DataGridExtractError> {
|
||||
validate_request_budget(request)?;
|
||||
if request.rows.is_empty() || request.selected_column_indexes.is_empty() {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::EmptySelection,
|
||||
"Select at least one data grid cell before extracting data.",
|
||||
));
|
||||
}
|
||||
let mut selected_columns = Vec::with_capacity(request.selected_column_indexes.len());
|
||||
let mut unique_selected_indexes = std::collections::HashSet::with_capacity(request.selected_column_indexes.len());
|
||||
for index in &request.selected_column_indexes {
|
||||
if !unique_selected_indexes.insert(*index) {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidColumnIndex,
|
||||
format!("Selected column index {index} is duplicated."),
|
||||
));
|
||||
}
|
||||
let column = request.columns.get(*index).ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidColumnIndex,
|
||||
format!("Selected column index {index} is outside the extractor column list."),
|
||||
)
|
||||
})?;
|
||||
selected_columns.push(column);
|
||||
}
|
||||
let selected_source_indexes = selected_columns.iter().map(|column| column.source_index).collect::<Vec<_>>();
|
||||
for row in &request.rows {
|
||||
for column in &selected_columns {
|
||||
row.get(column.source_index).ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidColumnMapping,
|
||||
format!("Column '{}' maps to missing row index {}.", column.display_name, column.source_index),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let column_info_by_name = request
|
||||
.table_meta
|
||||
.as_ref()
|
||||
.and_then(|table_meta| table_meta.columns.as_ref())
|
||||
.map(|columns| {
|
||||
columns
|
||||
.iter()
|
||||
.map(|info| (normalized_name(info.name.as_str()).to_ascii_uppercase(), info))
|
||||
.collect::<std::collections::HashMap<_, _>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let selected_column_info = selected_columns
|
||||
.iter()
|
||||
.map(|column| {
|
||||
let source_name = column.source_name.as_deref().unwrap_or(&column.display_name);
|
||||
column_info_by_name.get(&normalized_name(source_name).to_ascii_uppercase()).copied()
|
||||
})
|
||||
.collect();
|
||||
Ok(ExtractContext { request, selected_columns, selected_source_indexes, selected_column_info })
|
||||
}
|
||||
|
||||
fn validate_request_budget(request: &DataGridExtractRequest) -> Result<(), DataGridExtractError> {
|
||||
if request.rows.len() > DATA_GRID_EXTRACTOR_MAX_ROWS
|
||||
|| request.columns.len() > DATA_GRID_EXTRACTOR_MAX_COLUMNS
|
||||
|| request.selected_column_indexes.len() > DATA_GRID_EXTRACTOR_MAX_COLUMNS
|
||||
{
|
||||
return Err(input_too_large_error());
|
||||
}
|
||||
|
||||
let mut cell_count = 0usize;
|
||||
let mut estimated_bytes = request
|
||||
.columns
|
||||
.iter()
|
||||
.fold(0usize, |size, column| {
|
||||
size.saturating_add(column.display_name.len())
|
||||
.saturating_add(column.source_name.as_deref().map_or(0, str::len))
|
||||
})
|
||||
.saturating_add(request.options.dsv.column_separator.len())
|
||||
.saturating_add(request.options.dsv.row_separator.len())
|
||||
.saturating_add(request.options.dsv.null_text.len())
|
||||
.saturating_add(estimated_table_meta_bytes(request.table_meta.as_ref()));
|
||||
if estimated_bytes > DATA_GRID_EXTRACTOR_MAX_INPUT_BYTES {
|
||||
return Err(input_too_large_error());
|
||||
}
|
||||
for row in &request.rows {
|
||||
cell_count = cell_count.saturating_add(row.len());
|
||||
if cell_count > DATA_GRID_EXTRACTOR_MAX_CELLS {
|
||||
return Err(input_too_large_error());
|
||||
}
|
||||
for value in row {
|
||||
estimated_bytes = estimated_bytes.saturating_add(estimated_value_bytes(value));
|
||||
if estimated_bytes > DATA_GRID_EXTRACTOR_MAX_INPUT_BYTES {
|
||||
return Err(input_too_large_error());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn estimated_table_meta_bytes(table_meta: Option<&crate::data_grid_sql::DataGridTableMeta>) -> usize {
|
||||
let Some(table_meta) = table_meta else {
|
||||
return 0;
|
||||
};
|
||||
let mut size = table_meta
|
||||
.table_name
|
||||
.len()
|
||||
.saturating_add(table_meta.catalog.as_deref().map_or(0, str::len))
|
||||
.saturating_add(table_meta.database.as_deref().map_or(0, str::len))
|
||||
.saturating_add(table_meta.schema.as_deref().map_or(0, str::len));
|
||||
for primary_key in &table_meta.primary_keys {
|
||||
size = size.saturating_add(primary_key.len());
|
||||
}
|
||||
for column in table_meta.columns.as_deref().unwrap_or_default() {
|
||||
size = size
|
||||
.saturating_add(column.name.len())
|
||||
.saturating_add(column.data_type.len())
|
||||
.saturating_add(column.column_default.as_deref().map_or(0, str::len))
|
||||
.saturating_add(column.extra.as_deref().map_or(0, str::len));
|
||||
}
|
||||
size
|
||||
}
|
||||
|
||||
fn estimated_value_bytes(value: &Value) -> usize {
|
||||
match value {
|
||||
Value::Null => 4,
|
||||
Value::Bool(_) => 5,
|
||||
Value::Number(_) => 32,
|
||||
Value::String(value) => value.len(),
|
||||
Value::Array(values) => values
|
||||
.iter()
|
||||
.fold(2usize, |size, value| size.saturating_add(1).saturating_add(estimated_value_bytes(value))),
|
||||
Value::Object(values) => values.iter().fold(2usize, |size, (key, value)| {
|
||||
size.saturating_add(key.len()).saturating_add(2).saturating_add(estimated_value_bytes(value))
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn input_too_large_error() -> DataGridExtractError {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InputTooLarge,
|
||||
"Selected data is too large for clipboard extraction; export the data to a file instead.",
|
||||
)
|
||||
}
|
||||
|
||||
struct WriteMetadata {
|
||||
mime_type: &'static str,
|
||||
file_extension: &'static str,
|
||||
omitted_columns: Vec<String>,
|
||||
warnings: Vec<DataGridExtractWarning>,
|
||||
}
|
||||
|
||||
fn write_extraction(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<WriteMetadata, DataGridExtractError> {
|
||||
let extractor = context.request.extractor;
|
||||
match extractor {
|
||||
DataGridExtractorId::Tsv | DataGridExtractorId::TsvWithHeaders => {
|
||||
let options = DataGridDsvOptions {
|
||||
column_separator: "\t".to_string(),
|
||||
include_column_header: extractor == DataGridExtractorId::TsvWithHeaders,
|
||||
..context.request.options.dsv.clone()
|
||||
};
|
||||
write_dsv(context, output, &options)?;
|
||||
Ok(text_metadata("text/tab-separated-values", "tsv"))
|
||||
}
|
||||
DataGridExtractorId::Csv | DataGridExtractorId::CsvWithHeaders => {
|
||||
let options = DataGridDsvOptions {
|
||||
column_separator: ",".to_string(),
|
||||
include_column_header: extractor == DataGridExtractorId::CsvWithHeaders,
|
||||
..context.request.options.dsv.clone()
|
||||
};
|
||||
write_dsv(context, output, &options)?;
|
||||
Ok(text_metadata("text/csv", "csv"))
|
||||
}
|
||||
DataGridExtractorId::PipeSeparated => {
|
||||
let options =
|
||||
DataGridDsvOptions { column_separator: "|".to_string(), ..context.request.options.dsv.clone() };
|
||||
write_dsv(context, output, &options)?;
|
||||
Ok(text_metadata("text/plain", "txt"))
|
||||
}
|
||||
DataGridExtractorId::Dsv => {
|
||||
write_dsv(context, output, &context.request.options.dsv)?;
|
||||
Ok(text_metadata("text/plain", "txt"))
|
||||
}
|
||||
DataGridExtractorId::OneRow => {
|
||||
write_one_row(context, output)?;
|
||||
Ok(text_metadata("text/csv", "csv"))
|
||||
}
|
||||
DataGridExtractorId::Json => {
|
||||
let warnings = write_json(context, output)?;
|
||||
Ok(WriteMetadata {
|
||||
mime_type: "application/json",
|
||||
file_extension: "json",
|
||||
omitted_columns: Vec::new(),
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
DataGridExtractorId::JsonLines => {
|
||||
let warnings = write_json_lines(context, output)?;
|
||||
Ok(WriteMetadata {
|
||||
mime_type: "application/x-ndjson",
|
||||
file_extension: "jsonl",
|
||||
omitted_columns: Vec::new(),
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
DataGridExtractorId::SqlInList => {
|
||||
write_sql_in_list(context, output)?;
|
||||
Ok(text_metadata("application/sql", "sql"))
|
||||
}
|
||||
DataGridExtractorId::SqlInserts => write_sql_inserts(context, output),
|
||||
DataGridExtractorId::SqlUpdates => write_sql_updates(context, output),
|
||||
DataGridExtractorId::WhereClause => {
|
||||
write_where_clause(context, output)?;
|
||||
Ok(text_metadata("application/sql", "sql"))
|
||||
}
|
||||
DataGridExtractorId::Markdown => {
|
||||
write_markdown(context, output)?;
|
||||
Ok(text_metadata("text/markdown", "md"))
|
||||
}
|
||||
DataGridExtractorId::Html => {
|
||||
write_html(context, output)?;
|
||||
Ok(text_metadata("text/html", "html"))
|
||||
}
|
||||
DataGridExtractorId::Xml => {
|
||||
write_xml(context, output)?;
|
||||
Ok(text_metadata("application/xml", "xml"))
|
||||
}
|
||||
DataGridExtractorId::Pretty => {
|
||||
write_pretty(context, output)?;
|
||||
Ok(text_metadata("text/plain", "txt"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn text_metadata(mime_type: &'static str, extension: &'static str) -> WriteMetadata {
|
||||
WriteMetadata { mime_type, file_extension: extension, omitted_columns: Vec::new(), warnings: Vec::new() }
|
||||
}
|
||||
|
||||
fn normalized_name_eq(left: &str, right: &str) -> bool {
|
||||
normalized_name(left).eq_ignore_ascii_case(normalized_name(right))
|
||||
}
|
||||
|
||||
fn normalized_name(name: &str) -> &str {
|
||||
name.trim_matches(|character| matches!(character, '`' | '"' | '[' | ']'))
|
||||
}
|
||||
|
||||
fn value_text(value: &Value) -> Cow<'_, str> {
|
||||
match value {
|
||||
Value::Null => Cow::Borrowed("NULL"),
|
||||
Value::String(value) => Cow::Borrowed(value),
|
||||
Value::Bool(value) => Cow::Owned(value.to_string()),
|
||||
Value::Number(value) => Cow::Owned(value.to_string()),
|
||||
Value::Array(_) | Value::Object(_) => Cow::Owned(value.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_bytes(output: &mut dyn Write, bytes: &[u8]) -> Result<(), DataGridExtractError> {
|
||||
output.write_all(bytes).map_err(|error| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::EncodingFailed,
|
||||
format!("Failed to write extractor output: {error}"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,327 @@
|
||||
use super::{
|
||||
estimated_value_bytes, normalized_name_eq, write_bytes, DataGridExtractError, DataGridExtractErrorCode,
|
||||
DataGridExtractWarning, DataGridExtractWarningCode, ExtractContext, WriteMetadata,
|
||||
DATA_GRID_EXTRACTOR_MAX_OUTPUT_BYTES,
|
||||
};
|
||||
use crate::data_grid_sql::{
|
||||
build_column_predicate, build_data_grid_copy_insert_statement, build_data_grid_copy_update_statements,
|
||||
format_grid_sql_literal, is_auto_generated_column, is_grid_insert_omitted_column, is_non_identity_generated_column,
|
||||
supports_relational_copy_predicates, DataGridCopyInsertStatementOptions, DataGridCopyUpdateStatementOptions,
|
||||
DataGridTableMeta,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashSet;
|
||||
use std::io::Write;
|
||||
|
||||
pub(super) fn write_sql_in_list(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
write_bytes(output, b"(")?;
|
||||
for (row_index, row) in context.request.rows.iter().enumerate() {
|
||||
if row_index > 0 {
|
||||
write_bytes(output, b", ")?;
|
||||
}
|
||||
if context.selected_columns.len() > 1 {
|
||||
write_bytes(output, b"(")?;
|
||||
}
|
||||
for (column_index, source_index) in context.selected_source_indexes.iter().enumerate() {
|
||||
if column_index > 0 {
|
||||
write_bytes(output, b", ")?;
|
||||
}
|
||||
let value = &row[*source_index];
|
||||
let info = context.selected_column_info[column_index];
|
||||
write_bytes(output, format_grid_sql_literal(value, context.request.database_type, info).as_bytes())?;
|
||||
}
|
||||
if context.selected_columns.len() > 1 {
|
||||
write_bytes(output, b")")?;
|
||||
}
|
||||
}
|
||||
write_bytes(output, b")")
|
||||
}
|
||||
|
||||
pub(super) fn write_sql_inserts(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<WriteMetadata, DataGridExtractError> {
|
||||
ensure_sql_builder_budget(context)?;
|
||||
let data = sql_selected_data(context, false)?;
|
||||
let statement = build_data_grid_copy_insert_statement(DataGridCopyInsertStatementOptions {
|
||||
database_type: context.request.database_type,
|
||||
table_meta: context.request.table_meta.clone(),
|
||||
columns: data.columns,
|
||||
column_types: Some(data.column_types),
|
||||
source_columns: Some(data.source_columns),
|
||||
rows: data.rows,
|
||||
exclude_primary_keys: context.request.options.sql.exclude_primary_keys_from_insert,
|
||||
include_computed_columns: !context.request.options.sql.skip_computed_columns,
|
||||
insert_mode: context.request.options.sql.insert_mode,
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::NoWritableColumns,
|
||||
"No insertable columns remain after applying extractor column rules.",
|
||||
)
|
||||
})?;
|
||||
write_bytes(output, statement.as_bytes())?;
|
||||
Ok(sql_metadata(data.omitted_columns))
|
||||
}
|
||||
|
||||
pub(super) fn write_sql_updates(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<WriteMetadata, DataGridExtractError> {
|
||||
ensure_sql_builder_budget(context)?;
|
||||
let table_meta = context.request.table_meta.as_ref().ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::MissingTableMetadata,
|
||||
"SQL Updates requires table metadata.",
|
||||
)
|
||||
})?;
|
||||
if table_meta.primary_keys.is_empty() {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::MissingPrimaryKey,
|
||||
"SQL Updates requires at least one primary key column.",
|
||||
));
|
||||
}
|
||||
let mut data = sql_selected_data(context, true)?;
|
||||
append_missing_primary_keys(context, table_meta, &mut data.columns, &mut data.source_columns, &mut data.rows)?;
|
||||
|
||||
let primary_key_names = table_meta.primary_keys.iter().map(|name| normalize_name(name)).collect::<HashSet<_>>();
|
||||
let writable_count =
|
||||
data.source_columns.iter().flatten().filter(|name| !primary_key_names.contains(&normalize_name(name))).count();
|
||||
if writable_count == 0 {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::NoWritableColumns,
|
||||
"Select at least one writable non-primary-key column for SQL Updates.",
|
||||
));
|
||||
}
|
||||
let statements = build_data_grid_copy_update_statements(DataGridCopyUpdateStatementOptions {
|
||||
database_type: context.request.database_type,
|
||||
table_meta: table_meta.clone(),
|
||||
columns: data.columns,
|
||||
source_columns: Some(data.source_columns),
|
||||
rows: data.rows,
|
||||
});
|
||||
if statements.len() != context.request.rows.len() {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::NullPrimaryKey,
|
||||
"One or more selected rows could not produce SQL Updates.",
|
||||
));
|
||||
}
|
||||
write_bytes(output, statements.join("\n").as_bytes())?;
|
||||
Ok(sql_metadata(data.omitted_columns))
|
||||
}
|
||||
|
||||
fn ensure_sql_builder_budget(context: &ExtractContext<'_>) -> Result<(), DataGridExtractError> {
|
||||
let value_bytes = context
|
||||
.request
|
||||
.rows
|
||||
.iter()
|
||||
.flatten()
|
||||
.fold(0usize, |size, value| size.saturating_add(estimated_value_bytes(value)));
|
||||
let identifier_bytes = context
|
||||
.request
|
||||
.table_meta
|
||||
.as_ref()
|
||||
.map_or(0, |table_meta| {
|
||||
table_meta
|
||||
.table_name
|
||||
.len()
|
||||
.saturating_add(table_meta.catalog.as_deref().map_or(0, str::len))
|
||||
.saturating_add(table_meta.database.as_deref().map_or(0, str::len))
|
||||
.saturating_add(table_meta.schema.as_deref().map_or(0, str::len))
|
||||
.saturating_add(table_meta.primary_keys.iter().fold(0usize, |size, key| size.saturating_add(key.len())))
|
||||
})
|
||||
.saturating_add(context.selected_columns.iter().fold(0usize, |size, column| {
|
||||
size.saturating_add(column.display_name.len())
|
||||
.saturating_add(column.source_name.as_deref().map_or(0, str::len))
|
||||
}));
|
||||
let repeats_identifiers_per_row = context.request.extractor == super::DataGridExtractorId::SqlUpdates
|
||||
|| context.request.options.sql.insert_mode == crate::data_grid_sql::DataGridCopyInsertMode::RowByRow;
|
||||
let statement_overhead = identifier_bytes
|
||||
.saturating_mul(4)
|
||||
.saturating_add(256)
|
||||
.saturating_mul(if repeats_identifiers_per_row { context.request.rows.len() } else { 1 });
|
||||
let estimated_bytes = value_bytes.saturating_mul(6).saturating_add(statement_overhead);
|
||||
if estimated_bytes > DATA_GRID_EXTRACTOR_MAX_OUTPUT_BYTES {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::OutputTooLarge,
|
||||
"Estimated SQL clipboard output exceeds 32 MiB; export the data to a file instead.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append_missing_primary_keys(
|
||||
context: &ExtractContext<'_>,
|
||||
table_meta: &DataGridTableMeta,
|
||||
columns: &mut Vec<String>,
|
||||
source_columns: &mut Vec<Option<String>>,
|
||||
rows: &mut [Vec<Value>],
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
for primary_key in &table_meta.primary_keys {
|
||||
if source_columns.iter().flatten().any(|column| normalized_name_eq(column, primary_key)) {
|
||||
continue;
|
||||
}
|
||||
let column = context
|
||||
.request
|
||||
.columns
|
||||
.iter()
|
||||
.find(|column| {
|
||||
normalized_name_eq(column.source_name.as_deref().unwrap_or(&column.display_name), primary_key)
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::MissingPrimaryKey,
|
||||
format!("Primary key column '{primary_key}' is not present in the result set."),
|
||||
)
|
||||
})?;
|
||||
if context.request.rows.iter().any(|row| match row.get(column.source_index) {
|
||||
Some(value) => value.is_null(),
|
||||
None => true,
|
||||
}) {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::NullPrimaryKey,
|
||||
format!("Primary key column '{primary_key}' contains a NULL value."),
|
||||
));
|
||||
}
|
||||
columns.push(column.display_name.clone());
|
||||
source_columns.push(Some(primary_key.clone()));
|
||||
for (row_index, row) in context.request.rows.iter().enumerate() {
|
||||
let value = row.get(column.source_index).ok_or_else(|| {
|
||||
DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::InvalidColumnMapping,
|
||||
format!("Primary key column '{primary_key}' maps outside row {row_index}."),
|
||||
)
|
||||
})?;
|
||||
rows[row_index].push(value.clone());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SqlSelectedData {
|
||||
columns: Vec<String>,
|
||||
source_columns: Vec<Option<String>>,
|
||||
column_types: Vec<Option<String>>,
|
||||
rows: Vec<Vec<Value>>,
|
||||
omitted_columns: Vec<String>,
|
||||
}
|
||||
|
||||
fn sql_selected_data(context: &ExtractContext<'_>, for_update: bool) -> Result<SqlSelectedData, DataGridExtractError> {
|
||||
let mut included = Vec::new();
|
||||
let mut omitted = Vec::new();
|
||||
let mut included_indexes = Vec::new();
|
||||
for (index, column) in context.selected_columns.iter().enumerate() {
|
||||
let source_name = column.source_name.as_deref().unwrap_or(&column.display_name);
|
||||
let info = context.selected_column_info[index];
|
||||
let is_primary_key = context.request.table_meta.as_ref().is_some_and(|meta| {
|
||||
meta.primary_keys.iter().any(|primary_key| normalized_name_eq(primary_key, source_name))
|
||||
});
|
||||
let omit = (!for_update
|
||||
&& is_grid_insert_omitted_column(
|
||||
context.request.database_type,
|
||||
info,
|
||||
Some(source_name),
|
||||
!context.request.options.sql.skip_computed_columns,
|
||||
))
|
||||
|| (for_update
|
||||
&& context.request.options.sql.skip_computed_columns
|
||||
&& info.is_some_and(|column| is_non_identity_generated_column(Some(column))))
|
||||
|| (context.request.options.sql.skip_generated_columns && info.is_some_and(is_auto_generated_column))
|
||||
|| (for_update && is_primary_key)
|
||||
|| (!for_update && context.request.options.sql.exclude_primary_keys_from_insert && is_primary_key);
|
||||
if omit {
|
||||
omitted.push(source_name.to_string());
|
||||
} else {
|
||||
included.push(*column);
|
||||
included_indexes.push(index);
|
||||
}
|
||||
}
|
||||
if included.is_empty() {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::NoWritableColumns,
|
||||
"No writable columns remain after applying extractor column rules.",
|
||||
));
|
||||
}
|
||||
let columns = included.iter().map(|column| column.display_name.clone()).collect();
|
||||
let source_columns = included
|
||||
.iter()
|
||||
.map(|column| Some(column.source_name.clone().unwrap_or_else(|| column.display_name.clone())))
|
||||
.collect();
|
||||
let column_types = included_indexes
|
||||
.iter()
|
||||
.map(|index| context.selected_column_info[*index].map(|info| info.data_type.clone()))
|
||||
.collect();
|
||||
let rows = context
|
||||
.request
|
||||
.rows
|
||||
.iter()
|
||||
.map(|row| included.iter().map(|column| row[column.source_index].clone()).collect())
|
||||
.collect();
|
||||
Ok(SqlSelectedData { columns, source_columns, column_types, rows, omitted_columns: omitted })
|
||||
}
|
||||
|
||||
pub(super) fn write_where_clause(
|
||||
context: &ExtractContext<'_>,
|
||||
output: &mut dyn Write,
|
||||
) -> Result<(), DataGridExtractError> {
|
||||
// WHERE-clause extraction emits relational SQL predicates and is not meaningful
|
||||
// for graph/document/time-series stores. Unlike `write_sql_updates`, this
|
||||
// intentionally does NOT auto-append primary keys: the predicate reflects the
|
||||
// exact cells the user selected.
|
||||
if !supports_relational_copy_predicates(context.request.database_type) {
|
||||
return Err(DataGridExtractError::new(
|
||||
DataGridExtractErrorCode::UnsupportedDatabase,
|
||||
"WHERE-clause extraction is not supported for this database type.",
|
||||
));
|
||||
}
|
||||
for (row_index, row) in context.request.rows.iter().enumerate() {
|
||||
if row_index > 0 {
|
||||
write_bytes(output, b" OR ")?;
|
||||
}
|
||||
if context.request.rows.len() > 1 {
|
||||
write_bytes(output, b"(")?;
|
||||
}
|
||||
for (column_index, source_index) in context.selected_source_indexes.iter().enumerate() {
|
||||
if column_index > 0 {
|
||||
write_bytes(output, b" AND ")?;
|
||||
}
|
||||
let column = context.selected_columns[column_index];
|
||||
let value = &row[*source_index];
|
||||
let source_name = column.source_name.as_deref().unwrap_or(&column.display_name);
|
||||
// Reuse the UPDATE predicate builder so NULL -> IS NULL, MySQL BINARY,
|
||||
// and MySQL JSON CAST(...) stay consistent with copy-as-UPDATE.
|
||||
let predicate = build_column_predicate(
|
||||
context.request.database_type,
|
||||
source_name,
|
||||
value,
|
||||
context.selected_column_info[column_index],
|
||||
true,
|
||||
None,
|
||||
);
|
||||
write_bytes(output, predicate.as_bytes())?;
|
||||
}
|
||||
if context.request.rows.len() > 1 {
|
||||
write_bytes(output, b")")?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sql_metadata(omitted_columns: Vec<String>) -> WriteMetadata {
|
||||
let warnings = if omitted_columns.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![DataGridExtractWarning {
|
||||
code: DataGridExtractWarningCode::OmittedColumns,
|
||||
message: format!("Omitted columns: {}", omitted_columns.join(", ")),
|
||||
}]
|
||||
};
|
||||
WriteMetadata { mime_type: "application/sql", file_extension: "sql", omitted_columns, warnings }
|
||||
}
|
||||
|
||||
fn normalize_name(name: &str) -> String {
|
||||
name.trim_matches(|character| matches!(character, '`' | '"' | '[' | ']')).to_ascii_uppercase()
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
use super::*;
|
||||
use crate::data_grid_sql::{DataGridColumnInfo, DataGridTableMeta};
|
||||
use crate::models::connection::DatabaseType;
|
||||
use serde_json::json;
|
||||
|
||||
fn column(name: &str, source_index: usize) -> DataGridExtractColumn {
|
||||
DataGridExtractColumn { display_name: name.to_string(), source_name: Some(name.to_string()), source_index }
|
||||
}
|
||||
|
||||
fn request(extractor: DataGridExtractorId) -> DataGridExtractRequest {
|
||||
DataGridExtractRequest {
|
||||
version: DATA_GRID_EXTRACTOR_CONTRACT_VERSION,
|
||||
extractor,
|
||||
database_type: Some(DatabaseType::Postgres),
|
||||
table_meta: None,
|
||||
columns: vec![column("id", 0), column("name", 1)],
|
||||
selected_column_indexes: vec![0, 1],
|
||||
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Grace, Hopper")]],
|
||||
selection_kind: DataGridSelectionKind::Cells,
|
||||
options: DataGridExtractorOptions::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partially_deserialized_options_use_the_canonical_defaults() {
|
||||
let options = serde_json::from_value::<DataGridExtractorOptions>(json!({
|
||||
"dsv": { "columnSeparator": ";" },
|
||||
"sql": { "skipGeneratedColumns": false },
|
||||
"json": {}
|
||||
}))
|
||||
.expect("deserialize partial extractor options");
|
||||
|
||||
assert_eq!(options.dsv.column_separator, ";");
|
||||
assert_eq!(options.dsv.row_separator, "\n");
|
||||
assert_eq!(options.dsv.null_text, "NULL");
|
||||
assert_eq!(options.dsv.quote, '"');
|
||||
assert_eq!(options.dsv.quote_policy, DataGridQuotePolicy::Minimal);
|
||||
assert!(!options.dsv.include_column_header);
|
||||
assert!(!options.dsv.include_row_header);
|
||||
assert!(options.sql.skip_computed_columns);
|
||||
assert!(!options.sql.skip_generated_columns);
|
||||
assert_eq!(options.sql.insert_mode, crate::data_grid_sql::DataGridCopyInsertMode::Merged);
|
||||
assert!(!options.sql.exclude_primary_keys_from_insert);
|
||||
assert!(options.json.pretty);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_csv_with_minimal_standard_quoting() {
|
||||
let result = extract_data_grid_selection(request(DataGridExtractorId::Csv)).expect("CSV extraction");
|
||||
assert_eq!(result.text, "1,Ada\n2,\"Grace, Hopper\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_distinguishes_null_from_null_string() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.rows = vec![vec![json!(1), Value::Null], vec![json!(2), json!("NULL")]];
|
||||
let result = extract_data_grid_selection(request).expect("CSV extraction");
|
||||
// NULL emits the bare sentinel; the literal string "NULL" is force-quoted.
|
||||
assert_eq!(result.text, "1,NULL\n2,\"NULL\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_keeps_null_unquoted_under_always_quote() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.rows = vec![vec![json!(1), Value::Null], vec![json!(2), json!("NULL")]];
|
||||
request.options.dsv.quote_policy = DataGridQuotePolicy::Always;
|
||||
let result = extract_data_grid_selection(request).expect("CSV extraction");
|
||||
// NULL stays bare even under Always; the string is quoted, so they differ.
|
||||
assert_eq!(result.text, "\"1\",NULL\n\"2\",\"NULL\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_null_and_sentinel_collapse_under_never_quote() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.rows = vec![vec![json!(1), Value::Null], vec![json!(2), json!("NULL")]];
|
||||
request.options.dsv.quote_policy = DataGridQuotePolicy::Never;
|
||||
let result = extract_data_grid_selection(request).expect("CSV extraction");
|
||||
// Under Never quoting, NULL and the string "NULL" are inherently indistinguishable.
|
||||
assert_eq!(result.text, "1,NULL\n2,NULL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_empty_null_text_distinguishes_empty_string() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.rows = vec![vec![json!(1), Value::Null], vec![json!(2), json!("")]];
|
||||
request.options.dsv.null_text = String::new();
|
||||
let result = extract_data_grid_selection(request).expect("CSV extraction");
|
||||
// NULL -> bare empty; the empty string is force-quoted.
|
||||
assert_eq!(result.text, "1,\n2,\"\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_row_distinguishes_null_from_null_string() {
|
||||
let mut request = request(DataGridExtractorId::OneRow);
|
||||
request.rows = vec![vec![json!(1), Value::Null], vec![json!(2), json!("NULL")]];
|
||||
let result = extract_data_grid_selection(request).expect("one-row extraction");
|
||||
assert_eq!(result.text, "1,NULL,2,\"NULL\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_one_row_as_a_single_csv_record() {
|
||||
let result = extract_data_grid_selection(request(DataGridExtractorId::OneRow)).expect("One-row extraction");
|
||||
assert_eq!(result.text, "1,Ada,2,\"Grace, Hopper\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_all_remaining_text_table_formats() {
|
||||
let cases = [
|
||||
(DataGridExtractorId::Tsv, "1\tAda\n2\tGrace, Hopper"),
|
||||
(DataGridExtractorId::TsvWithHeaders, "id\tname\n1\tAda\n2\tGrace, Hopper"),
|
||||
(DataGridExtractorId::CsvWithHeaders, "id,name\n1,Ada\n2,\"Grace, Hopper\""),
|
||||
(DataGridExtractorId::PipeSeparated, "1|Ada\n2|Grace, Hopper"),
|
||||
(
|
||||
DataGridExtractorId::Markdown,
|
||||
"| id | name |\n| --- | --- |\n| 1 | Ada |\n| 2 | Grace, Hopper |",
|
||||
),
|
||||
(
|
||||
DataGridExtractorId::Pretty,
|
||||
"+----+---------------+\n| id | name |\n+----+---------------+\n| 1 | Ada |\n| 2 | Grace, Hopper |\n+----+---------------+\n",
|
||||
),
|
||||
];
|
||||
|
||||
for (extractor, expected) in cases {
|
||||
let result = extract_data_grid_selection(request(extractor)).expect("text table extraction");
|
||||
assert_eq!(result.text, expected, "unexpected output for {extractor:?}");
|
||||
assert_eq!(result.row_count, 2);
|
||||
assert_eq!(result.column_count, 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pretty_output_uses_terminal_width_for_cjk_and_emoji() {
|
||||
let mut request = request(DataGridExtractorId::Pretty);
|
||||
request.columns = vec![column("标签", 0), column("name", 1)];
|
||||
request.rows = vec![vec![json!("中文"), json!("🙂")]];
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("Unicode pretty extraction");
|
||||
|
||||
assert_eq!(
|
||||
result.text,
|
||||
"+------+------+
|
||||
| 标签 | name |
|
||||
+------+------+
|
||||
| 中文 | 🙂 |
|
||||
+------+------+
|
||||
"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_row_honors_standard_csv_quote_configuration() {
|
||||
let mut request = request(DataGridExtractorId::OneRow);
|
||||
request.options.dsv.quote_policy = DataGridQuotePolicy::Always;
|
||||
request.options.dsv.quote = '\'';
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("configured One-row extraction");
|
||||
|
||||
assert_eq!(result.text, "'1','Ada','2','Grace, Hopper'");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_multi_column_sql_in_as_row_value_tuples() {
|
||||
let result = extract_data_grid_selection(request(DataGridExtractorId::SqlInList)).expect("SQL IN extraction");
|
||||
assert_eq!(result.text, "((1, 'Ada'), (2, 'Grace, Hopper'))");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_duplicate_json_columns_without_overwriting_values() {
|
||||
let mut request = request(DataGridExtractorId::Json);
|
||||
request.columns[1].display_name = "id".to_string();
|
||||
let result = extract_data_grid_selection(request).expect("JSON extraction");
|
||||
assert!(result.text.contains("\"id_2\""));
|
||||
assert_eq!(result.warnings.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn avoids_collisions_between_generated_and_existing_json_column_suffixes() {
|
||||
let mut request = request(DataGridExtractorId::Json);
|
||||
request.columns = vec![column("id", 0), column("id", 1), column("id_2", 2)];
|
||||
request.selected_column_indexes = vec![0, 1, 2];
|
||||
request.rows = vec![vec![json!(1), json!(2), json!(3)]];
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("JSON extraction");
|
||||
let rows = serde_json::from_str::<Value>(&result.text).expect("valid JSON output");
|
||||
|
||||
assert_eq!(rows, json!([{"id": 1, "id_3": 2, "id_2": 3}]));
|
||||
assert_eq!(result.warnings.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_updates_with_hidden_primary_keys_and_selected_columns() {
|
||||
let mut request = request(DataGridExtractorId::SqlUpdates);
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: Some("public".to_string()),
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: Some(vec![
|
||||
DataGridColumnInfo {
|
||||
name: "id".to_string(),
|
||||
data_type: "int".to_string(),
|
||||
is_nullable: false,
|
||||
is_primary_key: true,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
},
|
||||
DataGridColumnInfo {
|
||||
name: "name".to_string(),
|
||||
data_type: "varchar".to_string(),
|
||||
is_nullable: false,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
},
|
||||
]),
|
||||
});
|
||||
request.selected_column_indexes = vec![1];
|
||||
let result = extract_data_grid_selection(request).expect("SQL Updates extraction");
|
||||
assert_eq!(
|
||||
result.text,
|
||||
"UPDATE \"public\".\"users\" SET \"name\" = 'Ada' WHERE \"id\" = 1;\nUPDATE \"public\".\"users\" SET \"name\" = 'Grace, Hopper' WHERE \"id\" = 2;"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_update_computed_column_option_matches_the_frontend_capability() {
|
||||
let mut request = request(DataGridExtractorId::SqlUpdates);
|
||||
request.columns[1].display_name = "search_text".to_string();
|
||||
request.columns[1].source_name = Some("search_text".to_string());
|
||||
request.selected_column_indexes = vec![1];
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: Some(vec![
|
||||
DataGridColumnInfo {
|
||||
name: "id".to_string(),
|
||||
data_type: "int".to_string(),
|
||||
is_nullable: false,
|
||||
is_primary_key: true,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
},
|
||||
DataGridColumnInfo {
|
||||
name: "search_text".to_string(),
|
||||
data_type: "text".to_string(),
|
||||
is_nullable: true,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: Some("GENERATED ALWAYS AS".to_string()),
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
let skipped = extract_data_grid_selection(request.clone()).expect_err("computed UPDATE is skipped by default");
|
||||
assert_eq!(skipped.code, DataGridExtractErrorCode::NoWritableColumns);
|
||||
|
||||
request.options.sql.skip_computed_columns = false;
|
||||
let included = extract_data_grid_selection(request).expect("explicit computed UPDATE extraction");
|
||||
assert!(included.text.contains("SET \"search_text\" ="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_null_primary_keys_fail_fast() {
|
||||
let mut request = request(DataGridExtractorId::SqlUpdates);
|
||||
request.rows[0][0] = Value::Null;
|
||||
request.selected_column_indexes = vec![1];
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: None,
|
||||
});
|
||||
let error = extract_data_grid_selection(request).expect_err("NULL primary key must fail");
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::NullPrimaryKey);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_custom_dsv_with_headers_nulls_and_standard_escaping() {
|
||||
let mut request = request(DataGridExtractorId::Dsv);
|
||||
request.rows[0][0] = Value::Null;
|
||||
request.rows[0][1] = json!("Ada;Lovelace");
|
||||
request.options.dsv = DataGridDsvOptions {
|
||||
column_separator: ";".to_string(),
|
||||
include_column_header: true,
|
||||
..DataGridDsvOptions::default()
|
||||
};
|
||||
let result = extract_data_grid_selection(request).expect("custom DSV extraction");
|
||||
assert_eq!(result.text, "id;name\nNULL;\"Ada;Lovelace\"\n2;Grace, Hopper");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_dsv_honors_row_headers_and_quote_policy() {
|
||||
let mut request = request(DataGridExtractorId::Dsv);
|
||||
request.options.dsv = DataGridDsvOptions {
|
||||
column_separator: ";".to_string(),
|
||||
include_row_header: true,
|
||||
quote_policy: DataGridQuotePolicy::Always,
|
||||
..DataGridDsvOptions::default()
|
||||
};
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("configured DSV extraction");
|
||||
|
||||
assert_eq!(result.text, "\"1\";\"1\";\"Ada\"\n\"2\";\"2\";\"Grace, Hopper\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_json_lines_as_one_object_per_record() {
|
||||
let result = extract_data_grid_selection(request(DataGridExtractorId::JsonLines)).expect("JSON Lines extraction");
|
||||
assert_eq!(result.text.lines().count(), 2);
|
||||
let first_row =
|
||||
serde_json::from_str::<Value>(result.text.lines().next().unwrap_or_default()).expect("first JSON Lines record");
|
||||
assert_eq!(first_row, json!({"id": 1, "name": "Ada"}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_compact_json_when_pretty_printing_is_disabled() {
|
||||
let mut request = request(DataGridExtractorId::Json);
|
||||
request.options.json.pretty = false;
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("compact JSON extraction");
|
||||
|
||||
assert_eq!(result.text, "[{\"id\":1,\"name\":\"Ada\"},{\"id\":2,\"name\":\"Grace, Hopper\"}]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_null_safe_where_clause_predicates() {
|
||||
let mut request = request(DataGridExtractorId::WhereClause);
|
||||
request.selected_column_indexes = vec![1];
|
||||
request.rows = vec![vec![json!(1), Value::Null]];
|
||||
let result = extract_data_grid_selection(request).expect("WHERE extraction");
|
||||
assert_eq!(result.text, "\"name\" IS NULL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn where_clause_applies_mysql_json_cast() {
|
||||
// WHERE must reuse the UPDATE predicate builder so MySQL JSON columns get
|
||||
// CAST(... AS JSON) instead of a raw string literal.
|
||||
let mut request = request(DataGridExtractorId::WhereClause);
|
||||
request.database_type = Some(DatabaseType::Mysql);
|
||||
request.selected_column_indexes = vec![1];
|
||||
request.rows = vec![vec![json!(1), json!("{\"k\":1}")]];
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "t".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: Some(vec![
|
||||
DataGridColumnInfo {
|
||||
name: "id".to_string(),
|
||||
data_type: "int".to_string(),
|
||||
is_nullable: false,
|
||||
is_primary_key: true,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
},
|
||||
DataGridColumnInfo {
|
||||
name: "name".to_string(),
|
||||
data_type: "json".to_string(),
|
||||
is_nullable: true,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
},
|
||||
]),
|
||||
});
|
||||
let result = extract_data_grid_selection(request).expect("WHERE extraction");
|
||||
assert!(
|
||||
result.text.contains("CAST(") && result.text.contains(" AS JSON)"),
|
||||
"expected MySQL JSON CAST predicate, got: {}",
|
||||
result.text
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn where_clause_rejects_unsupported_nosql_databases() {
|
||||
for database_type in [DatabaseType::MongoDb, DatabaseType::Neo4j, DatabaseType::Tdengine] {
|
||||
let mut request = request(DataGridExtractorId::WhereClause);
|
||||
request.database_type = Some(database_type);
|
||||
let error = extract_data_grid_selection(request).expect_err("WHERE must reject NoSQL");
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::UnsupportedDatabase, "{database_type:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_data_grid_copy_update_statements_returns_empty_for_mongodb() {
|
||||
use crate::data_grid_sql::{build_data_grid_copy_update_statements, DataGridCopyUpdateStatementOptions};
|
||||
let options = DataGridCopyUpdateStatementOptions {
|
||||
database_type: Some(DatabaseType::MongoDb),
|
||||
table_meta: DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "t".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: None,
|
||||
},
|
||||
columns: vec!["id".to_string()],
|
||||
source_columns: Some(vec![Some("id".to_string())]),
|
||||
rows: vec![vec![json!(1)]],
|
||||
};
|
||||
assert!(build_data_grid_copy_update_statements(options).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_grid_sql_literal_uses_numeric_bool_for_sqlserver() {
|
||||
use crate::data_grid_sql::format_grid_sql_literal;
|
||||
let column = DataGridColumnInfo {
|
||||
name: "active".to_string(),
|
||||
data_type: "tinyint".to_string(),
|
||||
is_nullable: true,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
};
|
||||
// SQL Server has no TRUE/FALSE literals; emit 1/0 even for non-BIT columns.
|
||||
assert_eq!(format_grid_sql_literal(&json!(true), Some(DatabaseType::SqlServer), Some(&column)), "1");
|
||||
assert_eq!(format_grid_sql_literal(&json!(false), Some(DatabaseType::SqlServer), Some(&column)), "0");
|
||||
// Other dialects still emit TRUE/FALSE for non-bit columns.
|
||||
assert_eq!(format_grid_sql_literal(&json!(true), Some(DatabaseType::Postgres), Some(&column)), "TRUE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escapes_html_and_preserves_xml_null_semantics() {
|
||||
let mut html_request = request(DataGridExtractorId::Html);
|
||||
html_request.rows = vec![vec![json!(1), json!("<Ada & Grace>")]];
|
||||
let html = extract_data_grid_selection(html_request).expect("HTML extraction");
|
||||
assert!(html.text.contains("<Ada & Grace>"));
|
||||
|
||||
let mut xml_request = request(DataGridExtractorId::Xml);
|
||||
xml_request.rows = vec![vec![json!(1), Value::Null]];
|
||||
let xml = extract_data_grid_selection(xml_request).expect("XML extraction");
|
||||
assert!(xml.text.contains("name=\"name\" null=\"true\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_skips_generated_and_computed_columns() {
|
||||
let mut request = request(DataGridExtractorId::SqlInserts);
|
||||
request.columns.push(column("search_text", 2));
|
||||
request.selected_column_indexes = vec![0, 1, 2];
|
||||
request.rows = vec![vec![json!(7), json!("Ada"), json!("generated")]];
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: Some(vec![
|
||||
DataGridColumnInfo {
|
||||
name: "id".to_string(),
|
||||
data_type: "int".to_string(),
|
||||
is_nullable: false,
|
||||
is_primary_key: true,
|
||||
column_default: None,
|
||||
extra: Some("auto_increment".to_string()),
|
||||
},
|
||||
DataGridColumnInfo {
|
||||
name: "name".to_string(),
|
||||
data_type: "varchar".to_string(),
|
||||
is_nullable: false,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: None,
|
||||
},
|
||||
DataGridColumnInfo {
|
||||
name: "search_text".to_string(),
|
||||
data_type: "text".to_string(),
|
||||
is_nullable: true,
|
||||
is_primary_key: false,
|
||||
column_default: None,
|
||||
extra: Some("GENERATED ALWAYS AS".to_string()),
|
||||
},
|
||||
]),
|
||||
});
|
||||
let mut include_computed_request = request.clone();
|
||||
include_computed_request.options.sql.skip_computed_columns = false;
|
||||
let mut include_generated_request = request.clone();
|
||||
include_generated_request.options.sql.skip_generated_columns = false;
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("SQL INSERT extraction");
|
||||
assert!(result.text.contains("name"));
|
||||
assert!(!result.text.contains("search_text"));
|
||||
assert!(!result.text.contains("(id,"));
|
||||
assert_eq!(result.omitted_columns, vec!["id", "search_text"]);
|
||||
|
||||
let included =
|
||||
extract_data_grid_selection(include_computed_request).expect("SQL INSERT extraction with computed columns");
|
||||
assert!(included.text.contains("search_text"));
|
||||
assert_eq!(included.omitted_columns, vec!["id"]);
|
||||
|
||||
let included =
|
||||
extract_data_grid_selection(include_generated_request).expect("SQL INSERT extraction with generated columns");
|
||||
assert!(included.text.contains("(\"id\", \"name\")"));
|
||||
assert_eq!(included.omitted_columns, vec!["search_text"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_honors_primary_key_exclusion_and_row_by_row_mode() {
|
||||
let mut request = request(DataGridExtractorId::SqlInserts);
|
||||
request.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: Some("public".to_string()),
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: None,
|
||||
});
|
||||
request.options.sql.exclude_primary_keys_from_insert = true;
|
||||
request.options.sql.insert_mode = crate::data_grid_sql::DataGridCopyInsertMode::RowByRow;
|
||||
|
||||
let result = extract_data_grid_selection(request).expect("row-by-row SQL INSERT extraction");
|
||||
|
||||
assert_eq!(
|
||||
result.text,
|
||||
"INSERT INTO \"public\".\"users\" (\"name\") VALUES ('Ada');\nINSERT INTO \"public\".\"users\" (\"name\") VALUES ('Grace, Hopper');"
|
||||
);
|
||||
assert_eq!(result.omitted_columns, vec!["id"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_dsv_configuration() {
|
||||
let mut request = request(DataGridExtractorId::Dsv);
|
||||
request.options.dsv.column_separator.clear();
|
||||
let error = extract_data_grid_selection(request).expect_err("empty separator must fail");
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::InvalidDsvConfiguration);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_ambiguous_or_oversized_dsv_configuration() {
|
||||
let mut overlapping = request(DataGridExtractorId::Dsv);
|
||||
overlapping.options.dsv.column_separator = "|".to_string();
|
||||
overlapping.options.dsv.row_separator = "||".to_string();
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(overlapping).expect_err("overlapping separators must fail").code,
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration
|
||||
);
|
||||
|
||||
let mut conflicting_quote = request(DataGridExtractorId::Csv);
|
||||
conflicting_quote.options.dsv.quote = ',';
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(conflicting_quote).expect_err("separator quote must fail").code,
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration
|
||||
);
|
||||
|
||||
let mut oversized = request(DataGridExtractorId::Dsv);
|
||||
oversized.options.dsv.column_separator = "123456789".to_string();
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(oversized).expect_err("oversized separator must fail").code,
|
||||
DataGridExtractErrorCode::InvalidDsvConfiguration
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_contract_uses_the_frontend_camel_case_wire_shape() {
|
||||
let request = request(DataGridExtractorId::CsvWithHeaders);
|
||||
let value = serde_json::to_value(&request).expect("serialize extractor request");
|
||||
|
||||
assert_eq!(value["extractor"], "csv-with-headers");
|
||||
assert_eq!(value["databaseType"], "postgres");
|
||||
assert_eq!(value["selectedColumnIndexes"], json!([0, 1]));
|
||||
assert_eq!(value["selectionKind"], "cells");
|
||||
assert_eq!(value["options"]["dsv"]["includeColumnHeader"], false);
|
||||
assert!(value.get("selected_column_indexes").is_none());
|
||||
|
||||
let decoded = serde_json::from_value::<DataGridExtractRequest>(value).expect("deserialize extractor request");
|
||||
assert_eq!(decoded.extractor, DataGridExtractorId::CsvWithHeaders);
|
||||
assert_eq!(decoded.selected_column_indexes, vec![0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_contract_requires_an_explicit_version() {
|
||||
let mut value = serde_json::to_value(request(DataGridExtractorId::Csv)).expect("serialize extractor request");
|
||||
value.as_object_mut().expect("request object").remove("version");
|
||||
|
||||
let error = serde_json::from_value::<DataGridExtractRequest>(value).expect_err("missing version must fail");
|
||||
|
||||
assert!(error.to_string().contains("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_contract_rejects_unknown_or_misspelled_fields() {
|
||||
let mut value = serde_json::to_value(request(DataGridExtractorId::Csv)).expect("serialize extractor request");
|
||||
value["options"]["sql"]["skipComputedColumn"] = json!(false);
|
||||
|
||||
let error = serde_json::from_value::<DataGridExtractRequest>(value).expect_err("unknown option must fail");
|
||||
|
||||
assert!(error.to_string().contains("skipComputedColumn"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_contract_and_column_mappings_fail_fast() {
|
||||
let mut unsupported = request(DataGridExtractorId::Csv);
|
||||
unsupported.version = 2;
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(unsupported).expect_err("unsupported contract version").code,
|
||||
DataGridExtractErrorCode::UnsupportedVersion
|
||||
);
|
||||
|
||||
let mut invalid_column = request(DataGridExtractorId::Csv);
|
||||
invalid_column.selected_column_indexes = vec![99];
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(invalid_column).expect_err("invalid selected column").code,
|
||||
DataGridExtractErrorCode::InvalidColumnIndex
|
||||
);
|
||||
|
||||
let mut duplicate_column = request(DataGridExtractorId::Csv);
|
||||
duplicate_column.selected_column_indexes = vec![0, 0];
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(duplicate_column).expect_err("duplicate selected column").code,
|
||||
DataGridExtractErrorCode::InvalidColumnIndex
|
||||
);
|
||||
|
||||
let mut invalid_mapping = request(DataGridExtractorId::Csv);
|
||||
invalid_mapping.columns[0].source_index = 99;
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(invalid_mapping).expect_err("invalid source mapping").code,
|
||||
DataGridExtractErrorCode::InvalidColumnMapping
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_updates_require_table_primary_key_and_writable_columns() {
|
||||
let missing_table = request(DataGridExtractorId::SqlUpdates);
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(missing_table).expect_err("missing table metadata").code,
|
||||
DataGridExtractErrorCode::MissingTableMetadata
|
||||
);
|
||||
|
||||
let mut missing_primary_key = request(DataGridExtractorId::SqlUpdates);
|
||||
missing_primary_key.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: Vec::new(),
|
||||
columns: None,
|
||||
});
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(missing_primary_key).expect_err("missing primary key").code,
|
||||
DataGridExtractErrorCode::MissingPrimaryKey
|
||||
);
|
||||
|
||||
let mut primary_key_only = request(DataGridExtractorId::SqlUpdates);
|
||||
primary_key_only.selected_column_indexes = vec![0];
|
||||
primary_key_only.table_meta = Some(DataGridTableMeta {
|
||||
catalog: None,
|
||||
database: None,
|
||||
schema: None,
|
||||
table_name: "users".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: None,
|
||||
});
|
||||
assert_eq!(
|
||||
extract_data_grid_selection(primary_key_only).expect_err("primary-key-only update").code,
|
||||
DataGridExtractErrorCode::NoWritableColumns
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_requests_that_exceed_the_column_budget() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.columns =
|
||||
(0..=DATA_GRID_EXTRACTOR_MAX_COLUMNS).map(|index| column(&format!("column_{index}"), index)).collect();
|
||||
|
||||
let error = extract_data_grid_selection(request).expect_err("oversized column list must fail");
|
||||
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::InputTooLarge);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_requests_that_exceed_the_selected_index_budget() {
|
||||
// columns list is small, but selected_column_indexes is oversized —
|
||||
// the budget must catch it before Vec::with_capacity allocates.
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.selected_column_indexes = (0..=DATA_GRID_EXTRACTOR_MAX_COLUMNS).collect();
|
||||
|
||||
let error = extract_data_grid_selection(request).expect_err("oversized index list must fail");
|
||||
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::InputTooLarge);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_output_stops_before_allocating_past_the_limit() {
|
||||
let mut output = BoundedOutput::new(4);
|
||||
assert_eq!(output.write(b"1234").expect("bounded write"), 4);
|
||||
assert!(output.write(b"5").is_err());
|
||||
assert!(output.exceeded_limit());
|
||||
assert_eq!(output.into_bytes(), b"1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_requests_that_exceed_the_row_budget_before_mapping_values() {
|
||||
let mut request = request(DataGridExtractorId::Csv);
|
||||
request.rows = vec![Vec::new(); DATA_GRID_EXTRACTOR_MAX_ROWS + 1];
|
||||
|
||||
let error = extract_data_grid_selection(request).expect_err("oversized request must fail");
|
||||
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::InputTooLarge);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_estimated_oversized_sql_before_building_the_statement() {
|
||||
let mut request = request(DataGridExtractorId::SqlInserts);
|
||||
request.columns = vec![column("payload", 0)];
|
||||
request.selected_column_indexes = vec![0];
|
||||
request.rows = vec![vec![json!("x".repeat(DATA_GRID_EXTRACTOR_MAX_OUTPUT_BYTES / 6 + 1))]];
|
||||
|
||||
let error = extract_data_grid_selection(request).expect_err("oversized SQL must fail before allocation");
|
||||
|
||||
assert_eq!(error.code, DataGridExtractErrorCode::OutputTooLarge);
|
||||
}
|
||||
@@ -30,6 +30,7 @@ const MYSQL_DATA_GRID_BATCH_MAX_ROWS: usize = 500;
|
||||
const MYSQL_DATA_GRID_BATCH_TARGET_SQL_BYTES: usize = 256 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DataGridTableMeta {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -48,6 +49,7 @@ pub struct DataGridTableMeta {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
pub struct DataGridColumnInfo {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
@@ -113,10 +115,13 @@ pub struct DataGridCopyInsertStatementOptions {
|
||||
#[serde(default)]
|
||||
pub exclude_primary_keys: bool,
|
||||
#[serde(default)]
|
||||
pub include_computed_columns: bool,
|
||||
#[serde(default)]
|
||||
pub insert_mode: DataGridCopyInsertMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum DataGridCopyInsertMode {
|
||||
#[default]
|
||||
@@ -289,8 +294,14 @@ pub fn prepare_data_grid_save(options: DataGridSaveStatementOptions) -> DataGrid
|
||||
}
|
||||
}
|
||||
|
||||
/// Relational SQL UPDATE/WHERE predicates are not meaningful for graph,
|
||||
/// document, or time-series stores that don't speak relational SQL.
|
||||
pub(crate) fn supports_relational_copy_predicates(database_type: Option<DatabaseType>) -> bool {
|
||||
!matches!(database_type, Some(DatabaseType::Neo4j | DatabaseType::Tdengine | DatabaseType::MongoDb))
|
||||
}
|
||||
|
||||
pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStatementOptions) -> Vec<String> {
|
||||
if matches!(options.database_type, Some(DatabaseType::Neo4j | DatabaseType::Tdengine)) {
|
||||
if !supports_relational_copy_predicates(options.database_type) {
|
||||
return Vec::new();
|
||||
}
|
||||
let primary_keys = &options.table_meta.primary_keys;
|
||||
@@ -310,13 +321,16 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
|
||||
let primary_key_indexes: Vec<usize> = primary_key_indexes.into_iter().flatten().collect();
|
||||
let primary_key_set: Vec<String> =
|
||||
primary_keys.iter().map(|primary_key| normalize_column_name(primary_key)).collect();
|
||||
let writable_indexes: Vec<(&str, usize)> = save_columns
|
||||
let writable_indexes: Vec<(&str, usize, Option<&DataGridColumnInfo>)> = save_columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, column)| Some((column.as_deref()?, index)))
|
||||
.filter(|(column, _)| !primary_key_set.contains(&normalize_column_name(column)))
|
||||
.filter(|(column, _)| !is_oracle_row_id(options.database_type, Some(column)))
|
||||
.map(|(column, index)| (column, index, column_info_for(column_info, column)))
|
||||
.collect();
|
||||
let primary_key_info =
|
||||
primary_keys.iter().map(|primary_key| column_info_for(column_info, primary_key)).collect::<Vec<_>>();
|
||||
|
||||
if writable_indexes.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -337,15 +351,11 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
|
||||
}
|
||||
let sets = writable_indexes
|
||||
.iter()
|
||||
.map(|(column, index)| {
|
||||
.map(|(column, index, info)| {
|
||||
format!(
|
||||
"{} = {}",
|
||||
data_grid_identifier(options.database_type, column, None),
|
||||
format_grid_sql_literal(
|
||||
row.get(*index).unwrap_or(&Value::Null),
|
||||
options.database_type,
|
||||
column_info_for(column_info, column)
|
||||
)
|
||||
format_grid_sql_literal(row.get(*index).unwrap_or(&Value::Null), options.database_type, *info)
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
@@ -361,7 +371,7 @@ pub fn build_data_grid_copy_update_statements(options: DataGridCopyUpdateStateme
|
||||
options.database_type,
|
||||
primary_key,
|
||||
row.get(primary_key_indexes[index]).unwrap_or(&Value::Null),
|
||||
column_info_for(column_info, primary_key),
|
||||
primary_key_info[index],
|
||||
false,
|
||||
None,
|
||||
)
|
||||
@@ -384,20 +394,30 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
|
||||
.as_ref()
|
||||
.map(|meta| meta.primary_keys.iter().map(|primary_key| normalize_column_name(primary_key)).collect())
|
||||
.unwrap_or_default();
|
||||
let insertable_columns: Vec<(&str, usize)> = save_columns
|
||||
let insertable_columns: Vec<(&str, usize, Option<DataGridColumnInfo>)> = save_columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, column)| Some((column.as_deref()?, index)))
|
||||
.filter(|(column, _)| {
|
||||
!is_grid_insert_omitted_column(options.database_type, column_info_for(column_info, column), Some(column))
|
||||
.map(|(column, index)| {
|
||||
let fallback_type =
|
||||
options.column_types.as_deref().and_then(|types| types.get(index)).and_then(|value| value.as_deref());
|
||||
(column, index, copy_column_info(column_info, column, fallback_type))
|
||||
})
|
||||
.filter(|(column, _, info)| {
|
||||
!is_grid_insert_omitted_column(
|
||||
options.database_type,
|
||||
info.as_ref(),
|
||||
Some(column),
|
||||
options.include_computed_columns,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let insert_columns: Vec<(&str, usize)> = insertable_columns
|
||||
let insert_columns: Vec<(&str, usize, Option<DataGridColumnInfo>)> = insertable_columns
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|(column, _)| {
|
||||
.filter(|(column, _, _)| {
|
||||
!options.exclude_primary_keys || !primary_key_set.contains(&normalize_column_name(column))
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
if insert_columns.is_empty() || options.rows.is_empty() {
|
||||
@@ -418,7 +438,7 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
|
||||
);
|
||||
let columns = insert_columns
|
||||
.iter()
|
||||
.map(|(column, _)| quote_ident(options.database_type, column))
|
||||
.map(|(column, _, _)| quote_ident(options.database_type, column))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let value_rows = options
|
||||
@@ -429,20 +449,11 @@ pub fn build_data_grid_copy_insert_statement(options: DataGridCopyInsertStatemen
|
||||
"({})",
|
||||
insert_columns
|
||||
.iter()
|
||||
.map(|(column, index)| {
|
||||
.map(|(_, index, info)| {
|
||||
format_grid_copy_insert_sql_literal(
|
||||
row.get(*index).unwrap_or(&Value::Null),
|
||||
options.database_type,
|
||||
copy_column_info(
|
||||
column_info,
|
||||
column,
|
||||
options
|
||||
.column_types
|
||||
.as_deref()
|
||||
.and_then(|types| types.get(*index))
|
||||
.and_then(|value| value.as_deref()),
|
||||
)
|
||||
.as_ref(),
|
||||
info.as_ref(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
@@ -1184,6 +1195,7 @@ fn build_data_grid_save_statements(options: &DataGridSaveStatementOptions) -> Ve
|
||||
options.database_type,
|
||||
column_info_for(column_info, column),
|
||||
Some(column),
|
||||
false,
|
||||
)
|
||||
})
|
||||
.filter(|(_, value)| !value.is_null())
|
||||
@@ -1280,6 +1292,7 @@ fn build_data_grid_rollback_statements(options: &DataGridSaveStatementOptions) -
|
||||
options.database_type,
|
||||
column_info_for(column_info, column),
|
||||
Some(column),
|
||||
false,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
@@ -1724,7 +1737,10 @@ pub fn format_grid_sql_literal(
|
||||
// is a numeric/boolean type rather than a bit-string type like
|
||||
// PostgreSQL's bit(n).
|
||||
if let Some(value) = value.as_bool() {
|
||||
if is_bit_literal_column(database_type, column_info) {
|
||||
// SQL Server has no TRUE/FALSE literals (its boolean type is BIT, which
|
||||
// is_bit_literal_column already covers); any other column there still
|
||||
// needs numeric 1/0 instead of a literal.
|
||||
if is_bit_literal_column(database_type, column_info) || database_type == Some(DatabaseType::SqlServer) {
|
||||
return if value { "1" } else { "0" }.to_string();
|
||||
}
|
||||
return if value { "TRUE" } else { "FALSE" }.to_string();
|
||||
@@ -2351,7 +2367,7 @@ fn build_save_row_where(
|
||||
.join(" AND ")
|
||||
}
|
||||
|
||||
fn build_column_predicate(
|
||||
pub(crate) fn build_column_predicate(
|
||||
database_type: Option<DatabaseType>,
|
||||
column: &str,
|
||||
value: &Value,
|
||||
@@ -2479,7 +2495,7 @@ pub(crate) fn is_neo4j_element_id(database_type: Option<DatabaseType>, name: Opt
|
||||
database_type == Some(DatabaseType::Neo4j) && name == Some(DBX_NEO4J_ELEMENT_ID_COLUMN)
|
||||
}
|
||||
|
||||
fn is_auto_generated_column(column: &DataGridColumnInfo) -> bool {
|
||||
pub(crate) fn is_auto_generated_column(column: &DataGridColumnInfo) -> bool {
|
||||
column
|
||||
.extra
|
||||
.as_deref()
|
||||
@@ -2493,14 +2509,15 @@ fn grid_value_is_empty(value: &Value) -> bool {
|
||||
value.is_null() || value.as_str().is_some_and(str::is_empty)
|
||||
}
|
||||
|
||||
fn is_grid_insert_omitted_column(
|
||||
pub(crate) fn is_grid_insert_omitted_column(
|
||||
database_type: Option<DatabaseType>,
|
||||
column_info: Option<&DataGridColumnInfo>,
|
||||
name: Option<&str>,
|
||||
include_computed_columns: bool,
|
||||
) -> bool {
|
||||
is_oracle_row_id(database_type, name)
|
||||
|| is_postgres_tsvector_column(database_type, column_info)
|
||||
|| is_non_identity_generated_column(column_info)
|
||||
|| (!include_computed_columns && is_non_identity_generated_column(column_info))
|
||||
}
|
||||
|
||||
fn is_grid_update_omitted_column(
|
||||
@@ -2548,7 +2565,7 @@ fn is_postgres_tsvector_type(data_type: &str) -> bool {
|
||||
normalized == "tsvector" || normalized.ends_with(".tsvector")
|
||||
}
|
||||
|
||||
fn is_non_identity_generated_column(column_info: Option<&DataGridColumnInfo>) -> bool {
|
||||
pub(crate) fn is_non_identity_generated_column(column_info: Option<&DataGridColumnInfo>) -> bool {
|
||||
let extra = column_info.and_then(|column| column.extra.as_deref()).unwrap_or("").to_ascii_lowercase();
|
||||
extra.contains("generated always as") && !extra.contains("identity")
|
||||
}
|
||||
@@ -2941,6 +2958,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]],
|
||||
exclude_primary_keys: true,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
assert_eq!(
|
||||
@@ -2966,6 +2984,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!("ada"), json!("Ada")]],
|
||||
exclude_primary_keys: true,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
|
||||
@@ -2992,6 +3011,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(1), json!("ada"), json!("Ada")], vec![json!(2), json!("linus"), json!("Linus")]],
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::RowByRow,
|
||||
});
|
||||
assert_eq!(
|
||||
@@ -3019,6 +3039,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]],
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
|
||||
@@ -3049,6 +3070,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: rows.clone(),
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
assert_eq!(
|
||||
@@ -3113,6 +3135,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]],
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
|
||||
@@ -3136,6 +3159,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")]],
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
|
||||
@@ -5643,6 +5667,7 @@ mod tests {
|
||||
source_columns: None,
|
||||
rows: vec![vec![json!(2), json!("new")]],
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
});
|
||||
assert_eq!(
|
||||
|
||||
@@ -18,6 +18,7 @@ pub mod connection;
|
||||
pub mod connection_secrets;
|
||||
pub mod csv_export;
|
||||
pub mod data_compare;
|
||||
pub mod data_grid_extractors;
|
||||
pub mod data_grid_sql;
|
||||
pub mod database_capabilities;
|
||||
pub mod database_export;
|
||||
|
||||
@@ -433,6 +433,7 @@ pub enum ProxyType {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
|
||||
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DatabaseType {
|
||||
Mysql,
|
||||
|
||||
@@ -4162,7 +4162,13 @@ mod tests {
|
||||
assert!(still_present);
|
||||
|
||||
drop(extra_reference);
|
||||
timeout(Duration::from_secs(5), async {
|
||||
// The draining-cleanup task (spawn_duckdb_draining_cleanup) cannot drop
|
||||
// the pool until the cancelled query's blocking DuckDB task has fully
|
||||
// unwound — the connection must stay alive while the query still holds
|
||||
// it. That unwind is near-instant on an idle runner but can exceed the
|
||||
// original 5s window under heavy CI load, flaking this test, so allow
|
||||
// ample headroom.
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if !state.connections.read().await.contains_key(pool_key) {
|
||||
break;
|
||||
|
||||
@@ -76,6 +76,7 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser
|
||||
source_columns: None,
|
||||
rows: result.rows.clone(),
|
||||
exclude_primary_keys: false,
|
||||
include_computed_columns: false,
|
||||
insert_mode: DataGridCopyInsertMode::Merged,
|
||||
})
|
||||
.expect("copy insert");
|
||||
|
||||
Reference in new issue
Block a user