feat(export): apply connection query timeout to database backup
This commit is contained in:
5 files changed
+155
-91
No files matched your search
@@ -3746,6 +3746,10 @@ onUnmounted(() => {
|
|||||||
setConnectionDialogOpen(false);
|
setConnectionDialogOpen(false);
|
||||||
openSettings('tunnels');
|
openSettings('tunnels');
|
||||||
"
|
"
|
||||||
|
@open-connection-settings="
|
||||||
|
setConnectionDialogOpen(false);
|
||||||
|
openConnectionSettings($event, 'advanced');
|
||||||
|
"
|
||||||
@open-lineage-target="openLineageTarget"
|
@open-lineage-target="openLineageTarget"
|
||||||
@open-database-search-target="openDatabaseSearchTarget"
|
@open-database-search-target="openDatabaseSearchTarget"
|
||||||
@open-diagram-target="openDiagramTarget"
|
@open-diagram-target="openDiagramTarget"
|
||||||
|
|||||||
@@ -17,8 +17,9 @@ import { buildSelectedTablesPayload, isDatabaseExportTableSelectionValid } from
|
|||||||
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
|
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
|
||||||
import { useToast } from "@/composables/useToast";
|
import { useToast } from "@/composables/useToast";
|
||||||
import { Input } from "@/components/ui/input";
|
import { Input } from "@/components/ui/input";
|
||||||
import { Download, Square, CheckSquare, Search, X, Loader2 } from "@lucide/vue";
|
import { Download, Square, CheckSquare, Search, X, Loader2, Wrench } from "@lucide/vue";
|
||||||
import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker";
|
import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker";
|
||||||
|
import { isQueryTimeoutErrorMessage } from "@/lib/sql/queryError";
|
||||||
|
|
||||||
const { t } = useI18n();
|
const { t } = useI18n();
|
||||||
const { toast } = useToast();
|
const { toast } = useToast();
|
||||||
@@ -35,6 +36,10 @@ const props = defineProps<{
|
|||||||
prefillAllDatabases?: boolean;
|
prefillAllDatabases?: boolean;
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
openConnectionSettings: [connectionId: string];
|
||||||
|
}>();
|
||||||
|
|
||||||
// Connection / Database / Schema selectors
|
// Connection / Database / Schema selectors
|
||||||
const connectionId = ref("");
|
const connectionId = ref("");
|
||||||
const database = ref("");
|
const database = ref("");
|
||||||
@@ -72,6 +77,7 @@ const exportProgress = ref<ExportProgress | null>(null);
|
|||||||
const exportId = ref("");
|
const exportId = ref("");
|
||||||
const exportDone = ref(false);
|
const exportDone = ref(false);
|
||||||
const exportError = ref<string | null>(null);
|
const exportError = ref<string | null>(null);
|
||||||
|
const exportWarning = ref<string | null>(null);
|
||||||
const exportCancelled = ref(false);
|
const exportCancelled = ref(false);
|
||||||
const exportStartedAt = ref<number | null>(null);
|
const exportStartedAt = ref<number | null>(null);
|
||||||
const exportFinishedAt = ref<number | null>(null);
|
const exportFinishedAt = ref<number | null>(null);
|
||||||
@@ -163,6 +169,8 @@ function toastDatabaseExportCompletion(errorCount: number, errorSummary: string
|
|||||||
toast(t("databaseExport.exportSuccess"), 3000);
|
toast(t("databaseExport.exportSuccess"), 3000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const canChangeQueryTimeout = computed(() => !!connectionId.value && !!exportWarning.value && isQueryTimeoutErrorMessage(exportWarning.value));
|
||||||
|
|
||||||
async function loadDatabases(connId: string) {
|
async function loadDatabases(connId: string) {
|
||||||
if (!connId) return;
|
if (!connId) return;
|
||||||
loadingMeta.value = true;
|
loadingMeta.value = true;
|
||||||
@@ -334,6 +342,7 @@ async function startExport() {
|
|||||||
exportFinishedAt.value = null;
|
exportFinishedAt.value = null;
|
||||||
exportDone.value = false;
|
exportDone.value = false;
|
||||||
exportError.value = null;
|
exportError.value = null;
|
||||||
|
exportWarning.value = null;
|
||||||
exportCancelled.value = false;
|
exportCancelled.value = false;
|
||||||
exportProgress.value = {
|
exportProgress.value = {
|
||||||
exportId: exportId.value,
|
exportId: exportId.value,
|
||||||
@@ -380,11 +389,13 @@ async function startExport() {
|
|||||||
if (progress.status === "Done") {
|
if (progress.status === "Done") {
|
||||||
finishExportTiming();
|
finishExportTiming();
|
||||||
exportDone.value = true;
|
exportDone.value = true;
|
||||||
|
exportWarning.value = progress.errorSummary ?? null;
|
||||||
isExporting.value = false;
|
isExporting.value = false;
|
||||||
toastDatabaseExportCompletion(progress.errorCount ?? 0, progress.errorSummary ?? null);
|
toastDatabaseExportCompletion(progress.errorCount ?? 0, progress.errorSummary ?? null);
|
||||||
} else if (progress.status === "Error") {
|
} else if (progress.status === "Error") {
|
||||||
finishExportTiming();
|
finishExportTiming();
|
||||||
exportError.value = progress.error;
|
exportError.value = progress.error;
|
||||||
|
exportWarning.value = null;
|
||||||
isExporting.value = false;
|
isExporting.value = false;
|
||||||
} else if (progress.status === "Cancelled") {
|
} else if (progress.status === "Cancelled") {
|
||||||
finishExportTiming();
|
finishExportTiming();
|
||||||
@@ -397,6 +408,7 @@ async function startExport() {
|
|||||||
);
|
);
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
exportError.value = e?.message || String(e);
|
exportError.value = e?.message || String(e);
|
||||||
|
exportWarning.value = null;
|
||||||
const lastProgress = exportProgress.value as api.ExportProgress | null;
|
const lastProgress = exportProgress.value as api.ExportProgress | null;
|
||||||
const fallbackProgress: api.ExportProgress = {
|
const fallbackProgress: api.ExportProgress = {
|
||||||
exportId: exportId.value,
|
exportId: exportId.value,
|
||||||
@@ -440,6 +452,7 @@ async function startAllDatabasesExport() {
|
|||||||
exportFinishedAt.value = null;
|
exportFinishedAt.value = null;
|
||||||
exportDone.value = false;
|
exportDone.value = false;
|
||||||
exportError.value = null;
|
exportError.value = null;
|
||||||
|
exportWarning.value = null;
|
||||||
exportCancelled.value = false;
|
exportCancelled.value = false;
|
||||||
batchDatabaseIndex.value = 0;
|
batchDatabaseIndex.value = 0;
|
||||||
batchRowsExported.value = 0;
|
batchRowsExported.value = 0;
|
||||||
@@ -532,6 +545,7 @@ async function startAllDatabasesExport() {
|
|||||||
if (progress.status === "Error") {
|
if (progress.status === "Error") {
|
||||||
finishExportTiming();
|
finishExportTiming();
|
||||||
exportError.value = progress.error;
|
exportError.value = progress.error;
|
||||||
|
exportWarning.value = null;
|
||||||
isExporting.value = false;
|
isExporting.value = false;
|
||||||
} else if (progress.status === "Cancelled") {
|
} else if (progress.status === "Cancelled") {
|
||||||
finishExportTiming();
|
finishExportTiming();
|
||||||
@@ -551,6 +565,7 @@ async function startAllDatabasesExport() {
|
|||||||
|
|
||||||
if (!exportError.value && !exportCancelled.value) {
|
if (!exportError.value && !exportCancelled.value) {
|
||||||
exportDone.value = true;
|
exportDone.value = true;
|
||||||
|
exportWarning.value = batchFirstErrorSummary;
|
||||||
finishExportTiming();
|
finishExportTiming();
|
||||||
isExporting.value = false;
|
isExporting.value = false;
|
||||||
const finalProgress: api.ExportProgress = {
|
const finalProgress: api.ExportProgress = {
|
||||||
@@ -575,6 +590,7 @@ async function startAllDatabasesExport() {
|
|||||||
}
|
}
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
exportError.value = e?.message || String(e);
|
exportError.value = e?.message || String(e);
|
||||||
|
exportWarning.value = null;
|
||||||
updateDatabaseExportTask(batchId, {
|
updateDatabaseExportTask(batchId, {
|
||||||
exportId: batchId,
|
exportId: batchId,
|
||||||
currentObject: t("databaseExport.allDatabasesTask", { count: dbs.length }),
|
currentObject: t("databaseExport.allDatabasesTask", { count: dbs.length }),
|
||||||
@@ -629,6 +645,7 @@ function resetState() {
|
|||||||
exportProgress.value = null;
|
exportProgress.value = null;
|
||||||
exportDone.value = false;
|
exportDone.value = false;
|
||||||
exportError.value = null;
|
exportError.value = null;
|
||||||
|
exportWarning.value = null;
|
||||||
exportCancelled.value = false;
|
exportCancelled.value = false;
|
||||||
exportStartedAt.value = null;
|
exportStartedAt.value = null;
|
||||||
exportFinishedAt.value = null;
|
exportFinishedAt.value = null;
|
||||||
@@ -940,6 +957,15 @@ watch(
|
|||||||
<div v-if="exportDone" class="text-xs text-green-600 font-medium">
|
<div v-if="exportDone" class="text-xs text-green-600 font-medium">
|
||||||
{{ t("databaseExport.exportSuccess") }}
|
{{ t("databaseExport.exportSuccess") }}
|
||||||
</div>
|
</div>
|
||||||
|
<div v-if="exportDone && exportWarning" class="space-y-2">
|
||||||
|
<div class="whitespace-pre-wrap break-words text-xs text-amber-600 dark:text-amber-400">
|
||||||
|
{{ exportWarning }}
|
||||||
|
</div>
|
||||||
|
<Button v-if="canChangeQueryTimeout" variant="outline" size="sm" @click="emit('openConnectionSettings', connectionId)">
|
||||||
|
<Wrench class="mr-1 h-3.5 w-3.5" />
|
||||||
|
{{ t("editor.changeQueryTimeout") }}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
<div v-else-if="exportError" class="text-xs text-destructive font-medium">
|
<div v-else-if="exportError" class="text-xs text-destructive font-medium">
|
||||||
{{ t("databaseExport.exportError", { error: exportError }) }}
|
{{ t("databaseExport.exportError", { error: exportError }) }}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ const emit = defineEmits<{
|
|||||||
connectFailed: [message: string];
|
connectFailed: [message: string];
|
||||||
openDriverStore: [focus?: DriverStoreFocus];
|
openDriverStore: [focus?: DriverStoreFocus];
|
||||||
openTunnelProfileSettings: [];
|
openTunnelProfileSettings: [];
|
||||||
|
openConnectionSettings: [connectionId: string, initialTab: "advanced"];
|
||||||
openLineageTarget: [
|
openLineageTarget: [
|
||||||
target: {
|
target: {
|
||||||
connectionId: string;
|
connectionId: string;
|
||||||
@@ -327,6 +328,7 @@ watch(
|
|||||||
:prefill-table="dialogs.databaseExportPrefillTable.value"
|
:prefill-table="dialogs.databaseExportPrefillTable.value"
|
||||||
:prefill-tables="dialogs.databaseExportPrefillTables.value"
|
:prefill-tables="dialogs.databaseExportPrefillTables.value"
|
||||||
:prefill-all-databases="dialogs.databaseExportAllDatabases.value"
|
:prefill-all-databases="dialogs.databaseExportAllDatabases.value"
|
||||||
|
@open-connection-settings="emit('openConnectionSettings', $event, 'advanced')"
|
||||||
/>
|
/>
|
||||||
<ConfigConnectionSelectDialog
|
<ConfigConnectionSelectDialog
|
||||||
v-if="dialogs.showConfigConnectionSelectDialog.value"
|
v-if="dialogs.showConfigConnectionSelectDialog.value"
|
||||||
|
|||||||
@@ -34,6 +34,30 @@ pub fn database_export_client_session_id(export_id: &str) -> String {
|
|||||||
task_client_session_id("database-export", export_id)
|
task_client_session_id("database-export", export_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn database_export_query_options(
|
||||||
|
state: &crate::connection::AppState,
|
||||||
|
connection_id: &str,
|
||||||
|
client_session_id: &str,
|
||||||
|
max_rows: Option<usize>,
|
||||||
|
) -> crate::query::QueryExecutionOptions {
|
||||||
|
let timeout_secs =
|
||||||
|
state.configs.read().await.get(connection_id).map(|config| config.effective_query_timeout_secs()).unwrap_or(0);
|
||||||
|
database_export_query_options_for_timeout(timeout_secs, client_session_id, max_rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn database_export_query_options_for_timeout(
|
||||||
|
timeout_secs: u64,
|
||||||
|
client_session_id: &str,
|
||||||
|
max_rows: Option<usize>,
|
||||||
|
) -> crate::query::QueryExecutionOptions {
|
||||||
|
crate::query::QueryExecutionOptions {
|
||||||
|
max_rows,
|
||||||
|
timeout_secs: Some(timeout_secs),
|
||||||
|
client_session_id: Some(client_session_id.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub enum DatabaseExportOutputCompression {
|
pub enum DatabaseExportOutputCompression {
|
||||||
@@ -196,7 +220,9 @@ async fn list_mysql_export_view_dependencies(
|
|||||||
state: &crate::connection::AppState,
|
state: &crate::connection::AppState,
|
||||||
connection_id: &str,
|
connection_id: &str,
|
||||||
database: &str,
|
database: &str,
|
||||||
|
client_session_id: &str,
|
||||||
) -> Result<Vec<(String, String)>, String> {
|
) -> Result<Vec<(String, String)>, String> {
|
||||||
|
let options = database_export_query_options(state, connection_id, client_session_id, Some(usize::MAX)).await;
|
||||||
let result = crate::query::execute_sql_statement_with_options(
|
let result = crate::query::execute_sql_statement_with_options(
|
||||||
state,
|
state,
|
||||||
connection_id,
|
connection_id,
|
||||||
@@ -204,7 +230,7 @@ async fn list_mysql_export_view_dependencies(
|
|||||||
&mysql_view_dependencies_sql(database),
|
&mysql_view_dependencies_sql(database),
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
crate::query::QueryExecutionOptions { max_rows: Some(usize::MAX), ..Default::default() },
|
options,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(mysql_view_dependencies_from_rows(&result.rows))
|
Ok(mysql_view_dependencies_from_rows(&result.rows))
|
||||||
@@ -237,18 +263,21 @@ fn mysql_database_export_preamble(database: &str, charset: Option<&str>, collati
|
|||||||
async fn mysql_database_export_preamble_for_request(
|
async fn mysql_database_export_preamble_for_request(
|
||||||
state: &crate::connection::AppState,
|
state: &crate::connection::AppState,
|
||||||
request: &DatabaseExportRequest,
|
request: &DatabaseExportRequest,
|
||||||
|
client_session_id: &str,
|
||||||
) -> String {
|
) -> String {
|
||||||
let metadata_sql = format!(
|
let metadata_sql = format!(
|
||||||
"SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = {}",
|
"SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = {}",
|
||||||
mysql_sql_string_literal(&request.database)
|
mysql_sql_string_literal(&request.database)
|
||||||
);
|
);
|
||||||
let metadata = crate::query::execute_sql_statement(
|
let options = database_export_query_options(state, &request.connection_id, client_session_id, Some(1)).await;
|
||||||
|
let metadata = crate::query::execute_sql_statement_with_options(
|
||||||
state,
|
state,
|
||||||
&request.connection_id,
|
&request.connection_id,
|
||||||
&request.database,
|
&request.database,
|
||||||
&metadata_sql,
|
&metadata_sql,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
|
options,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
@@ -2663,7 +2692,7 @@ async fn export_database_sql_core_inner(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let create_database_preamble = if request.include_create_database && matches!(db_type, DatabaseType::Mysql) {
|
let create_database_preamble = if request.include_create_database && matches!(db_type, DatabaseType::Mysql) {
|
||||||
Some(mysql_database_export_preamble_for_request(state, request).await)
|
Some(mysql_database_export_preamble_for_request(state, request, &client_session_id).await)
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
@@ -2739,7 +2768,9 @@ async fn export_database_sql_core_inner(
|
|||||||
let mut tables: Vec<_> = all_tables.iter().filter(|t| !t.table_type.contains("VIEW")).collect();
|
let mut tables: Vec<_> = all_tables.iter().filter(|t| !t.table_type.contains("VIEW")).collect();
|
||||||
let mut views: Vec<_> = all_tables.iter().filter(|t| t.table_type.contains("VIEW")).collect();
|
let mut views: Vec<_> = all_tables.iter().filter(|t| t.table_type.contains("VIEW")).collect();
|
||||||
if request.include_objects && db_type == DatabaseType::Mysql && views.len() > 1 {
|
if request.include_objects && db_type == DatabaseType::Mysql && views.len() > 1 {
|
||||||
match list_mysql_export_view_dependencies(state, &request.connection_id, &request.database).await {
|
match list_mysql_export_view_dependencies(state, &request.connection_id, &request.database, &client_session_id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(dependencies) => views = sort_export_views_by_dependencies(&views, &dependencies),
|
Ok(dependencies) => views = sort_export_views_by_dependencies(&views, &dependencies),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
log::debug!(
|
log::debug!(
|
||||||
@@ -3270,7 +3301,24 @@ async fn export_database_sql_core_inner(
|
|||||||
);
|
);
|
||||||
replace_database_export_select_list(sql, &col_names, &col_types, &db_type)
|
replace_database_export_select_list(sql, &col_names, &col_types, &db_type)
|
||||||
};
|
};
|
||||||
let result = match crate::transfer::execute_read_on_pool(state, &pool_key, &sql).await {
|
let options = database_export_query_options(
|
||||||
|
state,
|
||||||
|
&request.connection_id,
|
||||||
|
&client_session_id,
|
||||||
|
Some(batch_size),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let result = match crate::query::execute_sql_statement_with_options(
|
||||||
|
state,
|
||||||
|
&request.connection_id,
|
||||||
|
&request.database,
|
||||||
|
&sql,
|
||||||
|
Some(&request.schema),
|
||||||
|
None,
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
record_export_error(
|
record_export_error(
|
||||||
@@ -3605,18 +3653,18 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use super::{
|
use super::{
|
||||||
build_database_export_object_source_sql, build_database_sql_export, build_export_insert_statements,
|
build_database_export_object_source_sql, build_database_sql_export, build_export_insert_statements,
|
||||||
database_export_select_sql, database_export_total_objects, drop_table_if_exists_sql,
|
database_export_query_options_for_timeout, database_export_select_sql, database_export_total_objects,
|
||||||
ensure_export_destination_dir, export_destination_identity_mismatch, filter_export_table_infos,
|
drop_table_if_exists_sql, ensure_export_destination_dir, export_destination_identity_mismatch,
|
||||||
format_export_sql_literal, format_export_table_ddl, format_mysql_spatial_export_literal,
|
filter_export_table_infos, format_export_sql_literal, format_export_table_ddl,
|
||||||
format_xugu_spatial_export_literal, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl,
|
format_mysql_spatial_export_literal, format_xugu_spatial_export_literal, generate_postgres_extension_ddl,
|
||||||
generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql,
|
generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl,
|
||||||
is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows,
|
generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble,
|
||||||
mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity,
|
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl,
|
||||||
record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies,
|
record_export_destination_identity, record_export_error, replace_database_export_select_list,
|
||||||
split_postgres_export_table_triggers, write_database_export_rows, BuildDatabaseSqlExportOptions,
|
sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows,
|
||||||
BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter,
|
BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, DatabaseExportObjectCounts,
|
||||||
DdlNormalizeOptions, ExportedTableSql, PostgresExportExtension, PostgresExportSequence,
|
DatabaseExportRequest, DatabaseExportWriter, DdlNormalizeOptions, ExportedTableSql, PostgresExportExtension,
|
||||||
PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
|
PostgresExportSequence, PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
|
||||||
};
|
};
|
||||||
use super::{ExportProgress, LenientExportErrors};
|
use super::{ExportProgress, LenientExportErrors};
|
||||||
use crate::connection::AppState;
|
use crate::connection::AppState;
|
||||||
@@ -3632,6 +3680,19 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn database_export_query_options_preserve_connection_timeout() {
|
||||||
|
let options = database_export_query_options_for_timeout(7, "database-export-session", Some(500));
|
||||||
|
|
||||||
|
assert_eq!(options.timeout_secs, Some(7));
|
||||||
|
assert_eq!(options.max_rows, Some(500));
|
||||||
|
assert_eq!(options.client_session_id.as_deref(), Some("database-export-session"));
|
||||||
|
|
||||||
|
let unlimited = database_export_query_options_for_timeout(0, "database-export-session", None);
|
||||||
|
assert_eq!(unlimited.timeout_secs, Some(0));
|
||||||
|
assert_eq!(unlimited.max_rows, None);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn await_export_operation_drops_pending_metadata_after_cancel() {
|
async fn await_export_operation_drops_pending_metadata_after_cancel() {
|
||||||
let export_id = format!("cancel-pending-{}", uuid::Uuid::new_v4());
|
let export_id = format!("cancel-pending-{}", uuid::Uuid::new_v4());
|
||||||
|
|||||||
@@ -5446,82 +5446,53 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
TxnConnection::Mysql(Some(conn)) => {
|
TxnConnection::Mysql(Some(conn)) => {
|
||||||
let query_result = match cancel_token.as_ref() {
|
// The query timeout is an inactivity budget reset by every received row,
|
||||||
Some(cancel_token) => {
|
// not a cap on the total duration of a long backup/export stream.
|
||||||
tokio::select! {
|
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||||
_ = cancel_token.cancelled() => Err(QUERY_CANCELED.to_string()),
|
let progress_clock_for_rows = progress_clock.clone();
|
||||||
result = conn.query_iter(sql) => result.map_err(|error| format!("Query failed: {error}")),
|
let timeout_error = format!(
|
||||||
|
"Query timed out after {} seconds",
|
||||||
|
operation_budget.query_timeout.map_or(0, |timeout| timeout.as_secs())
|
||||||
|
);
|
||||||
|
let stream_future = async {
|
||||||
|
let mut result = conn.query_iter(sql).await.map_err(|error| format!("Query failed: {error}"))?;
|
||||||
|
let Some(mut stream) =
|
||||||
|
result.stream::<mysql_async::Row>().await.map_err(|error| format!("Query failed: {error}"))?
|
||||||
|
else {
|
||||||
|
return Err("Empty result set stream".to_string());
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut batch = Vec::with_capacity(batch_size);
|
||||||
|
let mut total_rows = 0_u64;
|
||||||
|
while let Some(row_result) = stream.next().await {
|
||||||
|
match row_result {
|
||||||
|
Ok(row) => {
|
||||||
|
batch.push(
|
||||||
|
(0..row.len()).map(|index| db::mysql::mysql_value_to_json(&row, index)).collect(),
|
||||||
|
);
|
||||||
|
total_rows += 1;
|
||||||
|
if batch.len() >= batch_size {
|
||||||
|
on_batch(std::mem::take(&mut batch))?;
|
||||||
|
batch = Vec::with_capacity(batch_size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => return Err(format!("Query failed: {err}")),
|
||||||
}
|
}
|
||||||
|
progress_clock_for_rows.mark();
|
||||||
}
|
}
|
||||||
None => conn.query_iter(sql).await.map_err(|error| format!("Query failed: {error}")),
|
if !batch.is_empty() {
|
||||||
|
on_batch(batch)?;
|
||||||
|
}
|
||||||
|
Ok(total_rows)
|
||||||
};
|
};
|
||||||
match query_result {
|
await_stream_with_progress_timeout(
|
||||||
Ok(mut result) => {
|
stream_future,
|
||||||
let stream_result = match cancel_token.as_ref() {
|
operation_budget.query_timeout,
|
||||||
Some(cancel_token) => {
|
progress_clock,
|
||||||
tokio::select! {
|
cancel_token.as_ref(),
|
||||||
_ = cancel_token.cancelled() => Err(QUERY_CANCELED.to_string()),
|
timeout_error,
|
||||||
stream = result.stream::<mysql_async::Row>() => stream.map_err(|error| format!("Query failed: {error}")),
|
)
|
||||||
}
|
.await
|
||||||
}
|
|
||||||
None => {
|
|
||||||
result.stream::<mysql_async::Row>().await.map_err(|error| format!("Query failed: {error}"))
|
|
||||||
}
|
|
||||||
};
|
|
||||||
match stream_result {
|
|
||||||
Ok(Some(mut stream)) => {
|
|
||||||
let mut batch = Vec::with_capacity(batch_size);
|
|
||||||
let mut total_rows = 0_u64;
|
|
||||||
let mut error = None;
|
|
||||||
loop {
|
|
||||||
let next_row = match cancel_token.as_ref() {
|
|
||||||
Some(cancel_token) => {
|
|
||||||
tokio::select! {
|
|
||||||
_ = cancel_token.cancelled() => {
|
|
||||||
error = Some(QUERY_CANCELED.to_string());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
row = stream.next() => row,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None => stream.next().await,
|
|
||||||
};
|
|
||||||
let Some(row_result) = next_row else { break };
|
|
||||||
match row_result {
|
|
||||||
Ok(row) => {
|
|
||||||
batch.push(
|
|
||||||
(0..row.len())
|
|
||||||
.map(|index| db::mysql::mysql_value_to_json(&row, index))
|
|
||||||
.collect(),
|
|
||||||
);
|
|
||||||
total_rows += 1;
|
|
||||||
if batch.len() >= batch_size {
|
|
||||||
if let Err(err) = on_batch(std::mem::take(&mut batch)) {
|
|
||||||
error = Some(err);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
batch = Vec::with_capacity(batch_size);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
error = Some(format!("Query failed: {err}"));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if error.is_none() && !batch.is_empty() {
|
|
||||||
if let Err(err) = on_batch(batch) {
|
|
||||||
error = Some(err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
error.map_or(Ok(total_rows), Err)
|
|
||||||
}
|
|
||||||
Ok(None) => Err("Empty result set stream".to_string()),
|
|
||||||
Err(error) => Err(error),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(error) => Err(error),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
TxnConnection::Mysql(None) => Err(MANUAL_TRANSACTION_SESSION_NOT_FOUND_ERROR.to_string()),
|
TxnConnection::Mysql(None) => Err(MANUAL_TRANSACTION_SESSION_NOT_FOUND_ERROR.to_string()),
|
||||||
TxnConnection::Agent { .. } => {
|
TxnConnection::Agent { .. } => {
|
||||||
|
|||||||
Reference in new issue
Block a user