Skip SQLite page reclamation on Linux NFS mounts (#54417)

## Why

SQLite page reclamation on NFS creates I/O load and contention.

## What changed

Detect whether `sqlite_home` is on NFS on Linux and skip reclamation before opening a maintenance connection. Cache successful filesystem probes across `SqliteConfig` clones while allowing failed probes to retry.

## Testing

Add tests for shared probe caching and recovery after a missing directory is created. Update reclamation coverage to expect no reclaimed pages or file shrinkage on NFS.

GitOrigin-RevId: 4f627cabc6ad712171654c3f8b8659d2f632bbba
This commit is contained in:
Dmitriy Kovalenko
2026-10-15 21:07:40 +00:00
committed by copyberry
parent 341de32c33
commit 9c84e89ce0
9 changed files with 123 additions and 9 deletions
+2
View File
@@ -3485,6 +3485,7 @@ dependencies = [
"codex-utils-absolute-path", "codex-utils-absolute-path",
"codex-utils-path-uri", "codex-utils-path-uri",
"futures", "futures",
"rustix 1.1.4",
"serde", "serde",
] ]
@@ -4658,6 +4659,7 @@ version = "0.0.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"chrono", "chrono",
"codex-file-system",
"codex-git-utils", "codex-git-utils",
"codex-history", "codex-history",
"codex-protocol", "codex-protocol",
+3
View File
@@ -15,6 +15,9 @@ codex-utils-path-uri = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
[target.'cfg(target_os = "linux")'.dependencies]
rustix = { workspace = true, features = ["fs"] }
[lib] [lib]
test = false test = false
doctest = false doctest = false
+2
View File
@@ -1,6 +1,7 @@
mod environment_accessor; mod environment_accessor;
mod exec_permission_profile_serde; mod exec_permission_profile_serde;
mod find_up; mod find_up;
mod nfs;
use bytes::Bytes; use bytes::Bytes;
use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::config_types::WindowsSandboxLevel;
@@ -28,6 +29,7 @@ pub use find_up::FindUpErrorPolicy;
pub use find_up::find_nearest_ancestor_with_markers; pub use find_up::find_nearest_ancestor_with_markers;
pub use find_up::find_nearest_native_ancestor_with_markers; pub use find_up::find_nearest_native_ancestor_with_markers;
use futures::Stream; use futures::Stream;
pub use nfs::is_nfs;
use serde::Deserialize; use serde::Deserialize;
use serde::Serialize; use serde::Serialize;
use std::future::Future; use std::future::Future;
+16
View File
@@ -0,0 +1,16 @@
//! Detects NFS on Linux; other platforms retain their existing filesystem behavior.
//! Closing an extra file descriptor could release process-wide POSIX locks on its inode.
use std::io;
use std::path::Path;
/// On linux we sometimes do a special treatment for NFS related path. We don't support windows NFS
#[cfg(target_os = "linux")]
pub fn is_nfs(path: &Path) -> io::Result<bool> {
Ok(rustix::fs::statfs(path)?.f_type == rustix::fs::NFS_SUPER_MAGIC)
}
#[cfg(not(target_os = "linux"))]
pub fn is_nfs(_path: &Path) -> io::Result<bool> {
Ok(false)
}
+1
View File
@@ -7,6 +7,7 @@ license.workspace = true
[dependencies] [dependencies]
anyhow = { workspace = true } anyhow = { workspace = true }
chrono = { workspace = true } chrono = { workspace = true }
codex-file-system = { workspace = true }
codex-history = { workspace = true } codex-history = { workspace = true }
codex-protocol = { workspace = true } codex-protocol = { workspace = true }
codex-utils-absolute-path = { workspace = true } codex-utils-absolute-path = { workspace = true }
+12 -1
View File
@@ -1,4 +1,5 @@
//! Reclaims unused SQLite pages in opted-in databases, reducing batches after stalled passes. //! Reclaims unused SQLite pages in opted-in databases, reducing batches after stalled passes.
//! NFS on Linux skips opening a maintenance connection.
use crate::RuntimeDbPath; use crate::RuntimeDbPath;
use crate::SqliteConfig; use crate::SqliteConfig;
use log::LevelFilter; use log::LevelFilter;
@@ -64,7 +65,7 @@ impl SqliteReclamationWorker {
if owner.is_none() { if owner.is_none() {
continue; continue;
} }
delay = visit(&mut scheduled, &receiver).await; delay = visit(&sqlite, &mut scheduled, &receiver).await;
} }
}); });
@@ -79,6 +80,7 @@ impl SqliteReclamationWorker {
} }
async fn visit( async fn visit(
sqlite: &SqliteConfig,
scheduled: &mut [(RuntimeDbPath, Instant, ReclamationState)], scheduled: &mut [(RuntimeDbPath, Instant, ReclamationState)],
shutdown: &watch::Receiver<()>, shutdown: &watch::Receiver<()>,
) -> Duration { ) -> Duration {
@@ -91,6 +93,7 @@ async fn visit(
continue; continue;
} }
let result = reclaim( let result = reclaim(
sqlite,
&db.path, &db.path,
ReclamationOptions { ReclamationOptions {
budget: Budget { budget: Budget {
@@ -206,10 +209,18 @@ impl From<Budget> for ReclamationOptions {
} }
async fn reclaim( async fn reclaim(
sqlite: &SqliteConfig,
path: &Path, path: &Path,
options: impl Into<ReclamationOptions>, options: impl Into<ReclamationOptions>,
shutdown: &watch::Receiver<()>, shutdown: &watch::Receiver<()>,
) -> anyhow::Result<ReclamationPass> { ) -> anyhow::Result<ReclamationPass> {
// do not perform autovacuum and cleanup on NFS as it creates a lot of load and contention on IO
if sqlite.is_nfs().await? {
return Ok(ReclamationPass {
pages: 0,
outcome: PassOutcome::Idle,
});
}
if !tokio::fs::try_exists(path).await? { if !tokio::fs::try_exists(path).await? {
return Ok(ReclamationPass { return Ok(ReclamationPass {
pages: 0, pages: 0,
@@ -93,6 +93,7 @@ async fn interrupted_reclamation_releases_writer_and_resumes_without_data_loss()
.await?; .await?;
let (_shutdown, receiver) = watch::channel(()); let (_shutdown, receiver) = watch::channel(());
let resumed = reclaim( let resumed = reclaim(
&sqlite,
&sqlite.logs_db_path(), &sqlite.logs_db_path(),
Budget { Budget {
deadline: Some(Instant::now() + Duration::from_secs(30)), deadline: Some(Instant::now() + Duration::from_secs(30)),
@@ -104,16 +105,21 @@ async fn interrupted_reclamation_releases_writer_and_resumes_without_data_loss()
let free_after_retry: i64 = sqlx::query_scalar("PRAGMA freelist_count") let free_after_retry: i64 = sqlx::query_scalar("PRAGMA freelist_count")
.fetch_one(&pool) .fetch_one(&pool)
.await?; .await?;
assert!(resumed.pages > 0);
assert_eq!( assert_eq!(
free_before_retry - free_after_retry, free_before_retry - free_after_retry,
i64::from(resumed.pages) i64::from(resumed.pages)
); );
let bytes_after_retry = tokio::fs::metadata(sqlite.logs_db_path()).await?.len(); let bytes_after_retry = tokio::fs::metadata(sqlite.logs_db_path()).await?.len();
assert!( if sqlite.is_nfs().await? {
bytes_after_retry < bytes_before_retry, assert_eq!(resumed.pages, 0);
"reclamation must shrink the SQLite file: {bytes_before_retry} -> {bytes_after_retry} bytes" assert_eq!(bytes_after_retry, bytes_before_retry);
); } else {
assert!(resumed.pages > 0);
assert!(
bytes_after_retry < bytes_before_retry,
"reclamation must shrink the SQLite file: {bytes_before_retry} -> {bytes_after_retry} bytes"
);
}
assert_eq!( assert_eq!(
sqlx::query_as::<_, (i64, String)>(rows) sqlx::query_as::<_, (i64, String)>(rows)
.fetch_all(&pool) .fetch_all(&pool)
+21 -2
View File
@@ -23,8 +23,10 @@ use sqlx::sqlite::SqlitePoolOptions;
use sqlx::sqlite::SqliteSynchronous; use sqlx::sqlite::SqliteSynchronous;
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use std::time::Instant; use std::time::Instant;
use tokio::sync::OnceCell;
mod busy_retry; mod busy_retry;
mod validation; mod validation;
@@ -154,14 +156,16 @@ pub struct RuntimeDbPath {
} }
/// Resolved configuration shared by all Codex SQLite connections. /// Resolved configuration shared by all Codex SQLite connections.
/// Clones share quick-check attempts; a newly constructed config checks independently. /// Clones share filesystem probes and quick-check attempts; a new config checks independently.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct SqliteConfig { pub struct SqliteConfig {
sqlite_home: AbsolutePathBuf, sqlite_home: AbsolutePathBuf,
is_nfs: Arc<OnceCell<bool>>,
quick_check_manager: SqliteQuickCheckManager, quick_check_manager: SqliteQuickCheckManager,
} }
// Validation history is operational state, not part of configuration equality. // Cached filesystem information and validation history are operational state,
// not part of configuration equality.
impl PartialEq for SqliteConfig { impl PartialEq for SqliteConfig {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
self.sqlite_home == other.sqlite_home self.sqlite_home == other.sqlite_home
@@ -174,6 +178,7 @@ impl SqliteConfig {
pub fn from_sqlite_home(sqlite_home: AbsolutePathBuf) -> Self { pub fn from_sqlite_home(sqlite_home: AbsolutePathBuf) -> Self {
Self { Self {
sqlite_home, sqlite_home,
is_nfs: Arc::new(OnceCell::new()),
quick_check_manager: SqliteQuickCheckManager::new(), quick_check_manager: SqliteQuickCheckManager::new(),
} }
} }
@@ -186,6 +191,20 @@ impl SqliteConfig {
self.sqlite_home.as_path() self.sqlite_home.as_path()
} }
/// Detects if the sqlite_home directory is under the NFS mount (once per the lifetime)
pub async fn is_nfs(&self) -> anyhow::Result<bool> {
self.is_nfs
.get_or_try_init(|| async {
let sqlite_home = self.home().to_path_buf();
Ok(
tokio::task::spawn_blocking(move || codex_file_system::is_nfs(&sqlite_home))
.await??,
)
})
.await
.copied()
}
/// Return the path to the primary state database. /// Return the path to the primary state database.
pub fn state_db_path(&self) -> PathBuf { pub fn state_db_path(&self) -> PathBuf {
STATE_DB.path(self.home()) STATE_DB.path(self.home())
+55 -1
View File
@@ -1,4 +1,4 @@
//! Regression coverage for SQLite initialization under competing database locks. //! Regression coverage for SQLite initialization, cached filesystem detection, and competing locks.
use super::SqliteConfig; use super::SqliteConfig;
use codex_utils_absolute_path::test_support::PathExt; use codex_utils_absolute_path::test_support::PathExt;
@@ -8,6 +8,60 @@ use sqlx::sqlite::SqliteConnectOptions;
use sqlx::sqlite::SqliteJournalMode; use sqlx::sqlite::SqliteJournalMode;
use sqlx::sqlite::SqlitePoolOptions; use sqlx::sqlite::SqlitePoolOptions;
#[cfg(target_os = "linux")]
#[tokio::test]
async fn sqlite_home_filesystem_probe_is_shared_by_clones_until_config_replacement()
-> anyhow::Result<()> {
let sqlite_home = crate::runtime::test_support::unique_temp_dir();
tokio::fs::create_dir_all(&sqlite_home).await?;
let _cleanup = scopeguard::guard(sqlite_home.clone(), |sqlite_home| {
let _ = std::fs::remove_dir_all(sqlite_home);
});
let sqlite = SqliteConfig::new_for_testing(sqlite_home.as_path().abs());
let clone = sqlite.clone();
let is_nfs = sqlite.is_nfs().await?;
tokio::fs::remove_dir(&sqlite_home).await?;
// This clone has never queried the home. It must reuse the original's probe,
// because a fresh filesystem lookup would fail now that the home is absent.
assert_eq!(clone.is_nfs().await?, is_nfs);
let fresh = SqliteConfig::new_for_testing(sqlite_home.as_path().abs());
assert_eq!(
fresh
.is_nfs()
.await
.unwrap_err()
.downcast_ref::<std::io::Error>()
.unwrap()
.kind(),
std::io::ErrorKind::NotFound
);
Ok(())
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn failed_sqlite_home_filesystem_probe_recovers_after_home_creation() -> anyhow::Result<()> {
let sqlite_home = crate::runtime::test_support::unique_temp_dir();
let _cleanup = scopeguard::guard(sqlite_home.clone(), |sqlite_home| {
let _ = std::fs::remove_dir_all(sqlite_home);
});
let sqlite = SqliteConfig::new_for_testing(sqlite_home.as_path().abs());
assert_eq!(
sqlite
.is_nfs()
.await
.unwrap_err()
.downcast_ref::<std::io::Error>()
.unwrap()
.kind(),
std::io::ErrorKind::NotFound
);
tokio::fs::create_dir_all(&sqlite_home).await?;
sqlite.is_nfs().await?;
Ok(())
}
#[tokio::test] #[tokio::test]
async fn open_read_write_pool_initializes_fresh_database_settings() -> anyhow::Result<()> { async fn open_read_write_pool_initializes_fresh_database_settings() -> anyhow::Result<()> {
for interrupted in [false, true] { for interrupted in [false, true] {