diff --git a/Makefile b/Makefile index d1dd35c9..405c20df 100644 --- a/Makefile +++ b/Makefile @@ -271,10 +271,12 @@ install-strip: install-base $(STRIP) "$(INSTALLBINDIR)/mergerfs" .PHONY: uninstall -uninstall: uninstall-base uninstall-mount.mergerfs uninstall-man +uninstall: uninstall-base uninstall-mount.mergerfs uninstall-preload uninstall-man uninstall-base: $(RM) -f "$(INSTALLBINDIR)/mergerfs" + $(RM) -f "$(INSTALLBINDIR)/fsck.mergerfs" + $(RM) -f "$(INSTALLBINDIR)/mergerfs.collect-info" uninstall-mount.mergerfs: $(RM) -f "$(INSTALLBINDIR)/mount.mergerfs" @@ -282,6 +284,9 @@ uninstall-mount.mergerfs: uninstall-man: $(RM) -f "$(INSTALLMAN1DIR)/$(MANPAGE)" +uninstall-preload: + $(RM) -f "$(INSTALLLIBDIR)/preload.so" + .PHONY: tarball tarball: changelog version $(eval VERSION := $(shell cat VERSION)) @@ -341,7 +346,7 @@ install-build-pkgs: .PHONY: install-build-tools install-build-tools: - ./bulidtools/install-build-tools + ./buildtools/install-build-tools define build_release $(eval GITREF ?= $(shell git describe --exact-match --tags HEAD 2>/dev/null || git symbolic-ref --short HEAD 2>/dev/null || git rev-parse --short HEAD)) @@ -368,7 +373,7 @@ release-arm64: release-armhf: $(call build_release,"armhf") release-riscv64: - $(call build_release,"risc64") + $(call build_release,"riscv64") release-static: $(call build_release,"static") release-tarball: diff --git a/buildtools/install-build-tools b/buildtools/install-build-tools index ecab01de..ef332571 100755 --- a/buildtools/install-build-tools +++ b/buildtools/install-build-tools @@ -1,21 +1,32 @@ #!/bin/sh +if [ "$(id -u)" -eq 0 ]; then + SUDO= +elif command -v sudo >/dev/null 2>&1; then + SUDO=sudo +elif command -v doas >/dev/null 2>&1; then + SUDO=doas +else + printf '%s\n' "error: need root privileges; run as root or install sudo/doas" >&2 + exit 1 +fi + if [ -e /usr/bin/apt-get ]; then export DEBIAN_FRONTEND=noninteractive - apt-get -qy update - apt-get -qy --force-yes --no-install-suggests --no-install-recommends install \ + ${SUDO} apt-get -qy update + ${SUDO} apt-get -qy --no-install-suggests --no-install-recommends install \ qemu-user-static \ podman elif [ -e /usr/bin/dnf ]; then - dnf -y update - dnf -y install \ + ${SUDO} dnf -y update + ${SUDO} dnf -y install \ qemu-user-static \ podman elif [ -e /sbin/apk ]; then - apk add \ + ${SUDO} apk add \ python3 \ qemu-user-static \ podman elif [ -e /usr/bin/pacman ]; then - pacman -Sy --noconfirm qemu-user-static podman + ${SUDO} pacman -Sy --noconfirm qemu-user-static podman fi diff --git a/src/fuse_create.cpp b/src/fuse_create.cpp index ac5dee63..8a847d7b 100644 --- a/src/fuse_create.cpp +++ b/src/fuse_create.cpp @@ -326,6 +326,7 @@ _create(const fuse_req_ctx_t *ctx_, FUSE::passthrough_close(backing_id); fs::close(fi->fd); delete fi; + ffi_->fh = 0; SysLog::crit(msg); fmt::println(stderr,"{}",msg); diff --git a/src/fuse_open.cpp b/src/fuse_open.cpp index 5a47a8e3..4b99fb8e 100644 --- a/src/fuse_open.cpp +++ b/src/fuse_open.cpp @@ -17,17 +17,17 @@ */ /* - OPEN FILE MANAGEMENT AND CONCURRENCY CONTROL - ============================================ + OPEN FILE MANAGEMENT AND SHARED HANDLE TRACKING + =============================================== This file implements FUSE open() with handling for concurrent access to the same file. The key challenge is that multiple threads may simultaneously attempt to open the same file (same nodeid), and we need to: - 1. Share a single FileInfo instance among all opens of the same file - (to be able to re-open the same file and use the same backing id for - concurrent open requests) + 1. Share a single canonical FileInfo entry in the open_files map for all + opens of the same file while still giving each FUSE handle its own + FileInfo object and file descriptor 2. Properly reference count to know when to release resources 3. Handle race conditions without deadlocks or resource leaks 4. Support FUSE passthrough mode for performance @@ -41,8 +41,8 @@ When a file is opened: 1. First, we check if an entry already exists (using visit()) - 2. If it exists and is valid (fi != nullptr), we increment ref_count and - share the existing FileInfo by re-opening its file descriptor + 2. If it exists, we increment ref_count and create a new per-handle + FileInfo by re-opening the canonical file descriptor 3. If it doesn't exist, we create a new FileInfo and try to insert it 4. If insertion fails (another thread inserted first), we clean up and retry (because to use passthrough it MUST be the same underlying @@ -62,15 +62,13 @@ underlying file resources. The alternative is to not use passthrough on failure but this race should be rare. - ZOMBIE STATE PROTECTION - ----------------------- + RELEASE INTERACTION + ------------------- - When a file is being released (in fuse_release.cpp), the entry enters a - "zombie" state where fi is set to nullptr. The check at line ~305 prevents - new opens from acquiring a reference to a dying file: - - if(v_.second.fi == nullptr) - return; // Skip zombie entries + release() erases the map entry on the final close. Until then, the + canonical FileInfo in the map remains valid and overlapping opens can bump + ref_count and duplicate its fd. Secondary handles own their own FileInfo + objects and are freed independently during release(). PASSTHROUGH MODE ---------------- @@ -107,6 +105,8 @@ #include "fuse.h" +#include + #include #include #include @@ -365,7 +365,7 @@ _open(const fuse_req_ctx_t *ctx_, int backing_id = INVALID_BACKING_ID; // The ref increment keeps fuse_release.cpp from erasing and - // freeing the entry. + // freeing the canonical entry while we duplicate its fd. of.visit(ctx_->nodeid, [&](auto &v_) { @@ -455,9 +455,11 @@ _open(const fuse_req_ctx_t *ctx_, FUSE::passthrough_close(backing_id); fs::close(fi->fd); delete fi; + ffi_->fh = 0; + ::sched_yield(); } - return -EIO; + return -EAGAIN; } diff --git a/src/fuse_release.cpp b/src/fuse_release.cpp index 23167ae2..c05f754c 100644 --- a/src/fuse_release.cpp +++ b/src/fuse_release.cpp @@ -17,71 +17,26 @@ */ /* - TWO-PHASE FILE RELEASE AND RESOURCE CLEANUP - =========================================== + RELEASE OF SHARED AND PER-HANDLE FILEINFO OBJECTS + ================================================= - This file implements FUSE release() with a sophisticated two-phase cleanup - mechanism designed to handle concurrent access without deadlocks or leaks. + The open_files map stores one canonical FileInfo per nodeid together with a + ref_count and optional passthrough backing_id. Overlapping opens of the same + file increment that ref_count but still allocate their own FileInfo objects + with duplicated file descriptors for the individual FUSE handles. - THE PROBLEM: CONCURRENT RELEASE AND OPEN - ---------------------------------------- + That means release() must handle two ownership cases: + - the canonical FileInfo stored in open_files + - a per-handle FileInfo that exists only in fuse_file_info_t.fh - When multiple threads access the same file, we have reference counting to - track how many opens are active. When the last close happens, we need to: - 1. Close the file descriptor - 2. Close the passthrough backing_id (if any) - 3. Delete the FileInfo object - 4. Remove the entry from the open_files map + On non-final closes, secondary handle FileInfo objects are freed + immediately after the ref_count decrement while the canonical map entry stays + alive. - The challenge is that these operations can be slow (especially close()), - and we CANNOT hold the map lock during slow operations. Other threads - may need to open the same file or release other files. - - THE SOLUTION: TWO-PHASE CLEANUP - ------------------------------- - - Phase 1: _release_ref() - Atomic Reference Management (lines ~35-65) - - Runs inside visit() callback (map is locked) - - Decrements ref_count - - If ref_count reaches 0, captures resources and sets fi = nullptr - - This "zombie" state signals that cleanup is needed - - Phase 2: _release_cleanup() - Resource Destruction (lines ~69-88) - - Runs OUTSIDE visit() callback (map is unlocked) - - Performs slow operations: close(), passthrough_close(), delete - - Conditionally erases the map entry using erase_if - - WHY TWO PHASES? - -------------- - - Without this split, a slow close() would block: - - Other threads trying to open the same file - - Other threads trying to release other files (global map lock) - - The entire FUSE request pipeline - - By separating the fast reference counting from the slow cleanup, we achieve - both correctness and high concurrency. - - THE ZOMBIE STATE - ---------------- - - When ref_count hits 0, we set fi = nullptr. This "zombie" state means: - - The entry is marked for destruction - - New opens should skip this entry (see fuse_open.cpp line ~305) - - Cleanup is in progress or pending - - The erase_if() in _release_cleanup() only erases if fi is still nullptr, - protecting against the ABA problem where another thread might have already - created a new entry with the same nodeid. - - ORPHAN FILEINFO CLEANUP - ----------------------- - - The release() function (lines ~116-135) handles the case where a FileInfo - was created but never inserted into the map (e.g., open failure). The - fi_in_map flag distinguishes between: - - FileInfo in map: wait for normal cleanup - - FileInfo not in map (orphan): clean up immediately + On the final close, the map entry is erased atomically and cleanup happens + after the erase_if() callback returns so close() and delete do not execute + while the map bucket is locked. If the final handle is different from the + canonical map-owned FileInfo, both objects must be released. */ #include "fuse_release.hpp" @@ -103,6 +58,9 @@ _release(cu64 nodeid_, FileInfo *fi_, const bool dropcacheonclose_) { + // The double fadvise is necessary insofar as according to nocache + // author (https://github.com/Feh/nocache#limitations) the first one + // doesn't always work due to timing issues. if(dropcacheonclose_) { fs::fadvise_dontneed(fi_->fd); @@ -118,9 +76,10 @@ void FUSE::release(cu64 nodeid_, FileInfo * const fi_) { - auto &of = state.open_files; - int backing_id = INVALID_BACKING_ID; - FileInfo *fi_to_free = nullptr; + auto &of = state.open_files; + int backing_id = INVALID_BACKING_ID; + FileInfo *canonical_fi = nullptr; + FileInfo *fh_fi_to_free = nullptr; of.erase_if(nodeid_, [&](auto &v_) @@ -129,21 +88,28 @@ FUSE::release(cu64 nodeid_, if(v_.second.ref_count > 0) { if(fi_ != v_.second.fi) - fi_to_free = fi_; + fh_fi_to_free = fi_; return false; } - fi_to_free = v_.second.fi; + canonical_fi = v_.second.fi; backing_id = v_.second.backing_id; + if(fi_ != canonical_fi) + fh_fi_to_free = fi_; return true; }); FUSE::passthrough_close(backing_id); - if(fi_to_free) + if(fh_fi_to_free) { - fs::close(fi_to_free->fd); - delete fi_to_free; + fs::close(fh_fi_to_free->fd); + delete fh_fi_to_free; + } + if(canonical_fi) + { + fs::close(canonical_fi->fd); + delete canonical_fi; } } diff --git a/src/fuse_release.hpp b/src/fuse_release.hpp index 572525ff..8e3f90b2 100644 --- a/src/fuse_release.hpp +++ b/src/fuse_release.hpp @@ -36,7 +36,7 @@ The two-phase cleanup mechanism is implemented in fuse_release.cpp. See that file for detailed documentation on the reference counting and - zombie state handling. + canonical vs per-handle ownership rules. See commit 446e9a7b for the complete rationale behind this design. */ diff --git a/src/state.hpp b/src/state.hpp index 12bc8eb8..5f36db48 100644 --- a/src/state.hpp +++ b/src/state.hpp @@ -35,15 +35,13 @@ - backing_id: Kernel handle for FUSE passthrough mode. INVALID_BACKING_ID means passthrough is not enabled for this file. - - fi: Pointer to the FileInfo object containing file metadata and - the actual file descriptor to the **first** opened file. Since a - specific file descriptor has state associated with it you can not - simply reuse the FD for overlapping open requests. Instead it is - reopened using openat(/proc/self/fd/X). Additionally, FUSE - passthrough feature requires that the same underlying file be used - for the nodeid and backing_id. All secondary opens are stored in - the fuse_file_info_t.fh only. When set to nullptr, the entry is in - "zombie" state (being destroyed but not yet removed from the map). + - fi: Pointer to the canonical FileInfo object containing file metadata and + the file descriptor used as the source for overlapping opens. Since a + file descriptor has state associated with it, concurrent opens get their + own duplicated fd and their own FileInfo stored only in + fuse_file_info_t.fh. The map entry keeps the canonical FileInfo so later + opens can duplicate it and so the nodeid can retain a shared + passthrough backing_id. THREAD SAFETY ------------- @@ -65,12 +63,12 @@ An OpenFile entry goes through these states: 1. NOT IN MAP - File is not open - 2. ACTIVE (ref_count >= 1, fi != nullptr) - File is open and in use - 3. ZOMBIE (ref_count = 0, fi = nullptr) - File is being destroyed - 4. CLEANED UP - Resources freed, entry erased from map + 2. ACTIVE (ref_count >= 1) - File is open and the canonical FileInfo is in use + 3. ERASED - The final release removed the map entry and cleanup continues + outside the map callback See fuse_open.cpp and fuse_release.cpp for detailed documentation on the - concurrency patterns and two-phase cleanup mechanism. + concurrency patterns and per-handle cleanup rules. */ #pragma once