mirror of
https://github.com/trapexit/mergerfs.git
synced 2026-10-19 16:16:48 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e87bcbff2 | ||
|
|
fcd7527c9d | ||
|
|
83d1630f55 | ||
|
|
cfb925e5ae | ||
|
|
515b0549ef | ||
|
|
8ea47df13e | ||
|
|
42ab831d97 | ||
|
|
89b4046d11 | ||
|
|
7ec6f8e2f2 | ||
|
|
5f7fa17ad8 | ||
|
|
30d59d4458 | ||
|
|
02dc36080c | ||
|
|
8734847968 | ||
|
|
608cd00764 | ||
|
|
b13727abde | ||
|
|
231aac15cd | ||
|
|
8b17d5f35e | ||
|
|
8608b2be46 | ||
|
|
1e0bd54989 | ||
|
|
56285fc6e5 | ||
|
|
dfe053a0ae | ||
|
|
12ac368e8b | ||
|
|
c8f32f9f25 | ||
|
|
ce3737ff33 | ||
|
|
84e4cdeb7e | ||
|
|
6d6d73c7b3 | ||
|
|
bea31543a0 | ||
|
|
99835a0178 | ||
|
|
342a1efcfb | ||
|
|
c587a127fb | ||
|
|
f1034a3ad6 | ||
|
|
8117998205 | ||
|
|
cc780bc521 | ||
|
|
14b7c8e607 |
@@ -29,6 +29,8 @@ MKDIR ?= mkdir
|
||||
MKTEMP ?= mktemp
|
||||
MV ?= mv
|
||||
RM ?= rm
|
||||
RPMBUILD ?= rpmbuild
|
||||
RPMBUILD_FLAGS ?=
|
||||
SED ?= sed
|
||||
STRIP ?= strip
|
||||
TAR ?= tar
|
||||
@@ -37,7 +39,7 @@ TOUCH ?= touch
|
||||
BUILDDIR := build
|
||||
GITHUB_REPO ?= "https://github.com/trapexit/mergerfs"
|
||||
|
||||
DEFAULT_RELEASE_SAMPLE := debian:13.amd64
|
||||
DEFAULT_TARGET := debian:13.amd64
|
||||
|
||||
ifndef GIT_REPO
|
||||
ifneq ($(shell $(GIT) --version 2> /dev/null),)
|
||||
@@ -48,11 +50,7 @@ endif
|
||||
endif
|
||||
|
||||
USE_XATTR ?= 1
|
||||
UGID_USE_RWLOCK ?= 0
|
||||
|
||||
ifdef NDEBUG
|
||||
OPT_FLAGS := -O2 -DNDEBUG
|
||||
else
|
||||
ifdef SANITIZE
|
||||
ifeq ($(SANITIZE),1)
|
||||
override SANITIZE := -fsanitize=address,undefined,leak
|
||||
@@ -60,14 +58,22 @@ else ifeq ($(filter -fsanitize=%,$(SANITIZE)),)
|
||||
override SANITIZE := -fsanitize=$(SANITIZE)
|
||||
endif
|
||||
endif
|
||||
OPT_FLAGS := -O0 \
|
||||
-g \
|
||||
-fno-omit-frame-pointer \
|
||||
$(SANITIZE) \
|
||||
-fstack-protector-strong \
|
||||
-Wextra \
|
||||
-Wno-unused-parameter \
|
||||
-DDEBUG
|
||||
|
||||
COMMON_OPT_FLAGS += \
|
||||
-fstack-protector-strong \
|
||||
-Wextra \
|
||||
-Wno-unused-parameter
|
||||
|
||||
DEBUG_OPT_FLAGS += \
|
||||
-O0 \
|
||||
-g3 \
|
||||
-fno-omit-frame-pointer \
|
||||
$(SANITIZE)
|
||||
|
||||
ifdef RELEASE
|
||||
OPT_FLAGS := -O3 $(COMMON_OPT_FLAGS)
|
||||
else
|
||||
OPT_FLAGS ?= $(DEBUG_OPT_FLAGS) $(COMMON_OPT_FLAGS)
|
||||
endif
|
||||
|
||||
ifdef STATIC
|
||||
@@ -117,7 +123,7 @@ CXXFLAGS ?= \
|
||||
-Wno-unused-result \
|
||||
-pipe
|
||||
override CXXFLAGS += \
|
||||
-std=c++17 \
|
||||
-std=c++20 \
|
||||
-MMD \
|
||||
-MP
|
||||
override INC_FLAGS := \
|
||||
@@ -125,8 +131,7 @@ override INC_FLAGS := \
|
||||
-Ivendored \
|
||||
-Ivendored/libfuse/include
|
||||
override MFS_FLAGS := \
|
||||
-DUSE_XATTR=$(USE_XATTR) \
|
||||
-DUGID_USE_RWLOCK=$(UGID_USE_RWLOCK)
|
||||
-DUSE_XATTR=$(USE_XATTR)
|
||||
override TESTS_FLAGS := \
|
||||
-Isrc \
|
||||
-Ivendored \
|
||||
@@ -198,7 +203,7 @@ libfuse: $(LIBFUSE)
|
||||
$(MAKE) -C vendored/libfuse
|
||||
|
||||
$(LIBFUSE):
|
||||
$(MAKE) -C vendored/libfuse $(BUILDDIR)/libfuse.a
|
||||
$(MAKE) -C vendored/libfuse $(BUILDDIR)/libfuse.a RELEASE=$(RELEASE)
|
||||
|
||||
tests: $(BUILDDIR)/tests
|
||||
|
||||
@@ -340,7 +345,7 @@ rpm: tarball
|
||||
$(SED) 's/__VERSION__/$(VERSION)/g' mergerfs.spec > \
|
||||
rpmbuild/SOURCES/mergerfs.spec
|
||||
cp -ar mergerfs-$(VERSION).tar.gz rpmbuild/SOURCES
|
||||
rpmbuild -ba rpmbuild/SOURCES/mergerfs.spec \
|
||||
$(RPMBUILD) $(RPMBUILD_FLAGS) -ba rpmbuild/SOURCES/mergerfs.spec \
|
||||
--define "_topdir $(CURDIR)/rpmbuild"
|
||||
|
||||
.PHONY: install-build-pkgs
|
||||
@@ -362,14 +367,16 @@ define build_release
|
||||
--branch=$(GITREF)
|
||||
endef
|
||||
|
||||
.PHONY: release
|
||||
.PHONY: release release-all
|
||||
.PHONY: release-amd64 release-arm64 release-armhf release-riscv64
|
||||
.PHONY: release-sample release-static release-tarball
|
||||
.PHONY: release-sample release-static release-tarball release-installer
|
||||
release:
|
||||
$(MAKE) RELEASE=1 CPPFLAGS="$(CPPFLAGS) -DNDEBUG" all
|
||||
release-all:
|
||||
$(call build_release,"all")
|
||||
release-sample:
|
||||
$(if $(RELEASE_SAMPLE),,$(eval RELEASE_SAMPLE := $(shell ./buildtools/detect-local-target 2>/dev/null)))
|
||||
$(call build_release,$(if $(RELEASE_SAMPLE),$(RELEASE_SAMPLE),$(DEFAULT_RELEASE_SAMPLE)))
|
||||
$(if $(TARGET),,$(eval TARGET := $(shell ./buildtools/detect-local-target 2>/dev/null)))
|
||||
$(call build_release,$(if $(TARGET),$(TARGET),$(DEFAULT_TARGET)))
|
||||
release-amd64:
|
||||
$(call build_release,"amd64")
|
||||
release-arm64:
|
||||
@@ -382,6 +389,8 @@ release-static:
|
||||
$(call build_release,"static")
|
||||
release-tarball:
|
||||
$(call build_release,"tarball")
|
||||
release-installer:
|
||||
$(call build_release,"installer")
|
||||
|
||||
container:
|
||||
$(eval GITREF ?= $(shell git describe --exact-match --tags HEAD 2>/dev/null || git symbolic-ref --short HEAD 2>/dev/null || git rev-parse --short HEAD))
|
||||
|
||||
@@ -1,6 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
if [ -e /tmp/build-env ]; then
|
||||
. /tmp/build-env
|
||||
fi
|
||||
|
||||
rpmbuild_flags()
|
||||
{
|
||||
case "$(command -v "${CXX:-g++}")" in
|
||||
/opt/rh/*)
|
||||
printf '%s' '--undefine _annotated_build'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: $0 <branch> <git_repo>"
|
||||
echo "argv: $@"
|
||||
@@ -28,10 +41,9 @@ mkdir "/build"
|
||||
if [ -e /usr/bin/apt-get ]; then
|
||||
make deb
|
||||
elif [ -e /usr/bin/dnf ]; then
|
||||
make rpm
|
||||
make RPMBUILD_FLAGS="$(rpmbuild_flags)" rpm
|
||||
elif [ -e /usr/bin/yum ]; then
|
||||
. /opt/rh/devtoolset-9/enable
|
||||
make rpm
|
||||
make RPMBUILD_FLAGS="$(rpmbuild_flags)" rpm
|
||||
elif [ -e /sbin/apk ]; then
|
||||
echo "NOT YET SUPPORTED"
|
||||
exit 1
|
||||
|
||||
@@ -18,10 +18,12 @@ def build(git_repo,
|
||||
'--force-rm']
|
||||
if os.path.exists(git_repo):
|
||||
git_repo = os.path.realpath(git_repo)
|
||||
args += ['-v',f'{git_repo}:/tmp/mergerfs.git:ro']
|
||||
args += ['-v',f'{git_repo}:/tmp/mergerfs.git:ro:z']
|
||||
args += ['--build-arg=GIT_REPO=file:///tmp/mergerfs.git']
|
||||
else:
|
||||
args += [f'--build-arg=GIT_REPO={git_repo}']
|
||||
os.makedirs(pkgdirpath,exist_ok=True)
|
||||
args += ['-v',f'{os.path.realpath(pkgdirpath)}:/pkgs:ro']
|
||||
args += ['-o',pkgdirpath,
|
||||
'-f',containerfile,
|
||||
f'--build-arg=BRANCH={branch}',
|
||||
@@ -30,8 +32,6 @@ def build(git_repo,
|
||||
# TODO: Capture output and write to log
|
||||
print(args)
|
||||
rv = subprocess.run(args)
|
||||
|
||||
os.makedirs(pkgdirpath,exist_ok=True)
|
||||
report_filepath = os.path.join(pkgdirpath,"build-report.txt")
|
||||
with open(report_filepath,"a+") as f:
|
||||
build = os.path.basename(containerfile)
|
||||
@@ -66,6 +66,30 @@ def setup():
|
||||
sys.exit(1)
|
||||
print(args)
|
||||
subprocess.run(args)
|
||||
setup_binfmt()
|
||||
|
||||
|
||||
def setup_binfmt():
|
||||
"""Register qemu-user-static binfmt handlers (needed on Arch, etc.)."""
|
||||
binfmt_conf = """\
|
||||
:qemu-aarch64:M::\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\xb7\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:/usr/bin/qemu-aarch64-static:F
|
||||
:qemu-arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:/usr/bin/qemu-arm-static:F
|
||||
:qemu-armeb:M::\\x7fELF\\x01\\x02\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff:/usr/bin/qemu-armeb-static:F
|
||||
:qemu-riscv64:M::\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\xf3\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:/usr/bin/qemu-riscv64-static:F
|
||||
:qemu-i386:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x03\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:/usr/bin/qemu-i386-static:F
|
||||
"""
|
||||
conf_path = '/etc/binfmt.d/qemu-static.conf'
|
||||
print(f"Writing {conf_path}")
|
||||
try:
|
||||
with open(conf_path, 'w') as f:
|
||||
f.write(binfmt_conf)
|
||||
except PermissionError:
|
||||
print(f"Permission denied writing {conf_path}; trying with sudo...")
|
||||
subprocess.run(['sudo', 'tee', conf_path],
|
||||
input=binfmt_conf.encode(), check=True)
|
||||
|
||||
print("Restarting systemd-binfmt...")
|
||||
subprocess.run(['sudo', 'systemctl', 'restart', 'systemd-binfmt'])
|
||||
|
||||
|
||||
def podman_cleanup():
|
||||
@@ -92,7 +116,7 @@ def parse_args():
|
||||
required=False,
|
||||
action='store_true')
|
||||
p.add_argument('--git-repo',
|
||||
required=True,
|
||||
required=False,
|
||||
type=str)
|
||||
|
||||
return p.parse_args()
|
||||
@@ -112,6 +136,10 @@ def main():
|
||||
setup()
|
||||
sys.exit(0)
|
||||
|
||||
if not args.git_repo:
|
||||
print("error: the following arguments are required: --git-repo")
|
||||
sys.exit(1)
|
||||
|
||||
containerfiles = []
|
||||
basepath = 'buildtools/containerfiles'
|
||||
if os.path.exists(f'{basepath}/{args.target}'):
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM --platform=amd64 fedora:44 as build
|
||||
COPY install-build-pkgs build-mergerfs /tmp/
|
||||
RUN /tmp/install-build-pkgs
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
ARG GIT_REPO
|
||||
ARG BRANCH
|
||||
RUN /tmp/build-mergerfs $GIT_REPO $BRANCH
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/ /
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM --platform=arm64 fedora:44 as build
|
||||
COPY install-build-pkgs build-mergerfs /tmp/
|
||||
RUN /tmp/install-build-pkgs
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
ARG GIT_REPO
|
||||
ARG BRANCH
|
||||
RUN /tmp/build-mergerfs $GIT_REPO $BRANCH
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/ /
|
||||
@@ -1,9 +1,13 @@
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
FROM --platform=linux/amd64 ubuntu:latest as build
|
||||
RUN apt-get update && apt-get install -y makeself
|
||||
COPY install-build-pkgs /tmp/
|
||||
RUN /tmp/install-build-pkgs && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -qy --no-install-suggests --no-install-recommends install \
|
||||
makeself
|
||||
RUN mkdir -p /tmp/mergerfs/
|
||||
RUN --mount=type=bind,src=pkgs,dst=/pkgs ls -lhd /pkgs
|
||||
RUN --mount=type=bind,src=pkgs,dst=/pkgs cp -v /pkg/*static*.gz /tmp/mergerfs/
|
||||
RUN ls -lhd /pkgs
|
||||
RUN cp -v /pkgs/*static*.gz /tmp/mergerfs/
|
||||
RUN mkdir -p /build
|
||||
RUN makeself /tmp/mergerfs/ /build/mergerfs-installer.run "mergerfs" echo "foo"
|
||||
|
||||
FROM scratch
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM --platform=amd64 ubuntu:26.04 as build
|
||||
COPY install-build-pkgs build-mergerfs /tmp/
|
||||
RUN /tmp/install-build-pkgs
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
ARG GIT_REPO
|
||||
ARG BRANCH
|
||||
RUN /tmp/build-mergerfs $GIT_REPO $BRANCH
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/ /
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM --platform=arm64 ubuntu:26.04 as build
|
||||
COPY install-build-pkgs build-mergerfs /tmp/
|
||||
RUN /tmp/install-build-pkgs
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
ARG GIT_REPO
|
||||
ARG BRANCH
|
||||
RUN /tmp/build-mergerfs $GIT_REPO $BRANCH
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/ /
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM --platform=armhf ubuntu:26.04 as build
|
||||
COPY install-build-pkgs build-mergerfs /tmp/
|
||||
RUN /tmp/install-build-pkgs
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
ARG GIT_REPO
|
||||
ARG BRANCH
|
||||
RUN /tmp/build-mergerfs $GIT_REPO $BRANCH
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/ /
|
||||
@@ -0,0 +1,10 @@
|
||||
FROM --platform=riscv64 ubuntu:26.04 as build
|
||||
COPY install-build-pkgs build-mergerfs /tmp/
|
||||
RUN /tmp/install-build-pkgs
|
||||
ARG BUILD_TIMESTAMP=0
|
||||
ARG GIT_REPO
|
||||
ARG BRANCH
|
||||
RUN /tmp/build-mergerfs $GIT_REPO $BRANCH
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /build/ /
|
||||
@@ -1,39 +1,162 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
BUILD_ENV=/tmp/build-env
|
||||
|
||||
write_gcc_env()
|
||||
{
|
||||
{
|
||||
printf 'export CC=%s\n' "$1"
|
||||
printf 'export CXX=%s\n' "$2"
|
||||
} > "$BUILD_ENV"
|
||||
}
|
||||
|
||||
write_toolset_env()
|
||||
{
|
||||
{
|
||||
printf '. %s\n' "$1"
|
||||
printf 'export CC=gcc\n'
|
||||
printf 'export CXX=g++\n'
|
||||
} > "$BUILD_ENV"
|
||||
}
|
||||
|
||||
latest_apt_gxx()
|
||||
{
|
||||
apt-cache --names-only search '^g[+][+]-[0-9]' \
|
||||
| sed -n 's/^\(g++-[0-9][0-9]*\) .*/\1/p' \
|
||||
| sort -t- -k2,2V \
|
||||
| tail -n 1
|
||||
}
|
||||
|
||||
latest_dnf_toolset()
|
||||
{
|
||||
dnf -q list --showduplicates 'gcc-toolset-*-gcc-c++' 2>/dev/null \
|
||||
| sed -n 's/^gcc-toolset-\([0-9][0-9]*\)-gcc-c++\..*/\1/p' \
|
||||
| sort -V \
|
||||
| tail -n 1
|
||||
}
|
||||
|
||||
latest_yum_toolset()
|
||||
{
|
||||
yum -q list available 'devtoolset-*-gcc-c++' 2>/dev/null \
|
||||
| sed -n 's/^devtoolset-\([0-9][0-9]*\)-gcc-c++\..*/\1/p' \
|
||||
| sort -V \
|
||||
| tail -n 1
|
||||
}
|
||||
|
||||
version_gt()
|
||||
{
|
||||
highest="$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n 1)"
|
||||
[ "$highest" = "$1" ] && [ "$1" != "$2" ]
|
||||
}
|
||||
|
||||
check_cxx20()
|
||||
{
|
||||
cxx="${CXX:-g++}"
|
||||
test_src=/tmp/mergerfs-cxx20-check.cpp
|
||||
test_bin=/tmp/mergerfs-cxx20-check
|
||||
|
||||
cat > "$test_src" <<'EOF'
|
||||
template <typename T>
|
||||
concept Incrementable = requires(T value) { value + 1; };
|
||||
|
||||
static_assert(Incrementable<int>);
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
EOF
|
||||
|
||||
if ! "$cxx" -std=c++20 "$test_src" -o "$test_bin" >/dev/null 2>&1; then
|
||||
echo "error: $cxx does not support C++20" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f "$test_src" "$test_bin"
|
||||
}
|
||||
|
||||
: > "$BUILD_ENV"
|
||||
|
||||
if [ -e /usr/bin/apt-get ]; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get -qy update
|
||||
cxx_pkg="$(latest_apt_gxx)"
|
||||
if [ -n "$cxx_pkg" ]; then
|
||||
gcc_pkg="gcc-${cxx_pkg#g++-}"
|
||||
compiler_pkgs="$gcc_pkg $cxx_pkg"
|
||||
write_gcc_env "$gcc_pkg" "$cxx_pkg"
|
||||
else
|
||||
compiler_pkgs="g++"
|
||||
write_gcc_env gcc g++
|
||||
fi
|
||||
apt-get -qy --force-yes --no-install-suggests --no-install-recommends \
|
||||
install \
|
||||
ca-certificates \
|
||||
build-essential \
|
||||
dpkg-dev \
|
||||
git \
|
||||
g++ \
|
||||
$compiler_pkgs \
|
||||
debhelper \
|
||||
lsb-release \
|
||||
fakeroot
|
||||
. "$BUILD_ENV"
|
||||
check_cxx20
|
||||
elif [ -e /usr/bin/dnf ]; then
|
||||
dnf -y update
|
||||
toolset="$(latest_dnf_toolset)"
|
||||
dnf -y install \
|
||||
git rpm-build gcc-c++ make which libatomic
|
||||
default_gxx_major="$(g++ -dumpversion | cut -f1 -d.)"
|
||||
if [ -n "$toolset" ] && version_gt "$toolset" "$default_gxx_major"; then
|
||||
dnf -y install \
|
||||
"gcc-toolset-${toolset}-gcc" \
|
||||
"gcc-toolset-${toolset}-gcc-c++"
|
||||
write_toolset_env "/opt/rh/gcc-toolset-${toolset}/enable"
|
||||
else
|
||||
write_gcc_env gcc g++
|
||||
fi
|
||||
. "$BUILD_ENV"
|
||||
check_cxx20
|
||||
elif [ -e /usr/bin/yum ]; then
|
||||
yum -y update
|
||||
yum -y install \
|
||||
git rpm-build gcc-c++ make which
|
||||
yum -y install centos-release-scl
|
||||
yum -y install devtoolset-9-gcc\*
|
||||
yum -y install centos-release-scl || true
|
||||
yum -y update
|
||||
toolset="$(latest_yum_toolset)"
|
||||
default_gxx_major="$(g++ -dumpversion | cut -f1 -d.)"
|
||||
if [ -n "$toolset" ] && version_gt "$toolset" "$default_gxx_major"; then
|
||||
yum -y install \
|
||||
"devtoolset-${toolset}-gcc" \
|
||||
"devtoolset-${toolset}-gcc-c++"
|
||||
write_toolset_env "/opt/rh/devtoolset-${toolset}/enable"
|
||||
else
|
||||
write_gcc_env gcc g++
|
||||
fi
|
||||
. "$BUILD_ENV"
|
||||
check_cxx20
|
||||
elif [ -e /usr/bin/zypper ]; then
|
||||
zypper update -y
|
||||
zypper install -y \
|
||||
git rpm-build gcc-c++ make which
|
||||
write_gcc_env gcc g++
|
||||
. "$BUILD_ENV"
|
||||
check_cxx20
|
||||
elif [ -e /sbin/apk ]; then
|
||||
apk add \
|
||||
abuild git gcc g++ make \
|
||||
linux-headers
|
||||
write_gcc_env gcc g++
|
||||
. "$BUILD_ENV"
|
||||
check_cxx20
|
||||
elif [ -e /usr/sbin/pkg ]; then
|
||||
pkg install \
|
||||
git gmake gcc
|
||||
elif [ -e /usr/bin/pacman ]; then
|
||||
pacman -Sy --noconfirm base-devel git fakeroot
|
||||
write_gcc_env gcc g++
|
||||
. "$BUILD_ENV"
|
||||
check_cxx20
|
||||
fi
|
||||
|
||||
Vendored
+4
-1
@@ -6,8 +6,11 @@ export DH_VERBOSE=1
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_auto_build:
|
||||
$(MAKE) release
|
||||
|
||||
override_dh_auto_install:
|
||||
$(MAKE) \
|
||||
DESTDIR=$(CURDIR)/debian/mergerfs \
|
||||
PREFIX=/usr \
|
||||
install
|
||||
install
|
||||
@@ -38,6 +38,32 @@ These files and xattrs are used to help `mergerfs` know whether or not
|
||||
a directory happens to be a branch or is the underlying directory
|
||||
mount point.
|
||||
|
||||
## Paths which appear after startup
|
||||
|
||||
The [`branches` resolution process](branches.md#resolution) runs before
|
||||
this option starts waiting. A missing literal path is retained as a
|
||||
branch, so a configuration such as `/mnt/hdd/disk0/foobar` can work
|
||||
when `/mnt/hdd/disk0` is mounted asynchronously and `foobar` only
|
||||
becomes visible afterward, provided the path then meets one of the
|
||||
"mounted" conditions above.
|
||||
|
||||
For each branch which is not ready, mergerfs first runs `mount` once
|
||||
with the full branch path as the target and then polls that same path
|
||||
until the timeout expires. It does not create the path, try to mount
|
||||
its parent, or rerun `mount`. In the example above, the initial
|
||||
`mount /mnt/hdd/disk0/foobar` may fail while the path is absent. If
|
||||
another service mounts `/mnt/hdd/disk0` and
|
||||
`/mnt/hdd/disk0/foobar` appears and meets one of the "mounted"
|
||||
conditions before the deadline, mergerfs detects it during polling
|
||||
and continues.
|
||||
|
||||
Globs are different. If `/mnt/hdd/disk0/*` has no matches during
|
||||
branch resolution, the literal path `/mnt/hdd/disk0/*` is retained.
|
||||
The mount wait does not expand it again after `/mnt/hdd/disk0`
|
||||
becomes available. Ensure the parent filesystem is mounted before
|
||||
mergerfs, configure a literal branch path, or update `branches`
|
||||
through the runtime API once the paths exist.
|
||||
|
||||
|
||||
**NOTE:** If on a `systemd` based system and using `fstab` it is a
|
||||
good idea to set the mount option
|
||||
|
||||
@@ -14,7 +14,7 @@ Branches currently have two options which can be set. A
|
||||
included in a policy calculation and a individual
|
||||
[minfreespace](#minfreespace) value. The values are set by
|
||||
prepending an `=` at the end of a branch designation and using commas
|
||||
as delimiters. Example: `/mnt/drive=RW,1234`
|
||||
as delimiters. Example: `/mnt/hdd/disk0=RW,1234`
|
||||
|
||||
|
||||
### branch mode
|
||||
@@ -34,6 +34,33 @@ Same purpose and syntax as the [global option](minfreespace.md) but
|
||||
specific to the branch. Defaults to the global value.
|
||||
|
||||
|
||||
### resolution
|
||||
|
||||
Each colon-delimited branch designation is resolved independently when
|
||||
the `branches` option is parsed, both at startup and when it is updated
|
||||
through the runtime API:
|
||||
|
||||
1. The path portion is expanded as a glob.
|
||||
2. If the glob matches one or more directories, each match becomes a
|
||||
branch.
|
||||
3. If the glob has no matches, mergerfs logs a notice and keeps the
|
||||
configured path unchanged as a single, literal branch.
|
||||
4. mergerfs resolves each resulting path with `realpath`. If that
|
||||
succeeds, the canonical path is stored. If it fails, typically
|
||||
because the path does not exist yet, the unchanged path is stored.
|
||||
5. A path which exists but is not a directory is logged and skipped. A
|
||||
path which does not exist is logged but retained.
|
||||
|
||||
Consequently, a literal branch such as `/mnt/hdd/disk0/foobar` may be
|
||||
configured before it exists and can become usable after
|
||||
`/mnt/hdd/disk0` is mounted. Branch resolution is not repeated in the
|
||||
background, however. If `/mnt/hdd/disk0/*` has no matches when it is
|
||||
resolved, mergerfs stores the literal path `/mnt/hdd/disk0/*`;
|
||||
directories which match it later are not discovered automatically.
|
||||
Update `branches` through the runtime API or restart mergerfs to
|
||||
resolve the glob again.
|
||||
|
||||
|
||||
### globbing
|
||||
|
||||
To make it easier to include multiple branches mergerfs supports
|
||||
@@ -42,18 +69,18 @@ MUST be escaped when using via the shell else the shell itself will
|
||||
apply the glob itself.**
|
||||
|
||||
```
|
||||
# mergerfs /mnt/hdd\*:/mnt/ssd /media
|
||||
# mergerfs /mnt/hdd/\*:/mnt/ssd/\* /media
|
||||
```
|
||||
|
||||
The above line will use all directories in /mnt prefixed with **hdd**
|
||||
as well as **ssd**.
|
||||
The above line will use all directories within `/mnt/hdd` and
|
||||
`/mnt/ssd`.
|
||||
|
||||
To have the pool mounted at boot or otherwise accessible from related
|
||||
tools use `/etc/fstab`.
|
||||
|
||||
```
|
||||
# <file system> <mount point> <type> <options> <dump> <pass>
|
||||
/mnt/hdd*:/mnt/ssd /media mergerfs minfreespace=16G 0 0
|
||||
/mnt/hdd/*:/mnt/ssd/* /media mergerfs minfreespace=16G 0 0
|
||||
```
|
||||
|
||||
**NOTE:** The globbing is done at mount or when updated using the
|
||||
|
||||
@@ -15,6 +15,8 @@ function or have been replaced. **They should not be used.**
|
||||
* **hard_remove**: Does nothing.
|
||||
* **kernel_cache**: Use [cache.files=full](cache.md).
|
||||
* **negative_entry**: Use [cache.negative-entry](cache.md).
|
||||
* **remember**: Use `never-forget-nodes`.
|
||||
* **remember-nodes**: Use `never-forget-nodes`.
|
||||
* **noforget**: Use `never-forget-nodes`.
|
||||
* **nonempty**: Does nothing.
|
||||
* **splice_move**: Does nothing.
|
||||
|
||||
@@ -213,12 +213,11 @@ config file.
|
||||
* **passthrough.max-stack-depth=INT**: Set to `1` another filesystem
|
||||
can be stacked on mergerfs. Set to `2` to have mergerfs stacked over
|
||||
another filesystem. (default: 1)
|
||||
* **remember-nodes=INT**: The number of seconds to keep the internal
|
||||
representation of a file once the OS tells mergerfs it is no longer
|
||||
needed. Really only needed for [exporting mergerfs via
|
||||
NFS](../remote_filesystems.md) (default: 0)
|
||||
* **never-forget-nodes=BOOL**: Sets `remember-nodes` to
|
||||
infinity. Formerly `noforget`. (default: false)
|
||||
* **never-forget-nodes=BOOL**: Keep the internal representation of
|
||||
a file after the OS tells mergerfs it is no longer needed, until the
|
||||
node is unlinked, removed, or replaced. Really only needed for
|
||||
[exporting mergerfs via NFS](../remote_filesystems.md). Formerly
|
||||
`noforget`. (default: false)
|
||||
* **allow-idmap=BOOL**: Enables idmap (identity mapping). (default:
|
||||
true)
|
||||
* **debug**: Enable debugging information. At the moment limited to a
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
If using path preservation and a `rename` fails with `EXDEV`:
|
||||
|
||||
1. Move file from `/branch/a/b/c` to `/branch/.mergerfs_rename_exdev/a/b/c`.
|
||||
1. Move file from `/mnt/hdd/disk0/a/b/c` to
|
||||
`/mnt/hdd/disk0/.mergerfs_rename_exdev/a/b/c`.
|
||||
2. symlink the rename's `newpath` to the moved file.
|
||||
|
||||
The `target` value is determined by the value of `rename-exdev`.
|
||||
|
||||
@@ -149,6 +149,70 @@ It is generally recommended to not use user namespacing / id mapping
|
||||
given the complication it introduces.
|
||||
|
||||
|
||||
## Does mergerfs work with SELinux relabeling?
|
||||
|
||||
FUSE filesystems, including mergerfs, do not support per-inode
|
||||
SELinux labels (see [Red Hat bug
|
||||
1272868](https://bugzilla.redhat.com/show_bug.cgi?id=1272868) and the
|
||||
[upstream fuse-devel
|
||||
thread](https://sourceforge.net/p/fuse/mailman/fuse-devel/thread/CAN9vWDLGA4%3Dg3peKVWkX7gLRiVUUvJAQx9kiXHgqdeX6eh0MYA%40mail.gmail.com/)).
|
||||
When a container runtime such as Podman or Docker is asked to relabel
|
||||
a bind mount with the `:z` or `:Z` mount option, it first probes the
|
||||
source for `security.selinux` xattr support. On a FUSE path that
|
||||
probe returns `EOPNOTSUPP` and the runtime skips relabeling entirely.
|
||||
Files inside the mount keep the `fusefs_t` type assigned by the
|
||||
kernel. No tree walk is performed and no journal noise is produced at
|
||||
default verbosity. The runtime logs `Labeling not supported on
|
||||
"<path>"` only at debug level (for example
|
||||
`podman --log-level=debug`). The flag is therefore a no-op against a
|
||||
mergerfs mount; not harmful, but misleading.
|
||||
|
||||
Containers can still read and write a mergerfs pool without `:z`
|
||||
because two independent access checks must succeed and both can be
|
||||
satisfied without any relabel:
|
||||
|
||||
* **SELinux:** on Fedora, RHEL / CentOS Stream, and SUSE the shipped
|
||||
[container-selinux](https://github.com/containers/container-selinux)
|
||||
policy already permits the `container_t` process type to manage
|
||||
files of type `fusefs_t`. See `container_selinux(8)`.
|
||||
* **DAC:** the in-container UID and GID must still be able to read
|
||||
and write the files on the underlying branches. With rootless
|
||||
Podman this is most easily arranged by aligning the container UID
|
||||
with the host owner of the branches (for example
|
||||
`--userns=keep-id`, or Quadlet `UserNS=keep-id`).
|
||||
|
||||
The recommendation is to omit `:z` and `:Z` on bind mounts whose
|
||||
source is a mergerfs (or any other FUSE) path, and to keep them on
|
||||
bind mounts whose source is a regular local filesystem (ext4, btrfs,
|
||||
xfs, etc.) where relabeling does real work. On distributions whose
|
||||
container-selinux policy does not already include the `fusefs_t`
|
||||
allow rule, the targeted fix is to set a single label at mount time
|
||||
by adding `context="system_u:object_r:container_file_t:s0"` to the
|
||||
mergerfs mount options (for example in `/etc/fstab`). Files inside
|
||||
the pool then appear as `container_file_t`, which container-selinux
|
||||
permits `container_t` to manage everywhere. See [Red Hat's mounting
|
||||
file systems
|
||||
guide](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/selinux_users_and_administrators_guide/sect-security-enhanced_linux-working_with_selinux-mounting_file_systems).
|
||||
This is preferred over the much broader
|
||||
`--security-opt label=disable`, which removes SELinux confinement
|
||||
from the container as a whole. The trade-off of `context=` is that
|
||||
host-side processes touching the mount (rsync, backup tools, web
|
||||
UIs, cron jobs) see `container_file_t` rather than `fusefs_t` and
|
||||
may require a small SELinux policy adjustment.
|
||||
|
||||
To verify on your own system:
|
||||
|
||||
```
|
||||
podman --log-level=debug run --rm \
|
||||
-v /path/to/mergerfs:/data:z \
|
||||
alpine true 2>&1 | grep -iE 'relabel|setfilecon|Labeling'
|
||||
```
|
||||
|
||||
Expected output is a single line of the form
|
||||
`Labeling not supported on "/path/to/mergerfs"` and nothing in
|
||||
`ausearch -m AVC,SELINUX_ERR` for the mount.
|
||||
|
||||
|
||||
## Can mergerfs be used with NFS root squash?
|
||||
|
||||
If mergerfs is pooling a NFS mount then root squash must be disabled
|
||||
|
||||
@@ -18,6 +18,12 @@ mergerfs [does not impact](usage_and_functionality.md) the underlying
|
||||
filesystems and can be added or removed without any impact it is
|
||||
extremely easy to test and experiment with different settings.
|
||||
|
||||
[mergerfs-webui](../tooling.md#mergerfs-webui) provides a browser-based
|
||||
way to inspect and change settings, branches, and policies on Linux,
|
||||
and edit startup configuration. It does not replace understanding
|
||||
what the options do or choosing policies suited to your use case.
|
||||
Runtime changes alone do not persist across remounts.
|
||||
|
||||
Additional reading:
|
||||
|
||||
* [Config/Options](../config/options.md)
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
# Downsides
|
||||
|
||||
## What are the downsides of using mergerfs?
|
||||
|
||||
mergerfs trades the features of an integrated storage system for the
|
||||
flexibility of pooling independent filesystems. It is a filesystem
|
||||
proxy, not RAID or a replacement for the underlying filesystems.
|
||||
|
||||
Some limitations are inherent to that design; others matter only for
|
||||
particular workloads, configurations, or applications. Many can be
|
||||
mitigated with appropriate settings or separate tools. The following
|
||||
is a checklist of tradeoffs to evaluate, not a list of problems every
|
||||
installation will experience.
|
||||
|
||||
### No built-in data protection or storage management
|
||||
|
||||
mergerfs does not provide redundancy, parity, integrity checks,
|
||||
snapshots, or file versioning. If a drive fails, mergerfs cannot
|
||||
recover the files on it. The other drives retain their independent
|
||||
filesystems, but that failure isolation is not redundancy or a backup.
|
||||
Protection must come from the underlying storage or separate tools,
|
||||
such as [SnapRAID](https://www.snapraid.it) or
|
||||
[NonRAID](https://github.com/qvr/nonraid).
|
||||
|
||||
It also does not automatically rebalance existing files when a branch
|
||||
is added or space usage changes. If redistribution is needed, it must
|
||||
be done separately.
|
||||
|
||||
See [Non-features](../index.md#non-features), [Project
|
||||
Comparisons](../project_comparisons.md), and [How can I balance files
|
||||
across the pool?](configuration_and_policies.md#how-can-i-balance-files-across-the-pool).
|
||||
|
||||
### No striping or contiguous pooled space
|
||||
|
||||
Each file lives entirely on one branch. Adding drives does not make a
|
||||
single-file transfer faster, and a file must fit on one eligible
|
||||
branch even if the pool reports enough total free space. For example,
|
||||
two filesystems with 6GiB free each cannot hold one 10GiB file.
|
||||
Concurrent access to different branches can still provide aggregate
|
||||
throughput.
|
||||
|
||||
See [Project Comparisons](../project_comparisons.md#raid0-jbod-span-drive-concatenation-striping)
|
||||
and [Performance](../performance.md#understanding-mergerfs-io-performance).
|
||||
|
||||
### Performance overhead
|
||||
|
||||
Filesystem requests generally pass through FUSE and the mergerfs
|
||||
process before reaching the underlying filesystem. This adds latency
|
||||
and CPU overhead compared to direct access. Directory listings and
|
||||
other metadata operations can require work across multiple branches,
|
||||
so their cost can grow as the pool grows. Page caching can also result
|
||||
in data being cached both through mergerfs and on the underlying
|
||||
filesystem.
|
||||
|
||||
[IO passthrough](../config/passthrough.md) can reduce data IO overhead,
|
||||
but does not eliminate the work of combining branches. Databases, VM
|
||||
images, and other latency-sensitive workloads should be benchmarked
|
||||
with the intended configuration rather than assumed to perform like
|
||||
direct access.
|
||||
|
||||
There is also memory overhead beyond page caching: request buffers,
|
||||
temporary directory buffers, and FUSE node bookkeeping. Usage depends
|
||||
on concurrency, buffer sizes, and the workload. Exporting the pool
|
||||
over NFS requires retaining nodes with `never-forget-nodes=true`,
|
||||
which can increase memory usage as more entries are accessed. This is
|
||||
worth considering on memory-constrained systems or large NFS exports.
|
||||
See [Resource Usage](../resource_usage.md) and [NFS exporting
|
||||
mergerfs](../remote_filesystems.md#nfs-exporting-mergerfs).
|
||||
|
||||
See [Understanding mergerfs IO
|
||||
Performance](../performance.md#understanding-mergerfs-io-performance),
|
||||
[Benchmarking](../benchmarking.md), and [Why use
|
||||
FUSE?](technical_behavior_and_limitations.md#why-use-fuse-why-not-a-kernel-based-solution).
|
||||
|
||||
### Placement and filesystem semantics need attention
|
||||
|
||||
A single mount point does not make the branches one filesystem.
|
||||
[Policies](../config/functions_categories_policies.md) determine where
|
||||
new files go and which copy is used when the same path exists on
|
||||
multiple branches. Path-preserving policies and
|
||||
[minfreespace](../config/minfreespace.md) can exclude branches from file
|
||||
creation even while the pool has free space. mergerfs cannot know a
|
||||
file's final size when it is created.
|
||||
|
||||
Hard links and renames still have underlying filesystem constraints.
|
||||
Depending on the layout and policies, an operation can return `EXDEV`
|
||||
(cross-device link) even within the pool. Applications which do not
|
||||
handle that error may fail rather than fall back to copying.
|
||||
|
||||
Directory metadata is also selected by policy rather than necessarily
|
||||
aggregated. With the default `getattr` policy of `ff`, a directory's
|
||||
timestamp comes from the first matching branch. A media scanner which
|
||||
uses that timestamp to skip unchanged directories can miss updates on
|
||||
another branch. `func.getattr=newest` can help with timestamp-based
|
||||
scans, but changes which branch supplies metadata; it does not change
|
||||
which file other operations select.
|
||||
|
||||
Corresponding directories on different branches can have different
|
||||
ownership or permissions. Since access uses the caller's credentials,
|
||||
this can result in unexpected access errors or missing entries.
|
||||
Keeping those permissions consistent and auditing with
|
||||
`fsck.mergerfs` helps. Container UID/GID mappings and non-POSIX
|
||||
branches may need additional attention; mergerfs does not bypass
|
||||
their access restrictions.
|
||||
|
||||
See [Configuration and Policies](configuration_and_policies.md),
|
||||
[rename and link](../config/rename_and_link.md), [Tips and
|
||||
Notes](../tips_notes.md), and [How does mergerfs interact with user
|
||||
namespaces?](compatibility_and_integration.md#how-does-mergerfs-interact-with-user-namespaces).
|
||||
|
||||
### Keeping drives spun down can be harder
|
||||
|
||||
Directory listings and file searches can query multiple branches,
|
||||
potentially waking drives beyond the one containing the requested
|
||||
file. If keeping disks asleep is important, this can increase power
|
||||
usage, noise, and access latency. It is not a concern for SSDs or
|
||||
drives which are kept spinning.
|
||||
|
||||
Caching and reducing background scans can limit some branch access,
|
||||
but mergerfs alone cannot guarantee that unused disks stay asleep.
|
||||
See [Limiting drive spinup](limit_drive_spinup.md) for the tradeoffs
|
||||
and possible approaches.
|
||||
|
||||
### Direct branch changes and cached views
|
||||
|
||||
Using the underlying filesystems directly is supported, but changes
|
||||
made outside the mergerfs mount are not forwarded as `inotify` or
|
||||
`fanotify` events to applications watching the pool. Changes made
|
||||
through the pool do generate notifications. If files are added
|
||||
directly to branches, applications such as media servers may need
|
||||
periodic scans, explicit refreshes, or watches on the branches
|
||||
themselves.
|
||||
|
||||
Metadata caching introduces a separate visibility tradeoff: direct
|
||||
branch changes may not appear through the pool until cached entries
|
||||
expire. Longer cache durations can reduce filesystem queries at the
|
||||
cost of less immediate visibility. Shorter durations or disabling the
|
||||
relevant caches can help if direct branch changes are common. Avoid
|
||||
uncoordinated writes to the same file through both paths, especially
|
||||
when file caching is enabled.
|
||||
|
||||
See [Does inotify and fanotify
|
||||
work?](compatibility_and_integration.md#does-inotify-and-fanotify-work),
|
||||
[Can filesystems still be used
|
||||
directly?](usage_and_functionality.md#can-filesystems-still-be-used-directly-outside-of-mergerfs-while-pooled),
|
||||
and [Caching](../config/cache.md).
|
||||
|
||||
### Branch paths do not guarantee a particular disk is mounted
|
||||
|
||||
mergerfs works on paths and does not require each branch to be a
|
||||
mount point. If an intended branch filesystem is not mounted but its
|
||||
mount-point directory still exists, mergerfs can use that directory
|
||||
on the filesystem beneath it. If writable and selected for creation,
|
||||
new files can land there instead of on the intended disk, potentially
|
||||
filling the root filesystem.
|
||||
|
||||
This is a deployment concern when pooling mounted filesystems, not a
|
||||
problem with intentionally using ordinary directories as branches.
|
||||
Configure mount dependencies or
|
||||
`branches-mount-timeout` with `branches-mount-timeout-fail=true` if
|
||||
startup should fail when branches are not ready. These are startup
|
||||
safeguards, not continuous checks that the intended disks remain
|
||||
mounted at runtime.
|
||||
|
||||
See [branches-mount-timeout](../config/branches-mount-timeout.md) and
|
||||
[What happens if a filesystem disappears at
|
||||
runtime?](usage_and_functionality.md#what-happens-if-a-filesystem-disappears-at-runtime).
|
||||
|
||||
### Dependence on branch responsiveness
|
||||
|
||||
mergerfs cannot make an unresponsive disk or remote filesystem
|
||||
responsive. A request to a blocked branch blocks the thread handling
|
||||
it. If enough threads are blocked, the pool can stop responding as
|
||||
well. Independent on-disk filesystems do not guarantee that a failing
|
||||
branch has no effect on access through the pool.
|
||||
|
||||
See [What happens if a branch filesystem
|
||||
blocks?](technical_behavior_and_limitations.md#what-happens-if-a-branch-filesystem-blocks).
|
||||
|
||||
### Compatibility limitations
|
||||
|
||||
Some filesystem features are unavailable through FUSE or require
|
||||
particular kernel versions and settings. For example, reflink requests
|
||||
(`FICLONE` / `FICLONERANGE`) are not supported through the mergerfs
|
||||
mount. Applications using `mmap` need page caching; older Linux or
|
||||
mergerfs versions require it to be explicitly enabled.
|
||||
|
||||
Advisory locks taken through the pool are not forwarded to the
|
||||
underlying branches. Local applications using only the mergerfs mount
|
||||
still have kernel-managed locks, but those locks do not coordinate
|
||||
with direct branch access or other hosts accessing a remote branch.
|
||||
|
||||
Inode identity also has tradeoffs when presenting multiple filesystems
|
||||
as one. Most applications do not need special settings, but backup or
|
||||
deduplication tools and NFS can be sensitive to inode values changing
|
||||
when files move between branches or a policy selects a different
|
||||
copy. [inodecalc](../config/inodecalc.md) offers different strategies:
|
||||
for example, `path-hash` stabilizes identity by path for NFS, but
|
||||
prevents applications from recognizing hard links by inode equality.
|
||||
The hard links themselves still work.
|
||||
|
||||
On SELinux systems, per-file relabeling is unavailable through the
|
||||
FUSE mount, so container bind-mount options `:z` and `:Z` do not
|
||||
perform their usual relabeling. This does not mean containers cannot
|
||||
use the pool: an appropriate SELinux policy or a mount-wide context
|
||||
can allow access. A mount-wide context is not a substitute for
|
||||
distinct per-file labels if the application requires them. See [Does
|
||||
mergerfs work with SELinux
|
||||
relabeling?](compatibility_and_integration.md#does-mergerfs-work-with-selinux-relabeling).
|
||||
|
||||
See [Technical Behavior and
|
||||
Limitations](technical_behavior_and_limitations.md) and [Known Issues
|
||||
and Bugs](../known_issues_bugs.md).
|
||||
|
||||
These tradeoffs do not require migrating data into a special format.
|
||||
The underlying filesystems remain usable without mergerfs, and
|
||||
mergerfs can be [removed without changing their
|
||||
data](usage_and_functionality.md#can-mergerfs-be-removed-without-affecting-the-data).
|
||||
@@ -81,7 +81,7 @@ where it was created.
|
||||
$ sudo touch /mnt/mergerfs/new-file
|
||||
$ sudo getfattr -n user.mergerfs.allpaths /mnt/mergerfs/new-file
|
||||
# file: mnt/mergerfs/new-file
|
||||
user.mergerfs.allpaths="/mnt/hdd/drive1/new-file"
|
||||
user.mergerfs.allpaths="/mnt/hdd/disk0/new-file"
|
||||
```
|
||||
|
||||
If the location of the file is where it should be according to the
|
||||
@@ -119,7 +119,7 @@ may be able to read some files and directories on one filesystem but
|
||||
not another resulting in an incomplete set. And if one of the branches
|
||||
it can access is empty then it will return an empty list.
|
||||
|
||||
Try using [mergerfs.fsck](https://github.com/trapexit/mergerfs-tools)
|
||||
Try using [fsck.mergerfs](../tooling.md#fsckmergerfs)
|
||||
tool to check for and fix inconsistencies in permissions. If you
|
||||
aren't seeing anything at all be sure that the basic permissions are
|
||||
correct. The user and group values are correct and that directories
|
||||
|
||||
@@ -105,7 +105,7 @@ The way in which mergerfs behaves is controlled by the
|
||||
|
||||
```
|
||||
A + B = C
|
||||
/disk1 /disk2 /merged
|
||||
/mnt/hdd/disk0 /mnt/hdd/disk1 /media
|
||||
| | |
|
||||
+-- /dir1 +-- /dir1 +-- /dir1
|
||||
| | | | | |
|
||||
|
||||
@@ -2,6 +2,30 @@
|
||||
|
||||
## mergerfs
|
||||
|
||||
### Advisory file locking not currently supported
|
||||
|
||||
mergerfs does not implement POSIX or BSD advisory file locks as
|
||||
provided by FUSE. Doing so is complex because, as a union filesystem,
|
||||
it would need to forward lock requests to the underlying branches,
|
||||
which could result in blocking calls. Locally this is not a major
|
||||
concern—the kernel still manages locks for applications that all go
|
||||
through the mergerfs mount. However, if an application locks a file
|
||||
via the mergerfs mount while another locks the same file directly on
|
||||
the underlying filesystem, the locks will not interact as one might
|
||||
expect. That usage pattern is uncommon, so it is generally not a
|
||||
problem.
|
||||
|
||||
The issue becomes more significant if you rely on locks over a remote
|
||||
filesystem like NFS across multiple hosts, because mergerfs never
|
||||
actually makes a lock call to the underlying branch. While network
|
||||
locks are possible, they are generally discouraged, so this may not be
|
||||
a huge loss either.
|
||||
|
||||
That said, there is a strategy that could add lock support if there is
|
||||
sufficient demand, though it would probably be opt-in for performance
|
||||
and resource-usage reasons.
|
||||
|
||||
|
||||
### FreeBSD version
|
||||
|
||||
* [https://wiki.freebsd.org/FUSEFS](https://wiki.freebsd.org/FUSEFS)
|
||||
@@ -27,7 +51,7 @@
|
||||
* ...
|
||||
|
||||
|
||||
#### Host vs Container identity
|
||||
### Host vs Container identity
|
||||
|
||||
While **not** a bug some users have found when using containers that
|
||||
supplemental groups defined inside the container don't work as
|
||||
@@ -68,15 +92,15 @@ If you always want the directory information from the one with the
|
||||
most recent mtime then use the `newest` policy for `getattr`.
|
||||
|
||||
|
||||
### 'mv /mnt/pool/foo /mnt/disk1/foo' removes 'foo'
|
||||
### 'mv /mnt/pool/foo /mnt/hdd/disk0/foo' removes 'foo'
|
||||
|
||||
This is not a bug.
|
||||
|
||||
Run in verbose mode to better understand what's happening:
|
||||
|
||||
```
|
||||
$ mv -v /mnt/pool/foo /mnt/disk1/foo
|
||||
copied '/mnt/pool/foo' -> '/mnt/disk1/foo'
|
||||
$ mv -v /mnt/pool/foo /mnt/hdd/disk0/foo
|
||||
copied '/mnt/pool/foo' -> '/mnt/hdd/disk0/foo'
|
||||
removed '/mnt/pool/foo'
|
||||
$ ls /mnt/pool/foo
|
||||
ls: cannot access '/mnt/pool/foo': No such file or directory
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
# Migration
|
||||
|
||||
## Unraid
|
||||
|
||||
mergerfs and a parity-protected Unraid array can both present
|
||||
directories from multiple filesystems as a single view, but they treat
|
||||
storage membership very differently. With mergerfs, membership is a
|
||||
property of its **configuration**: branches are paths and can be added
|
||||
or removed at runtime without modifying the underlying filesystems.
|
||||
With Unraid's main array, membership is part of the **array state**:
|
||||
adding a data disk to an array with active parity normally requires
|
||||
clearing and formatting it, and removing one normally requires parity
|
||||
to be rebuilt.
|
||||
|
||||
Those array-level requirements, not the User Share filesystem itself,
|
||||
are what shape migrations in either direction. When moving to mergerfs,
|
||||
existing data filesystems can usually be reused as-is. When moving to
|
||||
Unraid, populated filesystems can normally only be brought into the
|
||||
array before parity is activated. See [Project
|
||||
Comparisons](project_comparisons.md#unraid) for the comparison itself.
|
||||
|
||||
| Operation | mergerfs | Unraid array with parity |
|
||||
| --- | --- | --- |
|
||||
| Add an empty filesystem | Add its mount path as a branch | Clear if necessary, then format and add |
|
||||
| Add a populated filesystem | Add its mount path; data is immediately visible | Not normally possible once parity is active without destroying its contents |
|
||||
| Mix filesystem types | Any filesystem the host can mount | Supported array filesystem types only |
|
||||
| Mix device sizes | Arbitrary | Data disks cannot exceed the parity disk size |
|
||||
| Change membership while mounted | Supported through the [runtime interface](runtime_interface.md) | Array configuration normally requires stopping and restarting the array |
|
||||
| Remove a filesystem and retain it unchanged | Remove the branch | Possible, but normal array removal requires parity to be rebuilt |
|
||||
| Remove a filesystem without recalculating parity | Not applicable | Requires zeroing the disk, destroying its filesystem |
|
||||
|
||||
### Migrating from Unraid to mergerfs
|
||||
|
||||
Unraid array data disks each hold a complete, individually formatted
|
||||
filesystem. `shfs` only unions their top-level directories, so there
|
||||
is no striped or parity-dependent layout to unwind. After cleanly
|
||||
stopping the array, unencrypted data filesystems can be mounted under
|
||||
Linux and their mount points used directly as mergerfs branches. The
|
||||
files do not need to be rewritten merely because the pooling
|
||||
implementation changed.
|
||||
|
||||
For [encrypted Unraid disks](https://docs.unraid.net/unraid-os/system-administration/secure-your-server/securing-your-data/),
|
||||
securely retain the existing passphrase or key file before retiring
|
||||
Unraid. Open each disk's existing LUKS container with `cryptsetup`
|
||||
using those credentials, then mount the inner filesystem through its
|
||||
`/dev/mapper/` device and use that mount point as a branch. Do not
|
||||
reformat the disk or create a new LUKS container. Without a working
|
||||
passphrase or key file, the encrypted data cannot be accessed.
|
||||
|
||||
```
|
||||
Unraid:
|
||||
|
||||
disk1 ─┐
|
||||
disk2 ─┼─> shfs ─> /mnt/user
|
||||
disk3 ─┘
|
||||
+
|
||||
parity
|
||||
|
||||
mergerfs:
|
||||
|
||||
disk1 ─┐
|
||||
disk2 ─┼─> mergerfs ─> /mnt/storage
|
||||
disk3 ─┘
|
||||
```
|
||||
|
||||
Unraid's convention of representing shares as identically named
|
||||
top-level directories also maps naturally to mergerfs' union
|
||||
semantics.
|
||||
|
||||
* The parity disk is not a data filesystem and cannot become a branch
|
||||
as-is. It can be reformatted as another data disk if parity
|
||||
protection is no longer desired, or a separate parity solution such
|
||||
as [SnapRAID](https://www.snapraid.it) or
|
||||
[NonRAID](https://github.com/qvr/nonraid) can be used alongside
|
||||
mergerfs if redundancy is still wanted.
|
||||
* If migrating incrementally while keeping the remaining Unraid array
|
||||
operational, the normal Unraid disk-removal and parity-rebuild
|
||||
requirements still apply to each disk removed.
|
||||
* If the entire array is being retired at once, those intermediate
|
||||
Unraid parity rebuilds are unnecessary. Stop the array cleanly and
|
||||
use the existing data filesystems as branches.
|
||||
* Named pools need care. A pool backed by a single filesystem can be
|
||||
mounted and used as one branch. A multi-device Btrfs or ZFS pool is
|
||||
not equivalent to a collection of independent Unraid array disks: it
|
||||
can either remain intact and be mounted as one mergerfs branch, or
|
||||
its data must be migrated elsewhere before its component devices can
|
||||
be repurposed as independent filesystems.
|
||||
|
||||
### Migrating from mergerfs to Unraid
|
||||
|
||||
mergerfs places no special metadata on its branches, so their
|
||||
underlying filesystems remain unchanged. Direct reuse as Unraid array
|
||||
data disks requires local disks with a supported filesystem **and a
|
||||
partition layout accepted by the installed Unraid version**. A
|
||||
supported filesystem type alone is not sufficient.
|
||||
|
||||
In particular, mergerfs can use a filesystem formatted directly on a
|
||||
whole device, such as `/dev/sda`, without a partition table. Unraid
|
||||
array data disks require a filesystem within a compatible partition.
|
||||
Such unpartitioned filesystems cannot be imported directly: back up
|
||||
their data, let Unraid partition and format the disks, then restore
|
||||
the files. Creating a partition table over the existing filesystem is
|
||||
not a safe conversion. Check Unraid's
|
||||
[partitioning guidance](https://docs.unraid.net/unraid-os/troubleshooting/faq/)
|
||||
before planning a no-copy migration.
|
||||
|
||||
Current versions of Unraid support importing populated disks when
|
||||
constructing a new array, or after resetting the array configuration,
|
||||
provided that the populated data disks are added **before parity is
|
||||
activated**. Such a migration can therefore often be performed without
|
||||
copying the data:
|
||||
|
||||
1. Stop mergerfs and cleanly unmount the branch filesystems.
|
||||
2. Create or reset the Unraid array configuration.
|
||||
3. Assign all populated mergerfs disks as data disks while no parity
|
||||
disk is active.
|
||||
4. Verify that the existing filesystems and data are recognized
|
||||
correctly. If a populated disk is reported as unmountable, stop and
|
||||
investigate; do not format it, as formatting destroys its contents.
|
||||
5. Assign a parity disk at least as large as the largest data disk.
|
||||
6. Build parity over the completed array.
|
||||
|
||||
Once parity has been established, that convenient path is effectively
|
||||
closed. Adding another populated mergerfs disk to the parity-protected
|
||||
array would require Unraid to clear the disk, destroying the existing
|
||||
filesystem. A staged migration in which populated mergerfs disks are
|
||||
added one at a time therefore generally requires either copying their
|
||||
contents into the existing array first or resetting/reconstructing the
|
||||
Unraid configuration so that all populated data disks are present
|
||||
before parity is established.
|
||||
|
||||
Additional array-level constraints to keep in mind:
|
||||
|
||||
* Only filesystem types supported by Unraid can be assigned as array
|
||||
data disks.
|
||||
* No array data disk may be larger than the parity disk.
|
||||
* Array configuration changes normally require stopping and restarting
|
||||
the array. Removing a data disk normally requires a parity rebuild,
|
||||
and Unraid's parity-preserving removal procedure works by erasing
|
||||
the filesystem and zeroing the disk, so it is unsuitable when the
|
||||
goal is to retain that filesystem for use elsewhere.
|
||||
|
||||
When migrating into individual **Unraid array data slots**, the
|
||||
following configurations require copying data to suitable array
|
||||
filesystems rather than reusing the source storage directly:
|
||||
|
||||
* Remote filesystems or filesystem types Unraid does not support.
|
||||
* Storage that cannot be presented as a single array device, such as
|
||||
multi-device Btrfs or ZFS pools or `mdadm` arrays.
|
||||
|
||||
This restriction does not rule out retaining storage as a named pool.
|
||||
A compatible multi-device ZFS pool can be
|
||||
[imported into Unraid](https://docs.unraid.net/unraid-os/advanced-configurations/optimize-storage/zfs-storage/#importing-zfs-pools-created-on-other-systems)
|
||||
intact, without copying its data. Stop mergerfs and cleanly export the
|
||||
pool on the source system. With the Unraid array stopped, use **Add
|
||||
Pool**, assign all of the pool's devices (including support vdevs),
|
||||
set **File System** to **Auto**, and start the array to import it.
|
||||
If it is not recognized, investigate rather than format its devices.
|
||||
The imported pool remains separate from the parity-protected array;
|
||||
its redundancy comes from its ZFS layout, not Unraid array parity.
|
||||
|
||||
A branch pointing at a subdirectory does **not** by itself require a
|
||||
data copy: eligibility depends on its underlying disk and filesystem.
|
||||
For example, importing compatible disks that contributed
|
||||
`/mnt/disk1/media` and `/mnt/disk2/media` preserves their contents, and
|
||||
Unraid combines the top-level `media` directories into a
|
||||
[User Share](https://docs.unraid.net/unraid-os/using-unraid-to/manage-storage/shares/#user-shares).
|
||||
Review the desired share layout; deeper branch directories may need
|
||||
to be reorganized within their existing filesystems rather than
|
||||
copied to new storage.
|
||||
|
||||
### Summary
|
||||
|
||||
mergerfs treats storage membership as a property of its
|
||||
**configuration**, while a parity-protected Unraid array treats it as
|
||||
part of the **array state**. Moving an existing filesystem into or out
|
||||
of mergerfs normally does not alter that filesystem at all. Moving
|
||||
populated filesystems into or out of an active Unraid parity array can
|
||||
require clearing disks, rebuilding parity, moving data, or
|
||||
reconstructing the array configuration.
|
||||
|
||||
For semi-static data, mergerfs + SnapRAID provides pooling and parity
|
||||
while retaining the ability to add and remove independently formatted
|
||||
filesystems without modifying them. NonRAID provides an alternative
|
||||
real-time parity layer which can likewise be combined with mergerfs.
|
||||
@@ -48,6 +48,65 @@ before changing them to understand how functionality will change.
|
||||
* use LVM and LVM cache to place a SSD in front of your HDDs
|
||||
|
||||
|
||||
## Understanding mergerfs IO Performance
|
||||
|
||||
A common source of confusion when evaluating **mergerfs**' performance
|
||||
is comparing it to traditional RAID or filesystem level pooling
|
||||
technologies such as mdadm, ZFS, btrfs, or striped LVM. Those
|
||||
solutions aggregate devices into a single virtual device and spread IO
|
||||
across all members. Under favorable conditions that means the
|
||||
performance of the pool is the sum of the performance of its members:
|
||||
striped reads and writes hit multiple drives at once and the aggregate
|
||||
throughput scales with the number of devices, up to the limits of the
|
||||
interconnects (PCIe, SATA, network, etc.) which every member must
|
||||
share.
|
||||
|
||||
**mergerfs** does not aggregate or stripe data. Every file lives
|
||||
wholly on one branch and **mergerfs** simply routes access to it. As a
|
||||
result **mergerfs** does not aggregate the performance of its branches
|
||||
in the way RAID does. What it provides instead is **independent
|
||||
performance across branches**.
|
||||
|
||||
* **Best case:** workloads which access different branches
|
||||
concurrently perform independently of one another. If you have 3
|
||||
filesystems on 3 drives and each is being actively used, the total
|
||||
throughput can be A + B + C: the sum of each branch performing at
|
||||
full speed. No single drive is a bottleneck for the others.
|
||||
* **Worst case:** a single stream of IO - one process reading or
|
||||
writing one file - performs at the speed of the single branch the
|
||||
file is on, no differently than a lone filesystem. Additional
|
||||
branches do not make a single file faster.
|
||||
|
||||
This means that when reasoning about expected performance the key
|
||||
question is whether your workload is **single stream** or **multi
|
||||
stream**:
|
||||
|
||||
* Single large file transfers, video playback, or one process doing
|
||||
sequential IO will never exceed the speed of one branch regardless
|
||||
of how many are mounted. If a single stream must be faster, put a
|
||||
faster device (tiered cache, SSD, LVM cache) in front of the
|
||||
branches or use an actual striped RAID layer beneath **mergerfs**'
|
||||
branches.
|
||||
* Concurrency across branches is where **mergerfs** shines. Many
|
||||
parallel downloads, seeding torrents, multiple rsync streams,
|
||||
multiple applications reading different files: these naturally
|
||||
spread across branches and their throughput adds up.
|
||||
|
||||
There are practical upsides to this non-aggregated design as well:
|
||||
|
||||
* Drives of different sizes and speeds don't reduce the whole pool to
|
||||
the slowest member. Each branch performs at its own speed.
|
||||
* The failure of one drive only affects the files on that branch
|
||||
rather than the integrity of the entire pool.
|
||||
|
||||
Finally, keep in mind that **mergerfs** is a proxy above the branches,
|
||||
so measured throughput is the lesser of the branch's native performance
|
||||
and any overhead introduced by **mergerfs** and FUSE. Use
|
||||
[nullrw](config/options.md) or a ram disk as a branch to measure that
|
||||
overhead separately, and see the [benchmarking
|
||||
section](benchmarking.md) for how to benchmark meaningfully.
|
||||
|
||||
|
||||
## Additional Reading
|
||||
|
||||
* [Benchmarking](benchmarking.md)
|
||||
|
||||
+168
-123
@@ -2,6 +2,11 @@
|
||||
|
||||
There are no solutions, only trade-offs.
|
||||
|
||||
This page includes both direct pooling alternatives and adjacent storage
|
||||
technologies that solve different problems. Compare their operating layer,
|
||||
failure domain, redundancy model, and platform requirements rather than only
|
||||
their apparent pool capacity.
|
||||
|
||||
|
||||
## mhddfs
|
||||
|
||||
@@ -15,17 +20,20 @@ and security issues. mergerfs provides a superset of mhddfs' features
|
||||
and offers better performance. As of 2020, the author of mhddfs has
|
||||
[moved to using mergerfs.](https://romanrm.net/mhddfs#update)
|
||||
|
||||
* **Better than mergerfs for:** legacy setups that already depend on
|
||||
mhddfs-specific behavior and do not need newer features.
|
||||
* **Better than mergerfs for:** Nothing?
|
||||
* **Worse than mergerfs for:** most modern deployments due to maintenance
|
||||
status, feature coverage, and reliability.
|
||||
|
||||
Below is an example of mhddfs and mergerfs setup to work similarly.
|
||||
Below is an example of mhddfs and mergerfs set up to behave similarly.
|
||||
mhddfs falls back to the drive with the most free space when every drive
|
||||
has less than `mlimit` free; mergerfs with
|
||||
[minfreespace](config/minfreespace.md) filters those branches instead and
|
||||
may fail the create if none qualify.
|
||||
|
||||
```
|
||||
mhddfs -o mlimit=4G,allow_other /mnt/drive1,/mnt/drive2 /mnt/pool
|
||||
```sh
|
||||
mhddfs -o mlimit=4G,allow_other /mnt/hdd/disk0,/mnt/hdd/disk1 /mnt/pool
|
||||
|
||||
mergerfs -o minfreespace=4G,category.create=ff /mnt/drive1:/mnt/drive2 /mnt/pool
|
||||
mergerfs -o minfreespace=4G,category.create=ff /mnt/hdd/disk0:/mnt/hdd/disk1 /mnt/pool
|
||||
```
|
||||
|
||||
|
||||
@@ -46,10 +54,11 @@ available in most Linux distros, making it harder to get installed for
|
||||
the average user. Development has been largely dormant for years.
|
||||
|
||||
While aufs can often offer better peak performance due to being
|
||||
primarily kernel-based (at least when `passthrough.io` is disabled in
|
||||
mergerfs), mergerfs provides more configurability and is generally
|
||||
easier to use. mergerfs, however, does not offer the overlay /
|
||||
copy-on-write (CoW) features which aufs has.
|
||||
primarily kernel-based (at least when
|
||||
[passthrough.io](config/passthrough.md) is disabled in mergerfs),
|
||||
mergerfs provides more configurability and is generally easier to use.
|
||||
mergerfs, however, does not offer the overlay / copy-on-write (CoW)
|
||||
features which aufs has.
|
||||
|
||||
* **Better than mergerfs for:** kernel-level overlay/CoW workloads where
|
||||
that behavior is required.
|
||||
@@ -92,9 +101,10 @@ overlay/CoW-style layering, while mergerfs' primary feature is
|
||||
policy-based pooling and placement across multiple regular writable
|
||||
filesystems.
|
||||
|
||||
Since unionfs-fuse, as the name suggests, is a FUSE-based technology
|
||||
it can be used without elevated privileges that kernel solutions such
|
||||
as unionfs, aufs, and overlayfs require.
|
||||
As a FUSE filesystem, unionfs-fuse can often be mounted by an
|
||||
unprivileged user when the system's FUSE configuration permits it. Kernel
|
||||
solutions such as unionfs, aufs, and overlayfs normally require additional
|
||||
privileges.
|
||||
|
||||
* **Better than mergerfs for:** overlay-style unions with copy-on-write
|
||||
semantics in userspace.
|
||||
@@ -106,10 +116,10 @@ as unionfs, aufs, and overlayfs require.
|
||||
|
||||
* [https://docs.kernel.org/filesystems/overlayfs.html](https://docs.kernel.org/filesystems/overlayfs.html)
|
||||
|
||||
overlayfs is effectively the functional successor to unionfs,
|
||||
unionfs-fuse, and aufs and is widely used by Linux container platforms
|
||||
such as Docker and Podman. Both overlayfs and FUSE were originally
|
||||
created by Miklos Szeredi.
|
||||
overlayfs is effectively the successor to unionfs, unionfs-fuse, and
|
||||
aufs and is widely used by Linux container platforms such as Docker and
|
||||
Podman. Both overlayfs and FUSE were originally created by Miklos
|
||||
Szeredi.
|
||||
|
||||
If your use case is layering a writable filesystem on top of read-only
|
||||
filesystems then you should look first to overlayfs. Its feature set
|
||||
@@ -121,6 +131,24 @@ however is very different from mergerfs and solves different problems.
|
||||
flexible branch selection policies across many existing filesystems.
|
||||
|
||||
|
||||
## fuse-overlayfs
|
||||
|
||||
* [https://github.com/containers/fuse-overlayfs](https://github.com/containers/fuse-overlayfs)
|
||||
|
||||
fuse-overlayfs implements overlayfs-style lower, upper, and work directories
|
||||
in userspace. Its primary use case is rootless containers, where a kernel
|
||||
overlayfs mount may not be available.
|
||||
|
||||
Like unionfs-fuse and kernel overlayfs, it provides layering and copy-up
|
||||
semantics rather than mergerfs-style placement across a pool of writable
|
||||
filesystems.
|
||||
|
||||
* **Better than mergerfs for:** rootless container and other unprivileged
|
||||
overlay workloads.
|
||||
* **Worse than mergerfs for:** general storage pooling across independent
|
||||
writable filesystems.
|
||||
|
||||
|
||||
## RAID0, JBOD, SPAN, drive concatenation, striping
|
||||
|
||||
* [RAID0](https://en.wikipedia.org/wiki/Standard_RAID_levels#RAID_0)
|
||||
@@ -128,6 +156,10 @@ however is very different from mergerfs and solves different problems.
|
||||
* [SPAN](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#Concatenation_(SPAN,_BIG))
|
||||
* [striping](https://en.wikipedia.org/wiki/Data_striping)
|
||||
|
||||
"JBOD" is ambiguous: it can also mean disks exposed independently with
|
||||
no aggregation. In this section it refers only to concatenated or spanned
|
||||
layouts.
|
||||
|
||||
These are block device technologies which, in some form, aggregate
|
||||
devices into what appears to be a single device on which a traditional
|
||||
filesystem can be used. The filesystem has no understanding of the
|
||||
@@ -162,12 +194,12 @@ allow the pool to continue to run should one device fail and rebuild
|
||||
the data once the device is replaced.
|
||||
|
||||
mergerfs offers no parity or redundancy features so in that regard the
|
||||
technologies are not comparable. [SnapRAID](https://www.snapraid.it)
|
||||
or [NonRAID](https://github.com/qvr/nonraid) can be used in
|
||||
combination with mergerfs to provide redundancy. Unlike traditional
|
||||
RAID5 or RAID6 SnapRAID works with drives of different sizes and can
|
||||
have more than 2 parity drives. Parity calculations with SnapRAID are
|
||||
not done in real-time but NonRAID's are.
|
||||
technologies are not comparable. [SnapRAID](https://www.snapraid.it) or
|
||||
NonRAID (see [below](#nonraid)) can be used in combination with mergerfs
|
||||
to provide redundancy. Unlike traditional RAID5 or RAID6, SnapRAID works
|
||||
with drives of different sizes and can have more than 2 parity drives.
|
||||
Parity calculations with SnapRAID are not done in real-time but
|
||||
NonRAID's are.
|
||||
|
||||
For more details and comparison of SnapRAID to related technologies
|
||||
see [https://www.snapraid.it/compare](https://www.snapraid.it/compare).
|
||||
@@ -204,7 +236,7 @@ data on that disk.
|
||||
## mdadm
|
||||
|
||||
* [https://en.wikipedia.org/wiki/Mdadm](https://en.wikipedia.org/wiki/Mdadm)
|
||||
* [https://raid.wiki.kernel.org/index.php/A_guide_to_mdadm](https://raid.wiki.kernel.org/index.php/A_guide_to_mdadm)
|
||||
* [https://docs.kernel.org/admin-guide/md.html](https://docs.kernel.org/admin-guide/md.html)
|
||||
|
||||
`mdadm` is Linux userspace tooling for configuring and managing software
|
||||
RAID arrays provided by the kernel `md` driver. It can build RAID0/1/4/5/6/10
|
||||
@@ -226,7 +258,7 @@ be combined with tools like SnapRAID or NonRAID when redundancy is needed.
|
||||
## device mapper
|
||||
|
||||
* [https://en.wikipedia.org/wiki/Device_mapper](https://en.wikipedia.org/wiki/Device_mapper)
|
||||
* [https://docs.kernel.org/admin-guide/device-mapper](https://docs.kernel.org/admin-guide/device-mapper)
|
||||
* [https://docs.kernel.org/admin-guide/device-mapper/index.html](https://docs.kernel.org/admin-guide/device-mapper/index.html)
|
||||
|
||||
Device mapper is a kernel framework for creating virtual block devices
|
||||
from one or more underlying devices. Technologies such as LVM,
|
||||
@@ -244,11 +276,11 @@ later pool if desired.
|
||||
across many independent, already-formatted filesystems.
|
||||
|
||||
|
||||
## UnRAID
|
||||
## Unraid
|
||||
|
||||
* [https://unraid.net](https://unraid.net)
|
||||
|
||||
UnRAID is a full OS platform that offers pooled storage with integrated
|
||||
Unraid is a full OS platform that offers pooled storage with integrated
|
||||
real-time parity. Compared with mergerfs, its main advantage is that it
|
||||
bundles pooling and parity in one product, while mergerfs is a
|
||||
standalone Linux pooling filesystem without built-in redundancy. For
|
||||
@@ -257,14 +289,20 @@ approach is also preferable.
|
||||
|
||||
For semi-static data, mergerfs + [SnapRAID](https://www.snapraid.it)
|
||||
provides a similar, but not real-time, solution.
|
||||
[NonRAID](https://github.com/qvr/nonraid) (see below) is a
|
||||
fork of UnRAID's parity calculation solution and can also be used with
|
||||
[NonRAID](https://github.com/qvr/nonraid) (see [below](#nonraid)) is a
|
||||
fork of Unraid's parity calculation solution and can also be used with
|
||||
mergerfs.
|
||||
|
||||
Moving between the two is not symmetric because mergerfs membership is
|
||||
a configuration change while Unraid array membership is part of the
|
||||
array state. See [Migration](migration.md) for details.
|
||||
|
||||
* **Better than mergerfs for:** turnkey pooling plus integrated real-time
|
||||
parity in one product.
|
||||
* **Worse than mergerfs for:** users who prefer a Linux distro-agnostic,
|
||||
fully open-source component approach.
|
||||
* **Worse than mergerfs for:** non-destructive incremental addition and
|
||||
removal of populated filesystems, arbitrary branch types and filesystem
|
||||
layouts, and users who prefer a Linux distro-agnostic, fully open-source
|
||||
component approach.
|
||||
|
||||
|
||||
## NonRAID
|
||||
@@ -272,7 +310,7 @@ mergerfs.
|
||||
* [https://github.com/qvr/nonraid](https://github.com/qvr/nonraid)
|
||||
|
||||
[NonRAID](https://github.com/qvr/nonraid#how-it-works) is a fork of
|
||||
UnRAID's storage array kernel driver. Effectively it is like RAID5 at
|
||||
Unraid's storage array kernel driver. Effectively it is like RAID5 at
|
||||
the block device layer without treating the collection of devices as a
|
||||
single device. Each device gets a virtual block device associated with
|
||||
it which can be formatted with the filesystem of the user's choice and
|
||||
@@ -293,7 +331,7 @@ be used with mergerfs for those looking for a unified view.
|
||||
|
||||
## ZFS
|
||||
|
||||
* [https://en.wikipedia.org/wiki/ZFS](https://en.wikipedia.org/wiki/ZFS)
|
||||
* [https://openzfs.github.io/openzfs-docs](https://openzfs.github.io/openzfs-docs)
|
||||
|
||||
ZFS is an advanced filesystem that combines storage pooling,
|
||||
snapshots, data integrity checking, and built-in RAID-like redundancy
|
||||
@@ -313,72 +351,78 @@ without built-in redundancy.
|
||||
|
||||
## ZFS AnyRAID
|
||||
|
||||
* [https://github.com/openzfs/zfs/discussions/16773](https://github.com/openzfs/zfs/discussions/16773)
|
||||
* [Announcement](https://docs.hexos.com/blog/2025-05-22)
|
||||
* [Development update](https://docs.hexos.com/blog/2026-7-15)
|
||||
|
||||
[ZFS AnyRAID](https://github.com/openzfs/zfs/discussions/16773)
|
||||
([announcement](https://hexos.com/blog/introducing-zfs-anyraid-sponsored-by-eshtek))
|
||||
is a feature under development for ZFS intended to provide more
|
||||
flexibility in pools by allowing mixed-capacity disks while
|
||||
maintaining live redundancy.
|
||||
ZFS AnyRAID is under development to support mixed-capacity disks while
|
||||
retaining ZFS redundancy. The planned layouts include mirrored and RAID-Z
|
||||
variants, along with rebalancing and pool contraction work.
|
||||
|
||||
If released, ZFS AnyRAID would offer some of the flexibility associated
|
||||
with mergerfs-style mixed-capacity expansion, but with built-in
|
||||
redundancy inside ZFS. mergerfs already provides that mixed-size,
|
||||
add-drives-anytime flexibility today, albeit without built-in
|
||||
redundancy. If you need redundancy with that flexibility, ZFS AnyRAID
|
||||
could become an option when available; until then, mergerfs remains the
|
||||
simpler currently available choice for mixed-capacity pooling, with
|
||||
redundancy and integrity available via SnapRAID and/or NonRAID.
|
||||
Availability is still landing in OpenZFS. Published notes indicate initial
|
||||
mirror layouts may ship in a 2.4.x release, with broader RAID-Z variants,
|
||||
rebalancing, and pool contraction targeted for later releases. Treat
|
||||
release and downstream product-integration dates as roadmap targets
|
||||
rather than commitments.
|
||||
|
||||
Where available, AnyRAID would offer some of the mixed-size flexibility
|
||||
associated with mergerfs while retaining redundancy inside ZFS. mergerfs
|
||||
provides mixed-size, add-drives-anytime pooling today, but without
|
||||
built-in redundancy; SnapRAID or NonRAID can provide separate
|
||||
protection.
|
||||
|
||||
* **Better than mergerfs for:** users who want mixed-capacity flexibility
|
||||
and built-in live redundancy in one filesystem stack.
|
||||
* **Worse than mergerfs for:** users who need mature availability now and
|
||||
the simplest add-drives-anytime workflow without redundancy overhead.
|
||||
* **Worse than mergerfs for:** users who need production availability now or
|
||||
want to pool existing filesystems without migrating data.
|
||||
|
||||
|
||||
## Bcachefs
|
||||
## bcachefs
|
||||
|
||||
* [https://bcachefs.org](https://bcachefs.org)
|
||||
|
||||
Bcachefs is a modern copy-on-write filesystem for Linux designed for
|
||||
multi-device storage aggregation. It includes integrated features such
|
||||
as checksumming, snapshots, compression, encryption, and redundancy.
|
||||
bcachefs is a modern copy-on-write filesystem for Linux designed for
|
||||
multi-device storage aggregation. It includes integrated features such as
|
||||
checksumming, snapshots, compression, encryption, and redundancy.
|
||||
|
||||
Compared with mergerfs, bcachefs is a full integrated filesystem with
|
||||
built-in checksumming, snapshots, compression, and redundancy, while
|
||||
mergerfs is a lightweight pooling layer over existing filesystems.
|
||||
Bcachefs is still evolving and should be evaluated carefully for
|
||||
production use.
|
||||
Compared with mergerfs, bcachefs is a full integrated filesystem, while
|
||||
mergerfs is a lightweight pooling layer over existing filesystems. Beginning
|
||||
with Linux 6.18, bcachefs is no longer distributed in the mainline kernel; the
|
||||
project now distributes it as an external DKMS module. It remains actively
|
||||
developed, but deployments must account for kernel compatibility and its
|
||||
shorter operational history.
|
||||
|
||||
* **Better than mergerfs for:** integrated CoW filesystem features,
|
||||
checksumming, snapshots, compression, and built-in multi-device
|
||||
redundancy.
|
||||
* **Worse than mergerfs for:** conservative production setups prioritizing
|
||||
maturity, simplicity, and broad long-term deployment stability.
|
||||
* **Worse than mergerfs for:** pooling existing filesystems without
|
||||
reformatting, mainline-kernel availability, and per-disk native access.
|
||||
|
||||
|
||||
## Btrfs Single Data Profile
|
||||
## Btrfs multi-device filesystems
|
||||
|
||||
* [https://btrfs.readthedocs.io](https://btrfs.readthedocs.io)
|
||||
* [https://btrfs.readthedocs.io/en/latest/Volume-management.html](https://btrfs.readthedocs.io/en/latest/Volume-management.html)
|
||||
|
||||
[Btrfs'](https://btrfs.readthedocs.io) `single` data profile is a
|
||||
multi-device Btrfs mode that can spread data across drives without
|
||||
adding redundancy.
|
||||
Btrfs can combine multiple devices in one filesystem and select separate
|
||||
allocation profiles for data and metadata. The `single` data profile spreads
|
||||
one copy of data across available devices without redundancy; other profiles
|
||||
provide mirroring, striping, or parity with different capacity and failure
|
||||
trade-offs.
|
||||
|
||||
Compared with mergerfs, Btrfs `single` is managed as one integrated
|
||||
filesystem, while mergerfs pools separate existing filesystems. In
|
||||
Btrfs `single`, losing one device can impact the whole filesystem;
|
||||
with mergerfs, loss is typically limited to data on the failed disk.
|
||||
Compared with mergerfs, multi-device Btrfs is one integrated filesystem,
|
||||
while mergerfs pools separate existing filesystems. Losing a device from a
|
||||
`single` profile can make the whole Btrfs filesystem incomplete. Redundant
|
||||
profiles change that risk, but the member devices still are not independently
|
||||
mountable filesystems.
|
||||
|
||||
* **Better than mergerfs for:** integrated multi-device management in one
|
||||
filesystem namespace.
|
||||
* **Worse than mergerfs for:** users who value per-disk failure isolation
|
||||
and easy independent recovery.
|
||||
* **Better than mergerfs for:** integrated checksumming, snapshots, balancing,
|
||||
and selectable redundancy profiles.
|
||||
* **Worse than mergerfs for:** pooling existing mixed filesystem types,
|
||||
per-disk native access, and failure isolation without redundancy.
|
||||
|
||||
|
||||
## StableBit's DrivePool
|
||||
|
||||
* [https://stablebit.com](https://stablebit.com)
|
||||
* [https://stablebit.com/DrivePool/Features](https://stablebit.com/DrivePool/Features)
|
||||
|
||||
DrivePool is a commercial storage pooling product for Windows that
|
||||
combines multiple drives into a single volume and includes balancing
|
||||
@@ -399,19 +443,17 @@ external-tool composability.
|
||||
|
||||
* [https://learn.microsoft.com/en-us/windows-server/storage/storage-spaces/overview](https://learn.microsoft.com/en-us/windows-server/storage/storage-spaces/overview)
|
||||
|
||||
Windows Storage Spaces is Microsoft's software-defined storage
|
||||
platform for Windows Server and Windows Pro. It aggregates multiple
|
||||
physical disks into logical storage pools with built-in redundancy
|
||||
options (mirroring, parity) and automatic performance optimization
|
||||
through tiered caching.
|
||||
Windows Storage Spaces is Microsoft's block-level storage virtualization
|
||||
platform for Windows. It aggregates physical disks into storage pools from
|
||||
which virtual disks can be created with simple, mirror, or parity layouts.
|
||||
Supported Windows Server configurations can also use faster media as a
|
||||
Storage Bus Cache.
|
||||
|
||||
Storage Spaces is Windows-only and requires more planning upfront due
|
||||
to redundancy options and RAID-like features. mergerfs is
|
||||
Linux-focused and takes a simpler approach, pooling existing
|
||||
filesystems without built-in redundancy. If you need a Windows
|
||||
solution with integrated data protection, Storage Spaces is a good
|
||||
choice. For Linux users seeking lightweight pooling without redundancy
|
||||
overhead, mergerfs is the better option.
|
||||
Unlike mergerfs, Storage Spaces manages the member devices below the
|
||||
filesystem and stripes or mirrors data according to the selected layout.
|
||||
This can provide integrated resilience and one contiguous volume, but member
|
||||
disks are not independently readable filesystems. mergerfs instead pools
|
||||
existing filesystems and leaves their files directly accessible.
|
||||
|
||||
* **Better than mergerfs for:** Windows environments needing integrated
|
||||
pooling, resilience, and tiering in the platform stack.
|
||||
@@ -439,6 +481,25 @@ Plan 9 isn't a widely used OS so this comparison is mostly academic.
|
||||
practical deployment availability.
|
||||
|
||||
|
||||
## 9P
|
||||
|
||||
* [https://en.wikipedia.org/wiki/9P_(protocol)](https://en.wikipedia.org/wiki/9P_(protocol))
|
||||
|
||||
[9P](https://en.wikipedia.org/wiki/9P_(protocol)) is a filesystem
|
||||
protocol from the Plan 9 operating system used to access files over a
|
||||
network between systems.
|
||||
|
||||
Compared with mergerfs, 9P is a network filesystem protocol, while
|
||||
mergerfs is a local Linux FUSE filesystem for pooling directories and
|
||||
filesystems under one mount with branch policies. Like the Plan 9 binds
|
||||
comparison above, this is mostly academic for typical mergerfs users.
|
||||
|
||||
* **Better than mergerfs for:** lightweight network filesystem protocol
|
||||
use between compatible systems.
|
||||
* **Worse than mergerfs for:** local Linux filesystem pooling because it
|
||||
does not provide mergerfs-like branch policies or pooling semantics.
|
||||
|
||||
|
||||
## SnapRAID pooling
|
||||
|
||||
* [https://www.snapraid.it/manual](https://www.snapraid.it/manual)
|
||||
@@ -498,7 +559,7 @@ given the differing feature sets and focuses of the two projects.
|
||||
## distributed filesystems
|
||||
|
||||
* [AFS](https://www.openafs.org)
|
||||
* [Ceph/CephFS](https://ceph.io)
|
||||
* [Ceph/CephFS](https://docs.ceph.com/en/latest/cephfs/)
|
||||
* [GlusterFS](https://www.gluster.org)
|
||||
* [LizardFS](https://lizardfs.com)
|
||||
* [MooseFS](https://moosefs.com)
|
||||
@@ -539,8 +600,8 @@ pooling through normal POSIX filesystem calls.
|
||||
|
||||
Given its design nofs is not suited for general usage as third-party
|
||||
applications will not be able to take advantage of the unioning
|
||||
behavior it offers. It is primarily for more simple situations where
|
||||
something like mergerfs is unable to be used.
|
||||
behavior it offers. It is primarily for simpler situations where
|
||||
mergerfs cannot be used.
|
||||
|
||||
* **Better than mergerfs for:** constrained environments where a FUSE
|
||||
mount is not possible but command-style union behavior is acceptable.
|
||||
@@ -548,32 +609,34 @@ something like mergerfs is unable to be used.
|
||||
usable by arbitrary third-party applications.
|
||||
|
||||
|
||||
## policyfs
|
||||
## PolicyFS
|
||||
|
||||
* [https://policyfs.org](https://policyfs.org)
|
||||
* [https://github.com/hieutdo/policyfs](https://github.com/hieutdo/policyfs)
|
||||
|
||||
policyfs is a Linux FUSE storage daemon that unifies multiple storage
|
||||
PolicyFS is a Linux FUSE storage daemon that unifies multiple storage
|
||||
paths under a single mountpoint, similarly to mergerfs. Its
|
||||
distinguishing features are explicit path-pattern-based routing rules
|
||||
for placement, and an optional SQLite metadata index that can serve
|
||||
`readdir` and `getattr` operations without spinning up HDDs.
|
||||
|
||||
Compared with mergerfs, policyfs emphasizes explicit path-based routing
|
||||
and optional metadata indexing, while mergerfs emphasizes a broad,
|
||||
policy-driven pooling model with longer operational history and wider
|
||||
adoption. It doesn't try limiting spinup as doing so fundimentally is
|
||||
in [conflict with other features.](faq/limit_drive_spinup.md)
|
||||
Compared with mergerfs, PolicyFS emphasizes explicit path-based routing,
|
||||
scheduled SSD-to-HDD movement, and optional metadata indexing. mergerfs
|
||||
emphasizes a broader policy-driven pooling model with longer operational
|
||||
history and wider application compatibility. mergerfs does not try to prevent
|
||||
metadata operations from waking branches because doing so fundamentally
|
||||
[conflicts with other features.](faq/limit_drive_spinup.md)
|
||||
|
||||
Due to the nature of PolicyFS it can not really be used while changing
|
||||
files out-of-band and with the delayed writeback of metadata and
|
||||
unlinking there is a greater risk of inconsistancy in error
|
||||
conditions.
|
||||
PolicyFS's indexed storage deliberately defers some delete, rename, and
|
||||
attribute operations until a maintenance job runs. Out-of-band changes require
|
||||
the index to be refreshed, and its documentation identifies limits around
|
||||
`O_DIRECT`, advisory locks, NFS re-export, and full POSIX coverage. Those are
|
||||
material trade-offs, not only maturity differences.
|
||||
|
||||
* **Better than mergerfs for:** explicit path-based routing and optional
|
||||
metadata indexing to reduce HDD spin-ups.
|
||||
* **Worse than mergerfs for:** users prioritizing project maturity,
|
||||
adoption, performance, and long-established operational behavior.
|
||||
* **Better than mergerfs for:** explicit path-based routing, scheduled
|
||||
tiering, and metadata indexing designed to reduce HDD spin-ups.
|
||||
* **Worse than mergerfs for:** broad POSIX compatibility, immediate mutation
|
||||
semantics on every branch, and long-established operational behavior.
|
||||
|
||||
|
||||
## Greyhole
|
||||
@@ -593,22 +656,4 @@ Linux FUSE mount for local pooling via standard POSIX filesystem calls.
|
||||
* **Better than mergerfs for:** Samba-centric environments that need
|
||||
built-in per-share redundancy in the same tool.
|
||||
* **Worse than mergerfs for:** general-purpose local filesystem pooling
|
||||
outside Samba and heavy small-file churn workloads.
|
||||
|
||||
|
||||
## 9P
|
||||
|
||||
* [https://en.wikipedia.org/wiki/9P_(protocol)](https://en.wikipedia.org/wiki/9P_(protocol))
|
||||
|
||||
[9P](https://en.wikipedia.org/wiki/9P_(protocol)) is a filesystem
|
||||
protocol from the Plan 9 operating system used to access files over a
|
||||
network between systems.
|
||||
|
||||
Compared with mergerfs, 9P is a network filesystem protocol, while
|
||||
mergerfs is a local Linux FUSE filesystem for pooling directories and
|
||||
filesystems under one mount with branch policies.
|
||||
|
||||
* **Better than mergerfs for:** lightweight network filesystem protocol
|
||||
use between compatible systems.
|
||||
* **Worse than mergerfs for:** local Linux filesystem pooling because it
|
||||
does not provide mergerfs-like branch policies or pooling semantics.
|
||||
outside Samba, and for heavy small-file churn workloads.
|
||||
|
||||
+73
-14
@@ -62,28 +62,84 @@ For some suggestions on branch setup see [the page on
|
||||
branches](config/branches.md#branch-setup).
|
||||
|
||||
|
||||
### Web UI
|
||||
|
||||
[mergerfs-webui](tooling.md#mergerfs-webui) provides a browser-based
|
||||
alternative for creating and managing mounts on Linux. It can edit
|
||||
startup configuration and inspect and change settings on running
|
||||
mounts. See the
|
||||
[mergerfs-webui documentation](https://github.com/trapexit/mergerfs-webui#readme)
|
||||
for installation and secure access guidance. Runtime changes alone do
|
||||
not persist across remounts.
|
||||
|
||||
|
||||
### Command Line
|
||||
|
||||
```
|
||||
# Two positional options: `:` colon separated branches and mountpoint
|
||||
mergerfs -o opt[,opt...] /branch0:/branch1 /mountpoint
|
||||
mergerfs -o opt[,opt...] /mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media
|
||||
# or list of positional options where the last is the mountpoint
|
||||
mergerfs -o opt[,opt...] /branch0 /branch1 /mountpoint
|
||||
mergerfs -o opt[,opt...] /mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2 /media
|
||||
# or set mountpoint via -o mountpoint
|
||||
mergerfs -o mountpoint=/mountpoint /branch0 /branch1
|
||||
mergerfs -o mountpoint=/media /mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2
|
||||
```
|
||||
|
||||
```
|
||||
mergerfs -o cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false /mnt/hdd0:/mnt/hdd1 /media
|
||||
mergerfs -o cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false /mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media
|
||||
```
|
||||
|
||||
|
||||
### /etc/fstab
|
||||
|
||||
```
|
||||
/mnt/hdd0:/mnt/hdd1 /media mergerfs cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false 0 0
|
||||
/mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media mergerfs cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false 0 0
|
||||
```
|
||||
|
||||
### systemd mount dependencies
|
||||
|
||||
On a `systemd` based system, no custom service or setup script is
|
||||
needed when the branch filesystems and mergerfs are all configured in
|
||||
`/etc/fstab`. Add
|
||||
[`x-systemd.requires-mounts-for=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.mount.html#x-systemd.wants-mounts-for=)
|
||||
once for each branch mount target in the mergerfs entry:
|
||||
|
||||
```fstab
|
||||
LABEL=disk0 /mnt/hdd/disk0 ext4 defaults 0 2
|
||||
LABEL=disk1 /mnt/hdd/disk1 ext4 defaults 0 2
|
||||
LABEL=disk2 /mnt/hdd/disk2 ext4 defaults 0 2
|
||||
/mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media mergerfs cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false,x-systemd.requires-mounts-for=/mnt/hdd/disk0,x-systemd.requires-mounts-for=/mnt/hdd/disk1,x-systemd.requires-mounts-for=/mnt/hdd/disk2 0 0
|
||||
```
|
||||
|
||||
Replace the example labels and filesystem types with those used by the
|
||||
system. `systemd` creates `Requires=` and `After=` dependencies from
|
||||
the mergerfs mount to `mnt-hdd-disk0.mount`,
|
||||
`mnt-hdd-disk1.mount`, and `mnt-hdd-disk2.mount`. It starts those
|
||||
mounts first and does not start the mergerfs mount if a required mount
|
||||
fails. This checks the mount units, not merely whether the
|
||||
`/mnt/hdd/diskN` directories exist.
|
||||
|
||||
After editing `/etc/fstab`, reload the generated units:
|
||||
|
||||
```shell
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
The dependencies apply the next time `/media` is mounted and on
|
||||
subsequent boots.
|
||||
|
||||
When using a service such as the
|
||||
[`mergerfs-media.service`](#systemd-simple) example below instead of an
|
||||
`/etc/fstab` mergerfs entry, add the equivalent directive to its
|
||||
`[Unit]` section:
|
||||
|
||||
```systemd
|
||||
RequiresMountsFor=/mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2
|
||||
```
|
||||
|
||||
`RequiresMountsFor=` adds both the requirement and ordering
|
||||
dependencies, so separate `Requires=` and `After=` entries for these
|
||||
mounts are unnecessary.
|
||||
|
||||
### /etc/fstab w/ config file
|
||||
|
||||
For more complex setups it can be useful to separate out the config.
|
||||
@@ -114,10 +170,9 @@ resolve the symlinks and use the real path.
|
||||
`ls -lh /etc/mergerfs/branches/media/*`
|
||||
|
||||
```text
|
||||
lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd00 -> /mnt/hdd/hdd00
|
||||
lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd01 -> /mnt/hdd/hdd01
|
||||
lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd02 -> /mnt/hdd/hdd02
|
||||
lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd03 -> /mnt/hdd/hdd03
|
||||
lrwxrwxrwx 1 root root 14 Aug 4 2023 disk0 -> /mnt/hdd/disk0
|
||||
lrwxrwxrwx 1 root root 14 Aug 4 2023 disk1 -> /mnt/hdd/disk1
|
||||
lrwxrwxrwx 1 root root 14 Aug 4 2023 disk2 -> /mnt/hdd/disk2
|
||||
```
|
||||
|
||||
### systemd (simple)
|
||||
@@ -127,6 +182,7 @@ lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd03 -> /mnt/hdd/hdd03
|
||||
```systemd title="mergerfs-media.service" linenums="1"
|
||||
[Unit]
|
||||
Description=mergerfs /media service
|
||||
RequiresMountsFor=/mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2
|
||||
After=local-fs.target network.target
|
||||
|
||||
[Service]
|
||||
@@ -138,8 +194,9 @@ ExecStart=/usr/bin/mergerfs \
|
||||
-o category.create=pfrd \
|
||||
-o func.getattr=newest \
|
||||
-o dropcacheonclose=false \
|
||||
/mnt/hdd0 \
|
||||
/mnt/hdd1 \
|
||||
/mnt/hdd/disk0 \
|
||||
/mnt/hdd/disk1 \
|
||||
/mnt/hdd/disk2 \
|
||||
/media
|
||||
ExecStop=/usr/bin/umount /media
|
||||
# Or if you need fusermount
|
||||
@@ -193,8 +250,9 @@ WantedBy=default.target
|
||||
```systemd title="mergerfs-media.service" linenums="1"
|
||||
[Unit]
|
||||
Description=mergerfs /media service
|
||||
RequiresMountsFor=/mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2
|
||||
Requires=setup-for-mergerfs.service
|
||||
After=local-fs.target network.target prepare-for-mergerfs.service
|
||||
After=local-fs.target network.target setup-for-mergerfs.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
@@ -205,8 +263,9 @@ ExecStart=/usr/bin/mergerfs \
|
||||
-o category.create=pfrd \
|
||||
-o func.getattr=newest \
|
||||
-o dropcacheonclose=false \
|
||||
/mnt/hdd0 \
|
||||
/mnt/hdd1 \
|
||||
/mnt/hdd/disk0 \
|
||||
/mnt/hdd/disk1 \
|
||||
/mnt/hdd/disk2 \
|
||||
/media
|
||||
ExecStop=/usr/bin/umount /media
|
||||
# Or if you need fusermount
|
||||
|
||||
@@ -23,13 +23,16 @@
|
||||
|
||||
## Software and services commonly used with mergerfs
|
||||
|
||||
* [mergerfs-webui](tooling.md#mergerfs-webui): a browser-based tool for
|
||||
managing mergerfs mounts, runtime settings, and startup configuration
|
||||
on Linux.
|
||||
* [snapraid](https://www.snapraid.it): a backup program designed for
|
||||
disk arrays, storing parity information for data recovery in the
|
||||
event of up to six disk failures.
|
||||
* [nonraid](https://github.com/qvr/nonraid): NonRAID is a fork of the
|
||||
unRAID system's open-source md_unraid kernel driver enabling
|
||||
UnRAID-style storage arrays with parity protection outside of the
|
||||
commercial UnRAID system.
|
||||
Unraid system's open-source md_unraid kernel driver enabling
|
||||
Unraid-style storage arrays with parity protection outside of the
|
||||
commercial Unraid system.
|
||||
* [rclone](https://rclone.org): a command-line program to manage
|
||||
files on cloud storage. It is a feature-rich alternative to cloud
|
||||
vendors' web storage interfaces. rclone's
|
||||
@@ -43,10 +46,10 @@
|
||||
* [Proxmox](https://www.proxmox.com): Proxmox is a popular, Debian
|
||||
based, virtualization platform. Users tend to install mergerfs on
|
||||
the host and pass the mount into containers.
|
||||
* [UnRAID](https://unraid.net): "Unraid is a powerful, easy-to-use
|
||||
* [Unraid](https://unraid.net): "Unraid is a powerful, easy-to-use
|
||||
operating system for self-hosted servers and network-attached
|
||||
storage." While UnRAID has its own union filesystem it isn't
|
||||
uncommon to see UnRAID users leverage mergerfs given the differences
|
||||
storage." While Unraid has its own union filesystem it isn't
|
||||
uncommon to see Unraid users leverage mergerfs given the differences
|
||||
in the technologies. There is a [plugin available by
|
||||
Rysz](https://forums.unraid.net/topic/144999-plugin-mergerfs-for-unraid-support-topic)
|
||||
to ease installation and setup.
|
||||
|
||||
@@ -57,7 +57,7 @@ NFS export settings:
|
||||
* `softreval`: NFS client will serve up cached data after `retrans`
|
||||
attempts to revalidate the data. Helps with intermittent network
|
||||
issues.
|
||||
* `timeo=150`: Timeout till retrying request.
|
||||
* `timeo=150`: Timeout till retrying request.
|
||||
* `retrans=3`: Number of retrying a request.
|
||||
* `rsize=131072`: Read size.
|
||||
* `wsize=131072`: Write size.
|
||||
@@ -76,12 +76,11 @@ but if you run into problems it may be worth trying Samba/SMB.
|
||||
|
||||
**mergerfs settings:**
|
||||
|
||||
* `never-forget-nodes=true` or `remember-nodes=SECONDS`
|
||||
* `never-forget-nodes=true`
|
||||
* `inodecalc=path-hash`
|
||||
* `lazy-umount-mountpoint=false`
|
||||
|
||||
`never-forget-nodes=true` (or `remember-nodes` with a large value such
|
||||
as `86400` for a full day) is needed because NFS uses the
|
||||
`never-forget-nodes=true` is needed because NFS uses the
|
||||
`name_to_handle_at` and `open_by_handle_at` functions which allow a
|
||||
program to keep a reference to a file without technically having it
|
||||
open in the typical sense. The problem is that FUSE has no way to know
|
||||
@@ -89,8 +88,8 @@ that NFS has a handle that it will later use to open the file
|
||||
again. As a result, it is possible for the kernel to tell mergerfs to
|
||||
forget about the file node and should NFS ever ask for that node's
|
||||
details in the future it would have nothing to respond with. Keeping
|
||||
nodes around forever (or however long is set with `remember-nodes`) is
|
||||
not ideal but at the moment the only way to manage the situation.
|
||||
nodes around until they are unlinked, removed, or replaced is not ideal
|
||||
but at the moment the only way to manage the situation.
|
||||
|
||||
`inodecalc=path-hash` is needed because NFS is sensitive to
|
||||
out-of-band changes. FUSE doesn't care if a file's inode value changes
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
depending on [fuse-msg-size](config/fuse-msg-size.md)
|
||||
* buffers allocated temporarily for reading directories
|
||||
* FUSE nodes
|
||||
* never-forget-nodes=true nodes
|
||||
* never-forget-nodes=true retained nodes, until unlinked or removed
|
||||
|
||||
|
||||
## Management
|
||||
|
||||
@@ -14,6 +14,11 @@ getting and setting config and issuing certain commands. The other are
|
||||
files found on the filesystem for querying certain `mergerfs` specific
|
||||
information about them.
|
||||
|
||||
For a browser-based interface to inspect and change runtime settings,
|
||||
manage branches and policies, and issue commands, see
|
||||
[mergerfs-webui](tooling.md#mergerfs-webui). Runtime changes still need
|
||||
to be saved in startup configuration if they should persist.
|
||||
|
||||
|
||||
## .mergerfs pseudo file
|
||||
|
||||
@@ -110,7 +115,7 @@ situation and continue to be able to access the xattr interface.
|
||||
|
||||
```
|
||||
[trapexit:/mnt/mergerfs] $ getfattr -d .mergerfs
|
||||
user.mergerfs.branches="/mnt/a=RW:/mnt/b=RW"
|
||||
user.mergerfs.branches="/mnt/hdd/disk0=RW:/mnt/hdd/disk1=RW"
|
||||
user.mergerfs.minfreespace="4294967295"
|
||||
user.mergerfs.moveonenospc="false"
|
||||
...
|
||||
@@ -122,9 +127,9 @@ user.mergerfs.category.search="mfs"
|
||||
[trapexit:/mnt/mergerfs] $ getfattr -n user.mergerfs.category.create .mergerfs
|
||||
user.mergerfs.category.search="prfd"
|
||||
|
||||
[trapexit:/mnt/mergerfs] $ setfattr -n user.mergerfs.branches -v '+</mnt/c=RO' .mergerfs
|
||||
[trapexit:/mnt/mergerfs] $ setfattr -n user.mergerfs.branches -v '+</mnt/hdd/disk2=RO' .mergerfs
|
||||
[trapexit:/mnt/mergerfs] $ getfattr -n user.mergerfs.branches .mergerfs
|
||||
user.mergerfs.branches="/mnt/c=RO:/mnt/a=RW:/mnt/b=RW"
|
||||
user.mergerfs.branches="/mnt/hdd/disk2=RO:/mnt/hdd/disk0=RW:/mnt/hdd/disk1=RW"
|
||||
```
|
||||
|
||||
### Commands
|
||||
@@ -172,7 +177,7 @@ to set them will result in an error.
|
||||
|
||||
```
|
||||
[trapexit:/mnt/mergerfs] $ getfattr -n user.mergerfs.fullpath test
|
||||
user.mergerfs.fullpath="/mnt/a/test"
|
||||
user.mergerfs.fullpath="/mnt/hdd/disk0/test"
|
||||
[trapexit:/mnt/mergerfs] $ getfattr -n user.mergerfs.allpaths .
|
||||
user.mergerfs.allpaths="/mnt/a\000/mnt/b\000/mnt/c"
|
||||
user.mergerfs.allpaths="/mnt/hdd/disk0\000/mnt/hdd/disk1\000/mnt/hdd/disk2"
|
||||
```
|
||||
|
||||
@@ -135,11 +135,33 @@ docker pull ghcr.io/trapexit/mergerfs:TAG
|
||||
docker run --device=/dev/fuse --cap-add=SYS_ADMIN -v /mnt/to-merge:/mnt/to-merge:rshared -v /mnt/mergerfs:/mnt/mergerfs:z,shared ghcr.io/trapexit/mergerfs:TAG -f '/mnt/to-merge/*' '/mnt/mergerfs'
|
||||
```
|
||||
|
||||
or with docker compose
|
||||
|
||||
```
|
||||
services:
|
||||
mergerfs:
|
||||
image: ghcr.io/trapexit/mergerfs:TAG
|
||||
container_name: mergerfs
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse:/dev/fuse
|
||||
command: -f '/mnt/to-merge/*' '/mnt/mergerfs'
|
||||
volumes:
|
||||
- /mnt/to-merge:/mnt/to-merge:rshared
|
||||
- /mnt/mergerfs:/mnt/mergerfs:z,shared
|
||||
```
|
||||
|
||||
* `--device=/dev/fuse`: Pass in host FUSE device.
|
||||
* `--cap-add=SYS_ADMIN`: Give proper permissions to mount.
|
||||
* `-v /mnt/mergerfs:/mnt/mergerfs:z,shared`: `z` and `shared` will
|
||||
allow the mount to be shared between the container and host and
|
||||
therefore other containers.
|
||||
therefore other containers. Note that once mergerfs is mounted
|
||||
inside the container the resulting FUSE filesystem does not
|
||||
support SELinux relabeling. See [Does mergerfs work with SELinux
|
||||
relabeling?](../faq/compatibility_and_integration.md#does-mergerfs-work-with-selinux-relabeling)
|
||||
* `-v /mnt/to-merge:/mnt/to-merge:rshared`: The mounts from the host
|
||||
to merge within the container.
|
||||
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
* If you do not see some directories and files you expect, policies
|
||||
seem to skip branches, you get strange permission errors, etc. be
|
||||
sure the underlying filesystems' permissions are all the same. Use
|
||||
`mergerfs.fsck` to audit the filesystem for out of sync permissions.
|
||||
[fsck.mergerfs](tooling.md#fsckmergerfs) to audit the filesystem for
|
||||
out of sync permissions.
|
||||
* If you still have permission issues be sure you are using POSIX
|
||||
compliant filesystems. mergerfs doesn't generally make exceptions
|
||||
for FAT, NTFS, or other non-POSIX filesystem.
|
||||
|
||||
+27
-5
@@ -1,5 +1,28 @@
|
||||
# Tooling
|
||||
|
||||
## mergerfs-webui
|
||||
|
||||
[mergerfs-webui](https://github.com/trapexit/mergerfs-webui) is a
|
||||
browser-based tool for viewing and managing mergerfs mounts on Linux.
|
||||
It is a separate project with a single executable containing both the
|
||||
web server and the web page.
|
||||
|
||||
It can:
|
||||
|
||||
* Install mergerfs and create mounts through `/etc/fstab` or systemd
|
||||
`.mount` units.
|
||||
* Discover running mounts, mount and unmount them, and inspect and
|
||||
change their runtime settings, branches, and policies.
|
||||
* Edit startup configuration in `/etc/fstab`, systemd `.mount` units,
|
||||
and referenced mergerfs ini files, with previews of structured
|
||||
changes before saving.
|
||||
* Issue runtime commands and set up the web UI as a systemd service.
|
||||
|
||||
[Runtime changes](runtime_interface.md) are not automatically
|
||||
persistent; startup configuration must also be updated if changes
|
||||
should survive a remount.
|
||||
|
||||
|
||||
## mergerfs.collect-info
|
||||
|
||||
A tool included in recent releases of `mergerfs` which collects
|
||||
@@ -105,7 +128,8 @@ to `.bashrc`.
|
||||
|
||||
### Docker and Podman usage
|
||||
|
||||
Assume `/mnt/fs0` and `/mnt/fs1` are pooled with mergerfs at `/media`.
|
||||
Assume `/mnt/hdd/disk0` and `/mnt/hdd/disk1` are pooled with mergerfs
|
||||
at `/media`.
|
||||
|
||||
All mergerfs branch paths _must_ be bind mounted into the container at
|
||||
the same path as found on the host so the preload library can see
|
||||
@@ -134,8 +158,8 @@ docker run \
|
||||
-e LD_PRELOAD=/usr/lib/mergerfs/preload.so \
|
||||
-v /usr/lib/mergerfs/preload.so:/usr/lib/mergerfs/preload.so:ro \
|
||||
-v /media:/media \
|
||||
-v /mnt/fs0:/mnt/fs0 \
|
||||
-v /mnt/fs1:/mnt/fs1 \
|
||||
-v /mnt/hdd/disk0:/mnt/hdd/disk0 \
|
||||
-v /mnt/hdd/disk1:/mnt/hdd/disk1 \
|
||||
ubuntu:latest \
|
||||
bash
|
||||
```
|
||||
@@ -160,8 +184,6 @@ Environment=LD_PRELOAD=/usr/lib/mergerfs/preload.so
|
||||
behaviors that can be build on top of mergerfs. They may not
|
||||
have all the features you are looking for.**
|
||||
* mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime
|
||||
* mergerfs.fsck: Provides permissions and ownership auditing and
|
||||
the ability to fix them (should use `fsck.mergerfs` instead)
|
||||
* mergerfs.dedup: Will help identify and optionally remove duplicate files
|
||||
* mergerfs.dup: Ensure there are at least N copies of a file across the pool
|
||||
* mergerfs.balance: Rebalance files across filesystems by moving them from the most filled to the least filled
|
||||
|
||||
@@ -103,6 +103,7 @@ nav:
|
||||
- Tips and Notes: tips_notes.md
|
||||
- Known Issues and Bugs: known_issues_bugs.md
|
||||
- Project Comparisons: project_comparisons.md
|
||||
- Migration: migration.md
|
||||
- Benchmarking: benchmarking.md
|
||||
- Performance: performance.md
|
||||
- Tooling: tooling.md
|
||||
@@ -115,6 +116,7 @@ nav:
|
||||
- faq/compatibility_and_integration.md
|
||||
- faq/technical_behavior_and_limitations.md
|
||||
- faq/have_you_considered.md
|
||||
- faq/downsides.md
|
||||
- faq/limit_drive_spinup.md
|
||||
- Related Projects: related_projects.md
|
||||
- Media and Publicity: media_and_publicity.md
|
||||
|
||||
+68
-28
@@ -28,8 +28,10 @@
|
||||
#include "str.hpp"
|
||||
#include "syslog.hpp"
|
||||
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <shared_mutex>
|
||||
#include <string>
|
||||
|
||||
#include <fnmatch.h>
|
||||
|
||||
@@ -437,24 +439,44 @@ Branches::from_string(const std::string_view str_)
|
||||
Branches::Ptr impl;
|
||||
Branches::Ptr new_impl;
|
||||
|
||||
impl = std::atomic_load(&_impl);
|
||||
|
||||
new_impl = std::make_shared<Branches::Impl>(&minfreespace);
|
||||
*new_impl = *impl;
|
||||
|
||||
rv = new_impl->from_string(str_);
|
||||
if(rv < 0)
|
||||
return rv;
|
||||
for(;;)
|
||||
{
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
impl = _impl;
|
||||
}
|
||||
|
||||
std::atomic_store(&_impl,new_impl);
|
||||
*new_impl = *impl;
|
||||
|
||||
return 0;
|
||||
rv = new_impl->from_string(str_);
|
||||
if(rv < 0)
|
||||
return rv;
|
||||
|
||||
{
|
||||
std::unique_lock<std::shared_mutex> lk(_mutex);
|
||||
if(impl != _impl)
|
||||
continue;
|
||||
|
||||
_impl = std::move(new_impl);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
Branches::to_string(void) const
|
||||
{
|
||||
return std::atomic_load(&_impl)->to_string();
|
||||
Branches::Ptr impl;
|
||||
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
impl = _impl;
|
||||
}
|
||||
|
||||
return impl->to_string();
|
||||
}
|
||||
|
||||
void
|
||||
@@ -462,32 +484,50 @@ Branches::find_and_set_mode_ro()
|
||||
{
|
||||
Branches::Ptr impl;
|
||||
Branches::Ptr new_impl;
|
||||
std::vector<fs::path> ro_paths;
|
||||
|
||||
impl = std::atomic_load(&_impl);
|
||||
new_impl = std::make_shared<Branches::Impl>(&minfreespace);
|
||||
|
||||
*new_impl = *impl;
|
||||
|
||||
bool changed = false;
|
||||
for(auto &branch : *new_impl)
|
||||
for(;;)
|
||||
{
|
||||
if(branch.mode != Branch::Mode::RW)
|
||||
continue;
|
||||
ro_paths.clear();
|
||||
|
||||
if(!fs::is_rofs_but_not_mounted_ro(branch.path))
|
||||
continue;
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
impl = _impl;
|
||||
}
|
||||
|
||||
SysLog::warning("branch `{}` found to be readonly - setting its mode=RO",
|
||||
branch.path.string());
|
||||
*new_impl = *impl;
|
||||
|
||||
branch.mode = Branch::Mode::RO;
|
||||
changed = true;
|
||||
for(auto &branch : *new_impl)
|
||||
{
|
||||
if(branch.mode != Branch::Mode::RW)
|
||||
continue;
|
||||
|
||||
if(!fs::is_rofs_but_not_mounted_ro(branch.path))
|
||||
continue;
|
||||
|
||||
branch.mode = Branch::Mode::RO;
|
||||
ro_paths.emplace_back(branch.path);
|
||||
}
|
||||
|
||||
if(ro_paths.empty())
|
||||
return;
|
||||
|
||||
{
|
||||
std::unique_lock<std::shared_mutex> lk(_mutex);
|
||||
if(impl != _impl)
|
||||
continue;
|
||||
|
||||
_impl = std::move(new_impl);
|
||||
}
|
||||
|
||||
for(const auto &path : ro_paths)
|
||||
SysLog::warning("branch `{}` found to be readonly - setting its mode=RO",
|
||||
path.string());
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if(not changed)
|
||||
return;
|
||||
|
||||
std::atomic_store(&_impl,new_impl);
|
||||
}
|
||||
|
||||
SrcMounts::SrcMounts(Branches &b_)
|
||||
|
||||
+14
-4
@@ -23,9 +23,9 @@
|
||||
#include "strvec.hpp"
|
||||
#include "tofrom_string.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <shared_mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -69,7 +69,8 @@ public:
|
||||
u64 minfreespace = MINFREESPACE_DEFAULT;
|
||||
|
||||
private:
|
||||
mutable std::shared_ptr<Impl> _impl;
|
||||
Branches::Ptr _impl;
|
||||
mutable std::shared_mutex _mutex;
|
||||
|
||||
public:
|
||||
Branches()
|
||||
@@ -82,8 +83,17 @@ public:
|
||||
std::string to_string(void) const final;
|
||||
|
||||
public:
|
||||
operator Ptr() const { return std::atomic_load(&_impl); }
|
||||
Ptr operator->() const { return std::atomic_load(&_impl); }
|
||||
operator Ptr() const
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
return _impl;
|
||||
}
|
||||
|
||||
Ptr operator->() const
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
return _impl;
|
||||
}
|
||||
|
||||
public:
|
||||
void find_and_set_mode_ro();
|
||||
|
||||
+12
-11
@@ -135,10 +135,6 @@ Config::Config()
|
||||
_max_background(fuse_cfg.max_background),
|
||||
_mount(mountpoint),
|
||||
_mountpoint(mountpoint),
|
||||
_never_forget_nodes(),
|
||||
_noforget(),
|
||||
_remember(fuse_cfg.remember_nodes),
|
||||
_remember_nodes(fuse_cfg.remember_nodes),
|
||||
_srcmounts(branches),
|
||||
_threads(fuse_cfg.read_thread_count),
|
||||
_uid(fuse_cfg.uid),
|
||||
@@ -147,13 +143,22 @@ Config::Config()
|
||||
|
||||
_initialized(false)
|
||||
{
|
||||
allow_idmap.ro =
|
||||
_congestion_threshold.ro =
|
||||
_max_background.ro =
|
||||
_mount.ro =
|
||||
_mountpoint.ro =
|
||||
_never_forget_nodes.ro =
|
||||
_noforget.ro =
|
||||
_remember.ro =
|
||||
_remember_nodes.ro =
|
||||
_srcmounts.ro =
|
||||
_version.ro =
|
||||
allow_idmap.ro =
|
||||
async_read.ro =
|
||||
branches_mount_timeout.ro =
|
||||
branches_mount_timeout_fail.ro =
|
||||
cache_symlinks.ro =
|
||||
cache_writeback.ro =
|
||||
_congestion_threshold.ro =
|
||||
direct_io_allow_mmap.ro =
|
||||
export_support.ro =
|
||||
fsname.ro =
|
||||
@@ -161,9 +166,6 @@ Config::Config()
|
||||
handle_killpriv.ro =
|
||||
handle_killpriv_v2.ro =
|
||||
kernel_permissions_check.ro =
|
||||
_max_background.ro =
|
||||
_mount.ro =
|
||||
_mountpoint.ro =
|
||||
nullrw.ro =
|
||||
pid.ro =
|
||||
pin_threads.ro =
|
||||
@@ -172,8 +174,6 @@ Config::Config()
|
||||
process_thread_queue_depth.ro =
|
||||
read_thread_count.ro =
|
||||
scheduling_priority.ro =
|
||||
_srcmounts.ro =
|
||||
_version.ro =
|
||||
true;
|
||||
_congestion_threshold.display =
|
||||
_gid.display =
|
||||
@@ -181,6 +181,7 @@ Config::Config()
|
||||
_mount.display =
|
||||
_noforget.display =
|
||||
_remember.display =
|
||||
_remember_nodes.display =
|
||||
_srcmounts.display =
|
||||
_threads.display =
|
||||
_uid.display =
|
||||
|
||||
+2
-2
@@ -176,8 +176,8 @@ private:
|
||||
TFSRef<fs::path> _mountpoint;
|
||||
CfgNoforget _never_forget_nodes;
|
||||
CfgNoforget _noforget;
|
||||
TFSRef<s64> _remember;
|
||||
TFSRef<s64> _remember_nodes;
|
||||
CfgNoforget _remember;
|
||||
CfgNoforget _remember_nodes;
|
||||
SrcMounts _srcmounts;
|
||||
TFSRef<int> _threads;
|
||||
TFSRef<s64> _uid;
|
||||
|
||||
+15
-9
@@ -32,18 +32,24 @@ public:
|
||||
{
|
||||
int rv;
|
||||
bool b = false;
|
||||
s64 i = 0;
|
||||
|
||||
rv = str::from(s_,&b);
|
||||
if((b == true) || s_.empty())
|
||||
if(s_.empty())
|
||||
{
|
||||
fuse_cfg.remember_nodes = -1;
|
||||
return 0;
|
||||
b = true;
|
||||
}
|
||||
else if((rv = str::from(s_,&i)) == 0)
|
||||
{
|
||||
b = (i != 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
rv = str::from(s_,&b);
|
||||
if(rv)
|
||||
return rv;
|
||||
}
|
||||
|
||||
if(rv)
|
||||
return rv;
|
||||
|
||||
fuse_cfg.remember_nodes = 0;
|
||||
fuse_cfg.remember_nodes = b;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -51,7 +57,7 @@ public:
|
||||
std::string
|
||||
to_string() const
|
||||
{
|
||||
if(fuse_cfg.remember_nodes == -1)
|
||||
if(fuse_cfg.remember_nodes)
|
||||
return "true";
|
||||
return "false";
|
||||
}
|
||||
|
||||
+12
-3
@@ -48,10 +48,10 @@
|
||||
#include "fh.hpp"
|
||||
#include "fs_path.hpp"
|
||||
|
||||
#include "mutex.hpp"
|
||||
|
||||
#include "base_types.h"
|
||||
|
||||
#include <shared_mutex>
|
||||
|
||||
|
||||
class FileInfo : public FH
|
||||
{
|
||||
@@ -97,7 +97,16 @@ public:
|
||||
int fd;
|
||||
Branch branch;
|
||||
u32 direct_io:1;
|
||||
Mutex mutex;
|
||||
// Serializes the fd state across concurrent writes on the same open
|
||||
// file. Concurrent writes happen with:
|
||||
// 1) writeback-cache + page-cache mode
|
||||
// 2) parallel_direct_writes + direct_io=true
|
||||
//
|
||||
// Common case (plain pwrite on stable fd) only needs a shared lock
|
||||
// since the kernel already serializes fd mutations. The danger is
|
||||
// the fs::dup2() rebinding when moveonenospc activates. Writers
|
||||
// take shared lock; move takes unique.
|
||||
std::shared_mutex mutex;
|
||||
};
|
||||
|
||||
inline
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2026, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "to_neg_errno.hpp"
|
||||
|
||||
#include <sys/file.h>
|
||||
|
||||
|
||||
namespace fs
|
||||
{
|
||||
static
|
||||
inline
|
||||
int
|
||||
flock(const int fd_,
|
||||
const int operation_)
|
||||
{
|
||||
int rv;
|
||||
|
||||
rv = ::flock(fd_,operation_);
|
||||
|
||||
return ::to_neg_errno(rv);
|
||||
}
|
||||
}
|
||||
+24
-16
@@ -21,8 +21,8 @@
|
||||
#include "fs_statvfs.hpp"
|
||||
#include "statvfs_util.hpp"
|
||||
|
||||
#include "mutex.hpp"
|
||||
|
||||
#include <mutex>
|
||||
#include <shared_mutex>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
@@ -38,9 +38,11 @@ struct Element
|
||||
|
||||
typedef std::unordered_map<std::string,Element> statvfs_cache;
|
||||
|
||||
static u64 g_timeout = 0;
|
||||
static statvfs_cache g_cache;
|
||||
static Mutex g_cache_lock;
|
||||
// Cache is read heavy. Use a shared mutex so concurrent reads can
|
||||
// work in parallel and only inserts or refreshes take a unique lock.
|
||||
static std::shared_mutex g_cache_mutex;
|
||||
static u64 g_timeout = 0;
|
||||
static statvfs_cache g_cache;
|
||||
|
||||
|
||||
static
|
||||
@@ -70,29 +72,35 @@ int
|
||||
fs::statvfs_cache(const std::string &path_,
|
||||
struct statvfs *st_)
|
||||
{
|
||||
int rv;
|
||||
Element *e;
|
||||
u64 now;
|
||||
|
||||
if(g_timeout == 0)
|
||||
return fs::statvfs(path_,st_);
|
||||
|
||||
rv = 0;
|
||||
now = ::_get_time();
|
||||
|
||||
mutex_lock(g_cache_lock);
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(g_cache_mutex);
|
||||
|
||||
e = &g_cache[path_];
|
||||
auto it = g_cache.find(path_);
|
||||
if((it != g_cache.end()) && ((now - it->second.time) <= g_timeout))
|
||||
{
|
||||
*st_ = it->second.st;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if((now - e->time) > g_timeout)
|
||||
int rv = 0;
|
||||
std::unique_lock<std::shared_mutex> lk(g_cache_mutex);
|
||||
|
||||
Element &e = g_cache[path_];
|
||||
if((now - e.time) > g_timeout)
|
||||
{
|
||||
e->time = now;
|
||||
rv = fs::statvfs(path_,&e->st);
|
||||
e.time = now;
|
||||
rv = fs::statvfs(path_,&e.st);
|
||||
}
|
||||
|
||||
*st_ = e->st;
|
||||
|
||||
mutex_unlock(g_cache_lock);
|
||||
*st_ = e.st;
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
+1
-1
@@ -334,7 +334,7 @@ _create(const fuse_req_ctx_t *ctx_,
|
||||
delete fi;
|
||||
ffi_->fh = 0;
|
||||
|
||||
SysLog::crit(msg);
|
||||
SysLog::crit("{}",msg);
|
||||
fmt::println(stderr,"{}",msg);
|
||||
|
||||
return -EIO;
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2026, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "fuse_flock.hpp"
|
||||
|
||||
#include "errno.hpp"
|
||||
#include "fileinfo.hpp"
|
||||
#include "fs_flock.hpp"
|
||||
#include "state.hpp"
|
||||
|
||||
#include "fuse.h"
|
||||
|
||||
|
||||
static
|
||||
int
|
||||
_flock(const int fd_,
|
||||
const int operation_)
|
||||
{
|
||||
int rv;
|
||||
|
||||
rv = fs::flock(fd_,operation_);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
// In cases where the node was opened but unlinked there will be no
|
||||
// path and no guarentee of a `fh`. It could always do the lookup but
|
||||
// why bother if the kernel has provided it to us? The reason the
|
||||
// function doesn't need to run within the visit lambda is because
|
||||
// since the request is outstanding the kernel won't be releasing the
|
||||
// node and therefore the entry will be valid over the lifetime of
|
||||
// this function.
|
||||
int
|
||||
FUSE::flock(const fuse_req_ctx_t *ctx_,
|
||||
const fuse_file_info_t *ffi_,
|
||||
int op_)
|
||||
{
|
||||
FileInfo *fi;
|
||||
|
||||
fi = state.get_fi(ctx_,ffi_->fh);
|
||||
if(not fi)
|
||||
return -EBADF;
|
||||
|
||||
return ::_flock(fi->fd,op_);
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2026, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fuse.h"
|
||||
|
||||
|
||||
namespace FUSE
|
||||
{
|
||||
int
|
||||
flock(const fuse_req_ctx_t *ctx,
|
||||
const fuse_file_info_t *ffi,
|
||||
int op);
|
||||
}
|
||||
@@ -209,8 +209,6 @@ FUSE::init(fuse_conn_info_t *conn_)
|
||||
::_want_if_capable(conn_,FUSE_CAP_ALLOW_IDMAP,&cfg.allow_idmap);
|
||||
// ::_want_if_capable(conn_,FUSE_CAP_READDIR_PLUS_AUTO);
|
||||
::_want_if_capable_max_pages(conn_,cfg);
|
||||
conn_->want &= ~FUSE_CAP_POSIX_LOCKS;
|
||||
conn_->want &= ~FUSE_CAP_FLOCK_LOCKS;
|
||||
|
||||
::_spawn_thread_to_set_readahead();
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2026, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "fuse_lock.hpp"
|
||||
|
||||
#include "errno.hpp"
|
||||
|
||||
#include "fuse.h"
|
||||
|
||||
|
||||
int
|
||||
FUSE::lock(const fuse_req_ctx_t *ctx,
|
||||
const fuse_file_info_t *ffi,
|
||||
int cmd,
|
||||
struct flock *flock)
|
||||
{
|
||||
return -ENOSYS;
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
/*
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2026, Antonio SJ Musumeci <trapexit@spawn.link>
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fuse.h"
|
||||
|
||||
namespace FUSE
|
||||
{
|
||||
int
|
||||
lock(const fuse_req_ctx_t *ctx,
|
||||
const fuse_file_info_t *ffi,
|
||||
int cmd,
|
||||
struct flock *flock);
|
||||
}
|
||||
+10
-1
@@ -367,10 +367,19 @@ _open(const fuse_req_ctx_t *ctx_,
|
||||
|
||||
// The ref increment keeps fuse_release.cpp from erasing and
|
||||
// freeing the canonical entry while we duplicate its fd.
|
||||
//
|
||||
// memory_order_relaxed is sufficient for the increment. The
|
||||
// concurrent_flat_map bucket lock held inside visit() already
|
||||
// provides the happens-before relationship with other mutators
|
||||
// of the same key (the release() erase_if() callback and other
|
||||
// open() visitors.) The lock's release/acquire semantics
|
||||
// publish the ref_count update to subsequent observers of the
|
||||
// same entry, so we only need atomicity from the fetch_add
|
||||
// itself.
|
||||
of.visit(ctx_->nodeid,
|
||||
[&](auto &v_)
|
||||
{
|
||||
v_.second.ref_count++;
|
||||
v_.second.ref_count.fetch_add(1,std::memory_order_relaxed);
|
||||
fi = v_.second.fi;
|
||||
backing_id = v_.second.backing_id;
|
||||
});
|
||||
|
||||
+21
-15
@@ -48,30 +48,29 @@ FUSE::ReadDir::ReadDir(const std::string_view s_)
|
||||
std::string
|
||||
FUSE::ReadDir::to_string() const
|
||||
{
|
||||
std::shared_ptr<std::string> str;
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
|
||||
str = std::atomic_load(&_str);
|
||||
|
||||
return (*str);
|
||||
return _str;
|
||||
}
|
||||
|
||||
int
|
||||
FUSE::ReadDir::from_string(const std::string_view str_)
|
||||
{
|
||||
std::shared_ptr<std::string> tmp_str;
|
||||
std::shared_ptr<FUSE::ReadDirBase> tmp_readdir;
|
||||
|
||||
tmp_readdir = FUSE::ReadDirFactory::make(str_);
|
||||
if(!tmp_readdir)
|
||||
return -EINVAL;
|
||||
|
||||
tmp_str = std::make_shared<std::string>(str_);
|
||||
std::atomic_store(&_str,tmp_str);
|
||||
{
|
||||
std::unique_lock<std::shared_mutex> lk(_mutex);
|
||||
|
||||
if(_initialized == false)
|
||||
return 0;
|
||||
_str = str_;
|
||||
if(_initialized == false)
|
||||
return 0;
|
||||
|
||||
std::atomic_store(&_impl,tmp_readdir);
|
||||
_impl = std::move(tmp_readdir);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -114,7 +113,10 @@ FUSE::ReadDir::operator()(const fuse_req_ctx_t *ctx_,
|
||||
int rv;
|
||||
std::shared_ptr<FUSE::ReadDirBase> readdir;
|
||||
|
||||
readdir = std::atomic_load(&_impl);
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> lk(_mutex);
|
||||
readdir = _impl;
|
||||
}
|
||||
if(!readdir)
|
||||
fatal::abort("readdir impl is null");
|
||||
|
||||
@@ -128,10 +130,14 @@ FUSE::ReadDir::operator()(const fuse_req_ctx_t *ctx_,
|
||||
void
|
||||
FUSE::ReadDir::initialize()
|
||||
{
|
||||
std::shared_ptr<std::string> str;
|
||||
std::string str;
|
||||
|
||||
str = std::atomic_load(&_str);
|
||||
_initialized = true;
|
||||
{
|
||||
std::unique_lock<std::shared_mutex> lk(_mutex);
|
||||
|
||||
from_string(*str);
|
||||
str = _str;
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
from_string(str);
|
||||
}
|
||||
|
||||
@@ -22,9 +22,9 @@
|
||||
#include "tofrom_string.hpp"
|
||||
#include "fuse_readdir_base.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <shared_mutex>
|
||||
|
||||
|
||||
namespace FUSE
|
||||
@@ -59,8 +59,9 @@ namespace FUSE
|
||||
void initialize();
|
||||
|
||||
private:
|
||||
std::atomic<bool> _initialized = false;
|
||||
std::shared_ptr<std::string> _str;
|
||||
bool _initialized = false;
|
||||
mutable std::shared_mutex _mutex;
|
||||
std::string _str;
|
||||
std::shared_ptr<FUSE::ReadDirBase> _impl;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -43,6 +43,12 @@ FUSE::ReadDirCOR::~ReadDirCOR()
|
||||
#include "fuse_readdir_cor_readdir.icpp"
|
||||
#endif
|
||||
|
||||
// Uses a per-call mutex. Lifetime is one readdir operation. It
|
||||
// serializes the dirent-merge step (dedup'ing of names + append to
|
||||
// dirents list) across at-most-branches-worth of parallel worker
|
||||
// tasks. Per-branch opendir + getdents64 run fully outside the
|
||||
// critical section, so the lock holds only long enough to merge each
|
||||
// batch.
|
||||
static
|
||||
inline
|
||||
int
|
||||
|
||||
+12
-2
@@ -81,11 +81,21 @@ FUSE::release(cu64 nodeid_,
|
||||
FileInfo *canonical_fi = nullptr;
|
||||
FileInfo *fh_fi_to_free = nullptr;
|
||||
|
||||
// memory_order_acq_rel on the decrement gives us the canonical
|
||||
// refcount-publication pattern:
|
||||
// - release: prior writes to the OpenFile by this thread are
|
||||
// visible to whichever thread observes the final 0.
|
||||
// - acquire: the thread that drops the last ref acquires every
|
||||
// prior ref holder's writes before tearing down the entry.
|
||||
// The concurrent_flat_map bucket lock already serializes concurrent
|
||||
// visitors of this key so the acq_rel is good for any observer that
|
||||
// might examine ref_count outside a visit().
|
||||
of.erase_if(nodeid_,
|
||||
[&](auto &v_)
|
||||
{
|
||||
v_.second.ref_count--;
|
||||
if(v_.second.ref_count > 0)
|
||||
const int prev =
|
||||
v_.second.ref_count.fetch_sub(1,std::memory_order_acq_rel);
|
||||
if(prev > 1)
|
||||
{
|
||||
if(fi_ != v_.second.fi)
|
||||
fh_fi_to_free = fi_;
|
||||
|
||||
+50
-46
@@ -33,6 +33,9 @@
|
||||
|
||||
#include "fuse.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <shared_mutex>
|
||||
|
||||
|
||||
static
|
||||
bool
|
||||
@@ -117,47 +120,11 @@ _move_and_pwriten(char const *buf_,
|
||||
// N bytes written (short writes included)
|
||||
// -errno on error
|
||||
// See libfuse/include/fuse.h for more details
|
||||
static
|
||||
int
|
||||
_write_direct_io(const char *buf_,
|
||||
const size_t count_,
|
||||
const off_t offset_,
|
||||
FileInfo *fi_)
|
||||
{
|
||||
ssize_t rv;
|
||||
|
||||
rv = fs::pwrite(fi_->fd,buf_,count_,offset_);
|
||||
if(::_out_of_space(rv))
|
||||
rv = ::_move_and_pwrite(buf_,count_,offset_,fi_,rv);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
// When not in direct_io mode write's return value is more complex.
|
||||
// 0 or less than `count` on EOF
|
||||
// `count` on non-errors
|
||||
// -errno on error
|
||||
// See libfuse/include/fuse.h for more details
|
||||
static
|
||||
int
|
||||
_write_cached(const char *buf_,
|
||||
const size_t count_,
|
||||
const off_t offset_,
|
||||
FileInfo *fi_)
|
||||
{
|
||||
int err;
|
||||
ssize_t rv;
|
||||
|
||||
rv = fs::pwriten(fi_->fd,buf_,count_,offset_,&err);
|
||||
if(err == 0)
|
||||
return rv;
|
||||
if(err && !::_out_of_space(err))
|
||||
return err;
|
||||
|
||||
rv = ::_move_and_pwriten(buf_,count_,offset_,fi_,err,rv);
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
static
|
||||
int
|
||||
@@ -175,19 +142,56 @@ _write(const fuse_req_ctx_t *ctx_,
|
||||
|
||||
// Concurrent writes can only happen if:
|
||||
// 1) writeback-cache is enabled and using page caching
|
||||
// 2) parallel_direct_writes is enabled and file has `direct_io=true`
|
||||
// Will look into selectively locking in the future
|
||||
// A reader/writer lock would probably be the best option given
|
||||
// the expense of the write itself in comparison. Alternatively,
|
||||
// could change the move file behavior to use a known target file
|
||||
// and have threads use O_EXCL and back off and wait for the
|
||||
// transfer to complete before retrying.
|
||||
mutex_lockguard(fi->mutex);
|
||||
// 2) parallel_direct_writes is enabled and file has
|
||||
// `direct_io=true`
|
||||
|
||||
ssize_t rv;
|
||||
|
||||
if(fi->direct_io)
|
||||
return ::_write_direct_io(buf_,count_,offset_,fi);
|
||||
{
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> slk(fi->mutex);
|
||||
rv = fs::pwrite(fi->fd,buf_,count_,offset_);
|
||||
}
|
||||
|
||||
return ::_write_cached(buf_,count_,offset_,fi);
|
||||
if(not ::_out_of_space(rv))
|
||||
return rv;
|
||||
|
||||
std::unique_lock<std::shared_mutex> ulk(fi->mutex);
|
||||
// Re-check under exclusive lock: another writer may have
|
||||
// already moved the file and retired. If so the pwrite should
|
||||
// now succeed without our own move call.
|
||||
rv = fs::pwrite(fi->fd,buf_,count_,offset_);
|
||||
if(::_out_of_space(rv))
|
||||
rv = ::_move_and_pwrite(buf_,count_,offset_,fi,rv);
|
||||
return rv;
|
||||
}
|
||||
else
|
||||
{
|
||||
int err;
|
||||
ssize_t written;
|
||||
|
||||
{
|
||||
std::shared_lock<std::shared_mutex> slk(fi->mutex);
|
||||
written = fs::pwriten(fi->fd,buf_,count_,offset_,&err);
|
||||
}
|
||||
|
||||
if(err == 0)
|
||||
return written;
|
||||
if(err && not ::_out_of_space(err))
|
||||
return err;
|
||||
|
||||
std::unique_lock<std::shared_mutex> ulk(fi->mutex);
|
||||
// Re-check under exclusive lock as another move may have
|
||||
// already run.
|
||||
written = fs::pwriten(fi->fd,buf_,count_,offset_,&err);
|
||||
if(err == 0)
|
||||
return written;
|
||||
if(err && not ::_out_of_space(err))
|
||||
return err;
|
||||
|
||||
return ::_move_and_pwriten(buf_,count_,offset_,fi,err,written);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
|
||||
+8
-12
@@ -32,8 +32,12 @@ enum
|
||||
IOPRIO_WHO_PROCESS = 1
|
||||
};
|
||||
|
||||
namespace ioprio
|
||||
{
|
||||
std::atomic<bool> _enabled{false};
|
||||
}
|
||||
|
||||
thread_local int ioprio::SetFrom::thread_prio = -1;
|
||||
bool _enabled = false;
|
||||
|
||||
int
|
||||
ioprio::get(const int who_)
|
||||
@@ -69,22 +73,14 @@ ioprio::set(const int who_,
|
||||
void
|
||||
ioprio::enable(const bool enable_)
|
||||
{
|
||||
_enabled = enable_;
|
||||
_enabled.store(enable_,std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
bool
|
||||
ioprio::enabled()
|
||||
{
|
||||
return _enabled;
|
||||
}
|
||||
|
||||
ioprio::SetFrom::SetFrom(const pid_t pid_)
|
||||
void
|
||||
ioprio::SetFrom::_slow_apply(const pid_t pid_)
|
||||
{
|
||||
int client_prio;
|
||||
|
||||
if(!_enabled)
|
||||
return;
|
||||
|
||||
client_prio = ioprio::get(pid_);
|
||||
if(client_prio < 0)
|
||||
return;
|
||||
|
||||
+25
-2
@@ -18,13 +18,27 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
|
||||
namespace ioprio
|
||||
{
|
||||
// Exposed so the disabled-path check in SetFrom can inline into
|
||||
// every fuse_read/fuse_write as an atomic load + branch, with no
|
||||
// function overhead.
|
||||
extern std::atomic<bool> _enabled;
|
||||
|
||||
[[gnu::always_inline]]
|
||||
inline
|
||||
bool
|
||||
enabled()
|
||||
{
|
||||
return _enabled.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void enable(const bool);
|
||||
bool enabled();
|
||||
|
||||
int get(const int who);
|
||||
int set(const int who, const int ioprio);
|
||||
@@ -33,6 +47,15 @@ namespace ioprio
|
||||
{
|
||||
static thread_local int thread_prio;
|
||||
|
||||
SetFrom(const pid_t pid);
|
||||
[[gnu::always_inline]]
|
||||
inline
|
||||
SetFrom(const pid_t pid_)
|
||||
{
|
||||
if(ioprio::enabled())
|
||||
_slow_apply(pid_);
|
||||
}
|
||||
|
||||
private:
|
||||
void _slow_apply(const pid_t);
|
||||
};
|
||||
};
|
||||
|
||||
+1
-5
@@ -46,7 +46,6 @@
|
||||
#include "fuse_fchmod.hpp"
|
||||
#include "fuse_fchown.hpp"
|
||||
#include "fuse_fgetattr.hpp"
|
||||
#include "fuse_flock.hpp"
|
||||
#include "fuse_flush.hpp"
|
||||
#include "fuse_fsync.hpp"
|
||||
#include "fuse_fsyncdir.hpp"
|
||||
@@ -58,7 +57,6 @@
|
||||
#include "fuse_ioctl.hpp"
|
||||
#include "fuse_link.hpp"
|
||||
#include "fuse_listxattr.hpp"
|
||||
#include "fuse_lock.hpp"
|
||||
#include "fuse_mkdir.hpp"
|
||||
#include "fuse_mknod.hpp"
|
||||
#include "fuse_open.hpp"
|
||||
@@ -111,7 +109,6 @@ _get_fuse_operations(struct fuse_operations &ops_,
|
||||
ops_.fchmod = FUSE::fchmod;
|
||||
ops_.fchown = FUSE::fchown;
|
||||
ops_.fgetattr = FUSE::fgetattr;
|
||||
ops_.flock = FUSE::flock;
|
||||
ops_.flush = FUSE::flush;
|
||||
ops_.fsync = FUSE::fsync;
|
||||
ops_.fsyncdir = FUSE::fsyncdir;
|
||||
@@ -123,7 +120,6 @@ _get_fuse_operations(struct fuse_operations &ops_,
|
||||
ops_.ioctl = FUSE::ioctl;
|
||||
ops_.link = FUSE::link;
|
||||
ops_.listxattr = FUSE::listxattr;
|
||||
ops_.lock = FUSE::lock;
|
||||
ops_.mkdir = FUSE::mkdir;
|
||||
ops_.mknod = FUSE::mknod;
|
||||
ops_.open = FUSE::open;
|
||||
@@ -305,7 +301,7 @@ _warn_if_not_root()
|
||||
"mergerfs is not running as root"
|
||||
" and may not work correctly";
|
||||
fmt::println(stderr,"* WARNING: {}",s);
|
||||
SysLog::warning(s);
|
||||
SysLog::warning("{}",s);
|
||||
}
|
||||
|
||||
int
|
||||
|
||||
+5
-1
@@ -103,8 +103,12 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
// Move constructor runs during initial insertion into the
|
||||
// concurrent_flat_map before the new entry is reachable by the
|
||||
// other threads. There is no concurrent observer of the moved
|
||||
// value at this point so relaxed load is sufficient.
|
||||
OpenFile(OpenFile &&o_) noexcept
|
||||
: ref_count(o_.ref_count.load()),
|
||||
: ref_count(o_.ref_count.load(std::memory_order_relaxed)),
|
||||
backing_id(o_.backing_id),
|
||||
fi(o_.fi)
|
||||
{
|
||||
|
||||
@@ -100,8 +100,8 @@ Legend:
|
||||
| Option | Integration tests | Unit tests |
|
||||
|---|---|---|
|
||||
| `lazy-umount-mountpoint` | Live-upgrade remount scenario with stale previous mount | Startup control flow and error handling |
|
||||
| `remember-nodes` | NFS-like lookup stability over configured retention interval | Node cache retention/expiry logic |
|
||||
| `noforget` | No node expiry over extended period | Infinite retention mode mapping |
|
||||
| `remember-nodes` | NFS-like lookup stability until unlink/remove/rename replacement | Boolean and numeric compatibility parsing |
|
||||
| `noforget` | Retained nodes released on unlink/remove/rename replacement | Alias mapping to retained-node mode |
|
||||
| `allow-idmap` | UID/GID mapping behavior in idmapped mounts | Init flag propagation and compatibility checks |
|
||||
| `debug` | FUSE trace generation enabled/disabled | Debug option parse and logger toggling |
|
||||
| `log.file` | Trace output to stderr vs file path | Path parser and file sink setup |
|
||||
|
||||
+7
-1
@@ -6,10 +6,13 @@ and compare behavior against equivalent native syscalls on a local filesystem.
|
||||
The suite is "soft POSIX": it focuses on practical parity for return values,
|
||||
`errno`, and key side effects, including common error paths.
|
||||
|
||||
Each test mounts its own ephemeral mergerfs instance with random temporary
|
||||
branches and cleans everything up automatically.
|
||||
|
||||
Run with:
|
||||
|
||||
```bash
|
||||
python3 tests/run-tests /mnt/mergerfs/
|
||||
python3 tests/run-tests
|
||||
```
|
||||
|
||||
## Coverage Matrix
|
||||
@@ -69,3 +72,6 @@ Across tests, parity checks include combinations of:
|
||||
- Tests that cannot exercise a behavior in the current environment may exit 77
|
||||
and are reported as `SKIP` by `tests/run-tests`.
|
||||
- The xattr test uses a `user.*` key and assumes user xattrs are enabled.
|
||||
- Tests mount their own ephemeral mergerfs instances using the `mergerfs_mount`
|
||||
context manager in `posix_parity.py`. Random directories are created and
|
||||
cleaned up automatically.
|
||||
|
||||
+50
-54
@@ -2,80 +2,76 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_branches
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import mergerfs_branches
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_api_xattr <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
|
||||
try:
|
||||
ctrl = join(mount, ".mergerfs")
|
||||
if not os.path.exists(ctrl):
|
||||
return fail(".mergerfs control file not found")
|
||||
with mergerfs_mount() as (mount, branches):
|
||||
ctrl = join(mount, ".mergerfs")
|
||||
if not os.path.exists(ctrl):
|
||||
return fail(".mergerfs control file not found")
|
||||
|
||||
branches = mergerfs_branches(mount)
|
||||
if not branches:
|
||||
return fail("no branches found")
|
||||
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
merge_file = join(merge_base, "api_test")
|
||||
touch(merge_file, b"test")
|
||||
if not branches:
|
||||
return fail("no branches found")
|
||||
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
fullpath = os.getxattr(merge_file, "user.mergerfs.fullpath")
|
||||
fullpath_str = fullpath.decode("utf-8", errors="surrogateescape")
|
||||
if not fullpath_str:
|
||||
return fail("fullpath xattr is empty")
|
||||
if not os.path.exists(fullpath_str):
|
||||
return fail(f"fullpath '{fullpath_str}' does not exist on disk")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr fullpath: errno={exc.errno}")
|
||||
merge_file = join(merge_base, "api_test")
|
||||
touch(merge_file, b"test")
|
||||
|
||||
try:
|
||||
relpath = os.getxattr(merge_file, "user.mergerfs.relpath")
|
||||
relpath_str = relpath.decode("utf-8", errors="surrogateescape")
|
||||
if not relpath_str:
|
||||
return fail("relpath xattr is empty")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr relpath: errno={exc.errno}")
|
||||
try:
|
||||
fullpath = os.getxattr(merge_file, "user.mergerfs.fullpath")
|
||||
fullpath_str = fullpath.decode("utf-8", errors="surrogateescape")
|
||||
if not fullpath_str:
|
||||
return fail("fullpath xattr is empty")
|
||||
if not os.path.exists(fullpath_str):
|
||||
return fail(f"fullpath '{fullpath_str}' does not exist on disk")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr fullpath: errno={exc.errno}")
|
||||
|
||||
try:
|
||||
basepath = os.getxattr(merge_file, "user.mergerfs.basepath")
|
||||
basepath_str = basepath.decode("utf-8", errors="surrogateescape")
|
||||
if not basepath_str:
|
||||
return fail("basepath xattr is empty")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr basepath: errno={exc.errno}")
|
||||
try:
|
||||
relpath = os.getxattr(merge_file, "user.mergerfs.relpath")
|
||||
relpath_str = relpath.decode("utf-8", errors="surrogateescape")
|
||||
if not relpath_str:
|
||||
return fail("relpath xattr is empty")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr relpath: errno={exc.errno}")
|
||||
|
||||
try:
|
||||
allpaths = os.getxattr(merge_file, "user.mergerfs.allpaths")
|
||||
allpaths_str = allpaths.decode("utf-8", errors="surrogateescape")
|
||||
if not allpaths_str:
|
||||
return fail("allpaths xattr is empty")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr allpaths: errno={exc.errno}")
|
||||
try:
|
||||
basepath = os.getxattr(merge_file, "user.mergerfs.basepath")
|
||||
basepath_str = basepath.decode("utf-8", errors="surrogateescape")
|
||||
if not basepath_str:
|
||||
return fail("basepath xattr is empty")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr basepath: errno={exc.errno}")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
try:
|
||||
allpaths = os.getxattr(merge_file, "user.mergerfs.allpaths")
|
||||
allpaths_str = allpaths.decode("utf-8", errors="surrogateescape")
|
||||
if not allpaths_str:
|
||||
return fail("allpaths xattr is empty")
|
||||
except OSError as exc:
|
||||
return fail(f"getxattr allpaths: errno={exc.errno}")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
except Exception as exc:
|
||||
return fail(str(exc))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -10,6 +10,7 @@ from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_branches
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -56,96 +57,92 @@ def setup_paths(branches, mount, rel_base, name):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_cfg_link_rename_exdev <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
rel_base = f".mergerfs_exdev_test_{os.getpid()}_{uuid.uuid4().hex}"
|
||||
branches = mergerfs_branches(mount)
|
||||
if len(branches) < 2:
|
||||
print("EXDEV fallback test requires at least 2 branches", end="")
|
||||
return SKIP
|
||||
branches = branches[:2]
|
||||
if os.stat(branches[0]).st_dev == os.stat(branches[1]).st_dev:
|
||||
print("EXDEV fallback test requires branches on different devices", end="")
|
||||
return SKIP
|
||||
|
||||
try:
|
||||
try:
|
||||
orig_create = mergerfs_get_option(mount, "category.create")
|
||||
orig_link = mergerfs_get_option(mount, "link-exdev")
|
||||
orig_rename = mergerfs_get_option(mount, "rename-exdev")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
print("EXDEV fallback runtime options unavailable", end="")
|
||||
return SKIP
|
||||
|
||||
try:
|
||||
# Maximize chance of EXDEV path by path-preserving create policy.
|
||||
mergerfs_set_option(mount, "category.create", "epmfs")
|
||||
|
||||
# link-exdev passthrough => EXDEV when cross-device path preserving applies.
|
||||
src, dst, src_base = setup_paths(branches, mount, rel_base, "link_pass")
|
||||
mergerfs_set_option(mount, "link-exdev", "passthrough")
|
||||
err = get_errno(lambda: os.link(src, dst + ".link.pass"))
|
||||
if err != errno.EXDEV:
|
||||
print(f"link-exdev passthrough did not exercise EXDEV, errno={err}", end="")
|
||||
with mergerfs_mount(num_branches=2) as (mount, branches):
|
||||
rel_base = f".mergerfs_exdev_test_{os.getpid()}_{uuid.uuid4().hex}"
|
||||
if len(branches) < 2:
|
||||
print("EXDEV fallback test requires at least 2 branches", end="")
|
||||
return SKIP
|
||||
branches = branches[:2]
|
||||
if os.stat(branches[0]).st_dev == os.stat(branches[1]).st_dev:
|
||||
print("EXDEV fallback test requires branches on different devices", end="")
|
||||
return SKIP
|
||||
|
||||
for mode in ("rel-symlink", "abs-base-symlink", "abs-pool-symlink"):
|
||||
src, dst, src_base = setup_paths(branches, mount, rel_base, f"link_{mode}")
|
||||
mergerfs_set_option(mount, "link-exdev", mode)
|
||||
link_path = dst + ".link"
|
||||
err = get_errno(lambda: os.link(src, link_path))
|
||||
if err != 0:
|
||||
return fail(f"link-exdev={mode} returned errno={err}")
|
||||
err_msg = require_symlink(link_path, f"link-exdev={mode}")
|
||||
if err_msg:
|
||||
return fail(err_msg)
|
||||
resolved = symlink_resolves(link_path)
|
||||
if mode == "abs-base-symlink" and resolved != os.path.realpath(src_base):
|
||||
return fail(f"link-exdev={mode}: target {resolved!r} != {os.path.realpath(src_base)!r}")
|
||||
if mode != "abs-base-symlink" and resolved != os.path.realpath(src):
|
||||
return fail(f"link-exdev={mode}: target {resolved!r} != {os.path.realpath(src)!r}")
|
||||
|
||||
# rename-exdev passthrough => EXDEV in cross-device path preserving case.
|
||||
src_ren, dst_ren, _ = setup_paths(branches, mount, rel_base, "rename_pass")
|
||||
mergerfs_set_option(mount, "rename-exdev", "passthrough")
|
||||
err = get_errno(lambda: os.rename(src_ren, dst_ren + ".ren.pass"))
|
||||
if err != errno.EXDEV:
|
||||
print(f"rename-exdev passthrough did not exercise EXDEV, errno={err}", end="")
|
||||
return SKIP
|
||||
|
||||
for mode in ("rel-symlink", "abs-symlink"):
|
||||
src_ren, dst_ren, _ = setup_paths(branches, mount, rel_base, f"rename_{mode}")
|
||||
mergerfs_set_option(mount, "rename-exdev", mode)
|
||||
ren_path = dst_ren + ".ren"
|
||||
err = get_errno(lambda: os.rename(src_ren, ren_path))
|
||||
if err != 0:
|
||||
return fail(f"rename-exdev={mode} returned errno={err}")
|
||||
err_msg = require_symlink(ren_path, f"rename-exdev={mode}")
|
||||
if err_msg:
|
||||
return fail(err_msg)
|
||||
target = symlink_resolves(ren_path)
|
||||
if ".mergerfs_rename_exdev" not in target:
|
||||
return fail(f"rename-exdev={mode}: target not moved under .mergerfs_rename_exdev: {target!r}")
|
||||
with open(ren_path, "rb") as fp:
|
||||
if fp.read() != b"src":
|
||||
return fail(f"rename-exdev={mode}: symlink target content mismatch")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return SKIP
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "category.create", orig_create)
|
||||
mergerfs_set_option(mount, "link-exdev", orig_link)
|
||||
mergerfs_set_option(mount, "rename-exdev", orig_rename)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
orig_create = mergerfs_get_option(mount, "category.create")
|
||||
orig_link = mergerfs_get_option(mount, "link-exdev")
|
||||
orig_rename = mergerfs_get_option(mount, "rename-exdev")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
print("EXDEV fallback runtime options unavailable", end="")
|
||||
return SKIP
|
||||
|
||||
return 0
|
||||
finally:
|
||||
for branch in branches:
|
||||
cleanup_dir(join(branch, rel_base))
|
||||
try:
|
||||
mergerfs_set_option(mount, "category.create", "epmfs")
|
||||
|
||||
src, dst, src_base = setup_paths(branches, mount, rel_base, "link_pass")
|
||||
mergerfs_set_option(mount, "link-exdev", "passthrough")
|
||||
err = get_errno(lambda: os.link(src, dst + ".link.pass"))
|
||||
if err != errno.EXDEV:
|
||||
print(f"link-exdev passthrough did not exercise EXDEV, errno={err}", end="")
|
||||
return SKIP
|
||||
|
||||
for mode in ("rel-symlink", "abs-base-symlink", "abs-pool-symlink"):
|
||||
src, dst, src_base = setup_paths(branches, mount, rel_base, f"link_{mode}")
|
||||
mergerfs_set_option(mount, "link-exdev", mode)
|
||||
link_path = dst + ".link"
|
||||
err = get_errno(lambda: os.link(src, link_path))
|
||||
if err != 0:
|
||||
return fail(f"link-exdev={mode} returned errno={err}")
|
||||
err_msg = require_symlink(link_path, f"link-exdev={mode}")
|
||||
if err_msg:
|
||||
return fail(err_msg)
|
||||
resolved = symlink_resolves(link_path)
|
||||
if mode == "abs-base-symlink" and resolved != os.path.realpath(src_base):
|
||||
return fail(f"link-exdev={mode}: target {resolved!r} != {os.path.realpath(src_base)!r}")
|
||||
if mode != "abs-base-symlink" and resolved != os.path.realpath(src):
|
||||
return fail(f"link-exdev={mode}: target {resolved!r} != {os.path.realpath(src)!r}")
|
||||
|
||||
src_ren, dst_ren, _ = setup_paths(branches, mount, rel_base, "rename_pass")
|
||||
mergerfs_set_option(mount, "rename-exdev", "passthrough")
|
||||
err = get_errno(lambda: os.rename(src_ren, dst_ren + ".ren.pass"))
|
||||
if err != errno.EXDEV:
|
||||
print(f"rename-exdev passthrough did not exercise EXDEV, errno={err}", end="")
|
||||
return SKIP
|
||||
|
||||
for mode in ("rel-symlink", "abs-symlink"):
|
||||
src_ren, dst_ren, _ = setup_paths(branches, mount, rel_base, f"rename_{mode}")
|
||||
mergerfs_set_option(mount, "rename-exdev", mode)
|
||||
ren_path = dst_ren + ".ren"
|
||||
err = get_errno(lambda: os.rename(src_ren, ren_path))
|
||||
if err != 0:
|
||||
return fail(f"rename-exdev={mode} returned errno={err}")
|
||||
err_msg = require_symlink(ren_path, f"rename-exdev={mode}")
|
||||
if err_msg:
|
||||
return fail(err_msg)
|
||||
target = symlink_resolves(ren_path)
|
||||
if ".mergerfs_rename_exdev" not in target:
|
||||
return fail(f"rename-exdev={mode}: target not moved under .mergerfs_rename_exdev: {target!r}")
|
||||
with open(ren_path, "rb") as fp:
|
||||
if fp.read() != b"src":
|
||||
return fail(f"rename-exdev={mode}: symlink target content mismatch")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return SKIP
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "category.create", orig_create)
|
||||
mergerfs_set_option(mount, "link-exdev", orig_link)
|
||||
mergerfs_set_option(mount, "rename-exdev", orig_rename)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return 0
|
||||
finally:
|
||||
for branch in branches:
|
||||
cleanup_dir(join(branch, rel_base))
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -5,60 +5,59 @@ import sys
|
||||
|
||||
from posix_parity import fail
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_cfg_statfs_ignore <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
|
||||
try:
|
||||
orig_statfs = mergerfs_get_option(mount, "statfs")
|
||||
orig_ignore = mergerfs_get_option(mount, "statfs-ignore")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
with mergerfs_mount() as (mount, _):
|
||||
try:
|
||||
orig_statfs = mergerfs_get_option(mount, "statfs")
|
||||
orig_ignore = mergerfs_get_option(mount, "statfs-ignore")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
|
||||
try:
|
||||
mergerfs_set_option(mount, "statfs", "base")
|
||||
try:
|
||||
mergerfs_set_option(mount, "statfs", "base")
|
||||
|
||||
mergerfs_set_option(mount, "statfs-ignore", "none")
|
||||
s_none = os.statvfs(mount)
|
||||
mergerfs_set_option(mount, "statfs-ignore", "none")
|
||||
s_none = os.statvfs(mount)
|
||||
|
||||
mergerfs_set_option(mount, "statfs-ignore", "nc")
|
||||
s_nc = os.statvfs(mount)
|
||||
mergerfs_set_option(mount, "statfs-ignore", "nc")
|
||||
s_nc = os.statvfs(mount)
|
||||
|
||||
mergerfs_set_option(mount, "statfs-ignore", "ro")
|
||||
s_ro = os.statvfs(mount)
|
||||
mergerfs_set_option(mount, "statfs-ignore", "ro")
|
||||
s_ro = os.statvfs(mount)
|
||||
|
||||
# Ignoring additional branch classes should never increase available blocks.
|
||||
if s_nc.f_bavail > s_none.f_bavail:
|
||||
return fail(
|
||||
f"statfs-ignore=nc increased bavail unexpectedly nc={s_nc.f_bavail} none={s_none.f_bavail}"
|
||||
)
|
||||
if s_ro.f_bavail > s_nc.f_bavail:
|
||||
return fail(
|
||||
f"statfs-ignore=ro increased bavail unexpectedly ro={s_ro.f_bavail} nc={s_nc.f_bavail}"
|
||||
)
|
||||
if s_nc.f_bavail > s_none.f_bavail:
|
||||
return fail(
|
||||
f"statfs-ignore=nc increased bavail unexpectedly nc={s_nc.f_bavail} none={s_none.f_bavail}"
|
||||
)
|
||||
if s_ro.f_bavail > s_nc.f_bavail:
|
||||
return fail(
|
||||
f"statfs-ignore=ro increased bavail unexpectedly ro={s_ro.f_bavail} nc={s_nc.f_bavail}"
|
||||
)
|
||||
|
||||
mergerfs_set_option(mount, "statfs", "full")
|
||||
s_full = os.statvfs(mount)
|
||||
if s_full.f_bsize <= 0 or s_full.f_frsize <= 0:
|
||||
return fail(
|
||||
f"statfs=full invalid block sizes bsize={s_full.f_bsize} frsize={s_full.f_frsize}"
|
||||
)
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "statfs", orig_statfs)
|
||||
mergerfs_set_option(mount, "statfs-ignore", orig_ignore)
|
||||
except OSError:
|
||||
pass
|
||||
mergerfs_set_option(mount, "statfs", "full")
|
||||
s_full = os.statvfs(mount)
|
||||
if s_full.f_bsize <= 0 or s_full.f_frsize <= 0:
|
||||
return fail(
|
||||
f"statfs=full invalid block sizes bsize={s_full.f_bsize} frsize={s_full.f_frsize}"
|
||||
)
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "statfs", orig_statfs)
|
||||
mergerfs_set_option(mount, "statfs-ignore", orig_ignore)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return 0
|
||||
return 0
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+45
-48
@@ -9,6 +9,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
@@ -23,58 +24,54 @@ def get_errno(func):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_cfg_xattr_modes <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
try:
|
||||
path = join(merge_base, "file")
|
||||
xname = "user.mergerfs.cfg.test"
|
||||
xval = b"v"
|
||||
with mergerfs_mount() as (mount, _):
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
touch(path, b"x")
|
||||
|
||||
try:
|
||||
orig = mergerfs_get_option(mount, "xattr")
|
||||
except (PermissionError, FileNotFoundError):
|
||||
return 0
|
||||
|
||||
try:
|
||||
mergerfs_set_option(mount, "xattr", "passthrough")
|
||||
set_err = get_errno(lambda: os.setxattr(path, xname, xval))
|
||||
get_err = get_errno(lambda: os.getxattr(path, xname))
|
||||
if set_err != 0:
|
||||
return fail(f"xattr=passthrough setxattr expected success got errno={set_err}")
|
||||
if get_err == errno.ENODATA:
|
||||
return fail(f"xattr=passthrough getxattr returned ENODATA (known bug): this masks a parity violation")
|
||||
if get_err != 0:
|
||||
return fail(f"xattr=passthrough getxattr unexpected errno={get_err}")
|
||||
|
||||
mergerfs_set_option(mount, "xattr", "noattr")
|
||||
err = get_errno(lambda: os.getxattr(path, xname))
|
||||
if err != errno.ENODATA:
|
||||
return fail(f"xattr=noattr getxattr expected ENODATA got errno={err}")
|
||||
err = get_errno(lambda: os.setxattr(path, xname, xval))
|
||||
if err != errno.ENODATA:
|
||||
return fail(f"xattr=noattr setxattr expected ENODATA got errno={err}")
|
||||
|
||||
# xattr=nosys is intentionally not tested here. It disables the
|
||||
# xattr API, including the runtime interface this test uses to
|
||||
# restore options safely.
|
||||
except (PermissionError, FileNotFoundError):
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "xattr", orig)
|
||||
except OSError:
|
||||
pass
|
||||
path = join(merge_base, "file")
|
||||
xname = "user.test"
|
||||
xval = b"v"
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
touch(path, b"x")
|
||||
|
||||
try:
|
||||
orig = mergerfs_get_option(mount, "xattr")
|
||||
except (PermissionError, FileNotFoundError):
|
||||
return 0
|
||||
|
||||
try:
|
||||
mergerfs_set_option(mount, "xattr", "passthrough")
|
||||
set_err = get_errno(lambda: os.setxattr(path, xname, xval))
|
||||
get_err = get_errno(lambda: os.getxattr(path, xname))
|
||||
if set_err != 0:
|
||||
return fail(f"xattr=passthrough setxattr expected success got errno={set_err}")
|
||||
if get_err == errno.ENODATA:
|
||||
return fail(f"xattr=passthrough getxattr returned ENODATA, expected success")
|
||||
if get_err != 0:
|
||||
return fail(f"xattr=passthrough getxattr unexpected errno={get_err}")
|
||||
|
||||
mergerfs_set_option(mount, "xattr", "noattr")
|
||||
err = get_errno(lambda: os.getxattr(path, xname))
|
||||
if err != errno.ENODATA:
|
||||
return fail(f"xattr=noattr getxattr expected ENODATA got errno={err}")
|
||||
err = get_errno(lambda: os.setxattr(path, xname, xval))
|
||||
if err != errno.ENODATA:
|
||||
return fail(f"xattr=noattr setxattr expected ENODATA got errno={err}")
|
||||
except (PermissionError, FileNotFoundError):
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "xattr", orig)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,102 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import tempfile
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_io_passthrough_create_open <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
(fd, filepath) = tempfile.mkstemp(dir=sys.argv[1])
|
||||
|
||||
try:
|
||||
test_data = b"passthrough io test data for create\n" * 100
|
||||
from posix_parity import mergerfs_mount
|
||||
with mergerfs_mount() as (mount, _):
|
||||
(fd, filepath) = tempfile.mkstemp(dir=mount)
|
||||
|
||||
bytes_written = os.write(fd, test_data)
|
||||
if bytes_written != len(test_data):
|
||||
print("create write failed: expected {} bytes, wrote {}".format(len(test_data), bytes_written))
|
||||
return 1
|
||||
try:
|
||||
test_data = b"passthrough io test data for create\n" * 100
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
read_data = os.read(fd, len(test_data))
|
||||
if read_data != test_data:
|
||||
print("create read failed: data mismatch")
|
||||
return 1
|
||||
bytes_written = os.write(fd, test_data)
|
||||
if bytes_written != len(test_data):
|
||||
print("create write failed: expected {} bytes, wrote {}".format(len(test_data), bytes_written))
|
||||
return 1
|
||||
|
||||
os.close(fd)
|
||||
fd = -1
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
read_data = os.read(fd, len(test_data))
|
||||
if read_data != test_data:
|
||||
print("create read failed: data mismatch")
|
||||
return 1
|
||||
|
||||
fd = os.open(filepath, os.O_RDWR)
|
||||
os.close(fd)
|
||||
fd = -1
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
read_data = os.read(fd, len(test_data))
|
||||
if read_data != test_data:
|
||||
print("open read failed: data mismatch")
|
||||
return 1
|
||||
fd = os.open(filepath, os.O_RDWR)
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_END)
|
||||
more_data = b"additional passthrough data for open\n" * 50
|
||||
bytes_written = os.write(fd, more_data)
|
||||
if bytes_written != len(more_data):
|
||||
print("open write failed: expected {} bytes, wrote {}".format(len(more_data), bytes_written))
|
||||
return 1
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
read_data = os.read(fd, len(test_data))
|
||||
if read_data != test_data:
|
||||
print("open read failed: data mismatch")
|
||||
return 1
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
all_data = os.read(fd, len(test_data) + len(more_data))
|
||||
if all_data != test_data + more_data:
|
||||
print("open final read failed: data mismatch")
|
||||
return 1
|
||||
os.lseek(fd, 0, os.SEEK_END)
|
||||
more_data = b"additional passthrough data for open\n" * 50
|
||||
bytes_written = os.write(fd, more_data)
|
||||
if bytes_written != len(more_data):
|
||||
print("open write failed: expected {} bytes, wrote {}".format(len(more_data), bytes_written))
|
||||
return 1
|
||||
|
||||
fd2 = os.open(filepath, os.O_RDWR)
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
all_data = os.read(fd, len(test_data) + len(more_data))
|
||||
if all_data != test_data + more_data:
|
||||
print("open final read failed: data mismatch")
|
||||
return 1
|
||||
|
||||
try:
|
||||
os.lseek(fd2, 0, os.SEEK_SET)
|
||||
read_data2 = os.read(fd2, len(test_data) + len(more_data))
|
||||
if read_data2 != test_data + more_data:
|
||||
print("second open read failed: data mismatch")
|
||||
return 1
|
||||
fd2 = os.open(filepath, os.O_RDWR)
|
||||
|
||||
os.lseek(fd2, 0, os.SEEK_END)
|
||||
extra_data = b"data from second file descriptor\n" * 25
|
||||
bytes_written = os.write(fd2, extra_data)
|
||||
if bytes_written != len(extra_data):
|
||||
print("second open write failed: expected {} bytes, wrote {}".format(len(extra_data), bytes_written))
|
||||
return 1
|
||||
try:
|
||||
os.lseek(fd2, 0, os.SEEK_SET)
|
||||
read_data2 = os.read(fd2, len(test_data) + len(more_data))
|
||||
if read_data2 != test_data + more_data:
|
||||
print("second open read failed: data mismatch")
|
||||
return 1
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
combined_data = os.read(fd, len(test_data) + len(more_data) + len(extra_data))
|
||||
expected_data = test_data + more_data + extra_data
|
||||
if combined_data != expected_data:
|
||||
print("cross-fd read failed: data mismatch (writes from fd2 not visible on fd)")
|
||||
return 1
|
||||
finally:
|
||||
os.close(fd2)
|
||||
os.lseek(fd2, 0, os.SEEK_END)
|
||||
extra_data = b"data from second file descriptor\n" * 25
|
||||
bytes_written = os.write(fd2, extra_data)
|
||||
if bytes_written != len(extra_data):
|
||||
print("second open write failed: expected {} bytes, wrote {}".format(len(extra_data), bytes_written))
|
||||
return 1
|
||||
|
||||
fd3 = os.open(filepath, os.O_RDONLY)
|
||||
try:
|
||||
os.lseek(fd3, 0, os.SEEK_SET)
|
||||
read_data3 = os.read(fd3, len(expected_data))
|
||||
if read_data3 != expected_data:
|
||||
print("third open read failed: data mismatch")
|
||||
return 1
|
||||
finally:
|
||||
os.close(fd3)
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
combined_data = os.read(fd, len(test_data) + len(more_data) + len(extra_data))
|
||||
expected_data = test_data + more_data + extra_data
|
||||
if combined_data != expected_data:
|
||||
print("cross-fd read failed: data mismatch (writes from fd2 not visible on fd)")
|
||||
return 1
|
||||
finally:
|
||||
os.close(fd2)
|
||||
|
||||
os.close(fd)
|
||||
fd = -1
|
||||
fd3 = os.open(filepath, os.O_RDONLY)
|
||||
try:
|
||||
os.lseek(fd3, 0, os.SEEK_SET)
|
||||
read_data3 = os.read(fd3, len(expected_data))
|
||||
if read_data3 != expected_data:
|
||||
print("third open read failed: data mismatch")
|
||||
return 1
|
||||
finally:
|
||||
os.close(fd3)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
if fd >= 0:
|
||||
os.close(fd)
|
||||
try:
|
||||
os.unlink(filepath)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.close(fd)
|
||||
fd = -1
|
||||
|
||||
return 0
|
||||
finally:
|
||||
if fd >= 0:
|
||||
os.close(fd)
|
||||
try:
|
||||
os.unlink(filepath)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -5,6 +5,8 @@ import sys
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
from posix_parity import mergerfs_mount
|
||||
|
||||
NUM_THREADS = 50
|
||||
TEST_DATA = b"race condition test data\n"
|
||||
|
||||
@@ -46,87 +48,88 @@ def open_and_operate(filepath, barrier, results, index):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_io_passthrough_open_race <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
(fd, filepath) = tempfile.mkstemp(dir=mount)
|
||||
|
||||
(fd, filepath) = tempfile.mkstemp(dir=sys.argv[1])
|
||||
|
||||
total_size = NUM_THREADS * len(TEST_DATA)
|
||||
os.ftruncate(fd, total_size)
|
||||
os.close(fd)
|
||||
|
||||
barrier = threading.Barrier(NUM_THREADS)
|
||||
results = [None] * NUM_THREADS
|
||||
threads = []
|
||||
|
||||
for i in range(NUM_THREADS):
|
||||
t = threading.Thread(target=open_and_operate, args=(filepath, barrier, results, i))
|
||||
threads.append(t)
|
||||
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
failed = False
|
||||
fds_to_close = []
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if result is None:
|
||||
print("thread {} returned no result".format(i))
|
||||
failed = True
|
||||
elif result[0] == "success":
|
||||
fds_to_close.append(result[1])
|
||||
else:
|
||||
print("thread {} failed: {} - {}".format(i, result[0], result[1]))
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
for fd in fds_to_close:
|
||||
total_size = NUM_THREADS * len(TEST_DATA)
|
||||
os.ftruncate(fd, total_size)
|
||||
os.close(fd)
|
||||
os.unlink(filepath)
|
||||
return 1
|
||||
|
||||
if fds_to_close:
|
||||
verify_fd = fds_to_close[0]
|
||||
os.lseek(verify_fd, 0, os.SEEK_SET)
|
||||
all_data = os.read(verify_fd, total_size)
|
||||
barrier = threading.Barrier(NUM_THREADS)
|
||||
results = [None] * NUM_THREADS
|
||||
threads = []
|
||||
|
||||
for i in range(NUM_THREADS):
|
||||
t = threading.Thread(target=open_and_operate, args=(filepath, barrier, results, i))
|
||||
threads.append(t)
|
||||
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
failed = False
|
||||
fds_to_close = []
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if result is None:
|
||||
print("thread {} returned no result".format(i))
|
||||
failed = True
|
||||
elif result[0] == "success":
|
||||
fds_to_close.append(result[1])
|
||||
else:
|
||||
print("thread {} failed: {} - {}".format(i, result[0], result[1]))
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
for fd in fds_to_close:
|
||||
os.close(fd)
|
||||
os.unlink(filepath)
|
||||
return 1
|
||||
|
||||
if fds_to_close:
|
||||
verify_fd = fds_to_close[0]
|
||||
os.lseek(verify_fd, 0, os.SEEK_SET)
|
||||
all_data = os.read(verify_fd, total_size)
|
||||
|
||||
expected_data = TEST_DATA * NUM_THREADS
|
||||
if all_data != expected_data:
|
||||
print("final verification failed: data mismatch")
|
||||
print("expected {} bytes, got {} bytes".format(len(expected_data), len(all_data)))
|
||||
for fd in fds_to_close:
|
||||
os.close(fd)
|
||||
os.unlink(filepath)
|
||||
return 1
|
||||
|
||||
if len(fds_to_close) >= 2:
|
||||
fd_a = fds_to_close[0]
|
||||
fd_b = fds_to_close[1]
|
||||
|
||||
new_data = b"cross-fd visibility test\n"
|
||||
os.lseek(fd_a, 0, os.SEEK_SET)
|
||||
os.write(fd_a, new_data)
|
||||
|
||||
os.lseek(fd_b, 0, os.SEEK_SET)
|
||||
read_back = os.read(fd_b, len(new_data))
|
||||
|
||||
if read_back != new_data:
|
||||
print("cross-fd visibility failed: write from fd_a not visible on fd_b")
|
||||
for fd in fds_to_close:
|
||||
os.close(fd)
|
||||
os.unlink(filepath)
|
||||
return 1
|
||||
|
||||
expected_data = TEST_DATA * NUM_THREADS
|
||||
if all_data != expected_data:
|
||||
print("final verification failed: data mismatch")
|
||||
print("expected {} bytes, got {} bytes".format(len(expected_data), len(all_data)))
|
||||
for fd in fds_to_close:
|
||||
os.close(fd)
|
||||
|
||||
os.unlink(filepath)
|
||||
return 1
|
||||
|
||||
if len(fds_to_close) >= 2:
|
||||
fd_a = fds_to_close[0]
|
||||
fd_b = fds_to_close[1]
|
||||
|
||||
new_data = b"cross-fd visibility test\n"
|
||||
os.lseek(fd_a, 0, os.SEEK_SET)
|
||||
os.write(fd_a, new_data)
|
||||
|
||||
os.lseek(fd_b, 0, os.SEEK_SET)
|
||||
read_back = os.read(fd_b, len(new_data))
|
||||
|
||||
if read_back != new_data:
|
||||
print("cross-fd visibility failed: write from fd_a not visible on fd_b")
|
||||
for fd in fds_to_close:
|
||||
os.close(fd)
|
||||
os.unlink(filepath)
|
||||
return 1
|
||||
|
||||
for fd in fds_to_close:
|
||||
os.close(fd)
|
||||
|
||||
os.unlink(filepath)
|
||||
return 0
|
||||
return 0
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
+14
-50
@@ -1,70 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from posix_parity import fail
|
||||
|
||||
|
||||
def has_tool(name):
|
||||
return shutil.which(name) is not None
|
||||
from posix_parity import find_fusermount
|
||||
from posix_parity import find_mergerfs
|
||||
from posix_parity import mergerfs_mount
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_mount_lifecycle <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# This is a soft integration harness check for init/destroy wiring.
|
||||
# Skip when environment is not prepared for nested mount tests.
|
||||
if os.geteuid() != 0:
|
||||
return 0
|
||||
if not has_tool("mergerfs"):
|
||||
if find_mergerfs() is None:
|
||||
return 0
|
||||
if not has_tool("fusermount") and not has_tool("fusermount3"):
|
||||
if find_fusermount() is None:
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
b1 = os.path.join(td, "b1")
|
||||
b2 = os.path.join(td, "b2")
|
||||
mp = os.path.join(td, "mp")
|
||||
os.makedirs(b1, exist_ok=True)
|
||||
os.makedirs(b2, exist_ok=True)
|
||||
os.makedirs(mp, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
"mergerfs",
|
||||
f"{b1}:{b2}",
|
||||
mp,
|
||||
"-o",
|
||||
"defaults,allow_other,use_ino,category.create=mfs",
|
||||
]
|
||||
|
||||
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if p.returncode != 0:
|
||||
return fail(f"mount lifecycle: mergerfs mount failed: {p.stderr.decode(errors='replace')}")
|
||||
|
||||
try:
|
||||
with open(os.path.join(mp, "lifecycle-file"), "wb") as fp:
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with open(os.path.join(mount, "lifecycle-file"), "wb") as fp:
|
||||
fp.write(b"ok")
|
||||
if not os.path.exists(os.path.join(mp, "lifecycle-file")):
|
||||
if not os.path.exists(os.path.join(mount, "lifecycle-file")):
|
||||
return fail("mount lifecycle: expected created file to exist")
|
||||
finally:
|
||||
um = shutil.which("fusermount3") or shutil.which("fusermount")
|
||||
for attempt in range(3):
|
||||
u = subprocess.run([um, "-u", mp], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if u.returncode == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
if u.returncode != 0:
|
||||
subprocess.run([um, "-z", mp], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
return 0
|
||||
return 0
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
+49
-52
@@ -9,6 +9,7 @@ from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_branches
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
@@ -18,70 +19,66 @@ SKIP = 77
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_movefile_enospc <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
|
||||
try:
|
||||
orig_moveonenospc = mergerfs_get_option(mount, "moveonenospc")
|
||||
except (PermissionError, FileNotFoundError):
|
||||
print("moveonenospc runtime option unavailable", end="")
|
||||
return SKIP
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EOPNOTSUPP:
|
||||
print("moveonenospc runtime option unsupported", end="")
|
||||
return SKIP
|
||||
raise
|
||||
with mergerfs_mount(num_branches=2) as (mount, branches):
|
||||
try:
|
||||
orig_moveonenospc = mergerfs_get_option(mount, "moveonenospc")
|
||||
except (PermissionError, FileNotFoundError):
|
||||
print("moveonenospc runtime option unavailable", end="")
|
||||
return SKIP
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EOPNOTSUPP:
|
||||
print("moveonenospc runtime option unsupported", end="")
|
||||
return SKIP
|
||||
raise
|
||||
|
||||
branches = mergerfs_branches(mount)
|
||||
if len(branches) < 2:
|
||||
print("moveonenospc test requires at least 2 branches", end="")
|
||||
return SKIP
|
||||
if len(branches) < 2:
|
||||
print("moveonenospc test requires at least 2 branches", end="")
|
||||
return SKIP
|
||||
|
||||
# The documented feature only runs after write returns ENOSPC/EDQUOT. This
|
||||
# generic mountpoint test cannot safely force a branch into that state.
|
||||
merge_base = temp_dir(mount)
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
try:
|
||||
mergerfs_set_option(mount, "moveonenospc", "mfs")
|
||||
if mergerfs_get_option(mount, "moveonenospc") != "mfs":
|
||||
return fail("moveonenospc: failed to set documented policy value")
|
||||
try:
|
||||
mergerfs_set_option(mount, "moveonenospc", "mfs")
|
||||
if mergerfs_get_option(mount, "moveonenospc") != "mfs":
|
||||
return fail("moveonenospc: failed to set documented policy value")
|
||||
|
||||
merge_file = join(merge_base, "enospc_test")
|
||||
touch(merge_file, b"test data for enospc")
|
||||
merge_file = join(merge_base, "enospc_test")
|
||||
touch(merge_file, b"test data for enospc")
|
||||
|
||||
if not os.path.exists(merge_file):
|
||||
return fail("moveonenospc: file was not created")
|
||||
if not os.path.exists(merge_file):
|
||||
return fail("moveonenospc: file was not created")
|
||||
|
||||
fullpath = None
|
||||
try:
|
||||
fullpath = os.getxattr(merge_file, "user.mergerfs.fullpath").decode()
|
||||
except OSError:
|
||||
pass
|
||||
fullpath = None
|
||||
try:
|
||||
fullpath = os.getxattr(merge_file, "user.mergerfs.fullpath").decode()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if fullpath and not os.path.exists(fullpath):
|
||||
return fail(f"moveonenospc: fullpath does not exist on disk: {fullpath}")
|
||||
if fullpath and not os.path.exists(fullpath):
|
||||
return fail(f"moveonenospc: fullpath does not exist on disk: {fullpath}")
|
||||
|
||||
with open(merge_file, "r") as f:
|
||||
content = f.read()
|
||||
with open(merge_file, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
if content != "test data for enospc":
|
||||
return fail(f"moveonenospc: file content mismatch: got {content!r}")
|
||||
if content != "test data for enospc":
|
||||
return fail(f"moveonenospc: file content mismatch: got {content!r}")
|
||||
|
||||
mergerfs_set_option(mount, "moveonenospc", orig_moveonenospc)
|
||||
mergerfs_set_option(mount, "moveonenospc", orig_moveonenospc)
|
||||
|
||||
print("moveonenospc ENOSPC path requires a constrained branch", end="")
|
||||
return SKIP
|
||||
except Exception as exc:
|
||||
try:
|
||||
mergerfs_set_option(mount, "moveonenospc", orig_moveonenospc)
|
||||
except Exception:
|
||||
pass
|
||||
return fail(str(exc))
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
print("moveonenospc ENOSPC path requires a constrained branch", end="")
|
||||
return SKIP
|
||||
except Exception as exc:
|
||||
try:
|
||||
mergerfs_set_option(mount, "moveonenospc", orig_moveonenospc)
|
||||
except Exception:
|
||||
pass
|
||||
return fail(str(exc))
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+20
-13
@@ -1,27 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import tempfile
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from posix_parity import mergerfs_mount
|
||||
|
||||
|
||||
def main():
|
||||
(fd, filepath) = tempfile.mkstemp(dir=sys.argv[1])
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
(fd, filepath) = tempfile.mkstemp(dir=mount)
|
||||
|
||||
os.unlink(filepath)
|
||||
os.unlink(filepath)
|
||||
|
||||
found = False
|
||||
for entry in os.scandir(sys.argv[1]):
|
||||
if entry.name.startswith(".fuse_hidden"):
|
||||
print("found .fuse_hidden file {}".format(entry.name), end='')
|
||||
found = True
|
||||
found = False
|
||||
for entry in os.scandir(mount):
|
||||
if entry.name.startswith(".fuse_hidden"):
|
||||
print("found .fuse_hidden file {}".format(entry.name), end='')
|
||||
found = True
|
||||
|
||||
os.close(fd)
|
||||
os.close(fd)
|
||||
|
||||
if found:
|
||||
return 1
|
||||
return 0
|
||||
if found:
|
||||
return 1
|
||||
return 0
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
+38
-31
@@ -1,11 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import tempfile
|
||||
import ctypes
|
||||
import mmap
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from posix_parity import mergerfs_mount
|
||||
|
||||
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
@@ -28,38 +30,43 @@ def aligned_read(fd, size):
|
||||
|
||||
|
||||
def main():
|
||||
(fd, filepath) = tempfile.mkstemp(dir=sys.argv[1])
|
||||
|
||||
os.close(fd)
|
||||
|
||||
fd = os.open(filepath, os.O_RDWR|os.O_DIRECT|os.O_TRUNC)
|
||||
|
||||
os.unlink(filepath)
|
||||
|
||||
size = resource.getpagesize()
|
||||
pattern = (b"mergerfs-o-direct" * ((size // 16) + 1))[:size]
|
||||
buf = mmap.mmap(-1, size)
|
||||
buf.write(pattern)
|
||||
buf.seek(0)
|
||||
|
||||
try:
|
||||
written = os.write(fd, buf)
|
||||
if written != size:
|
||||
print(f"O_DIRECT short write: wrote {written}, expected {size}", end="")
|
||||
return 1
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
data = aligned_read(fd, size)
|
||||
if len(data) != size:
|
||||
print(f"O_DIRECT short read: read {len(data)}, expected {size}", end="")
|
||||
return 1
|
||||
if data != pattern:
|
||||
print("O_DIRECT data mismatch", end="")
|
||||
return 1
|
||||
finally:
|
||||
buf.close()
|
||||
os.close(fd)
|
||||
with mergerfs_mount() as (mount, _):
|
||||
(fd, filepath) = tempfile.mkstemp(dir=mount)
|
||||
|
||||
return 0
|
||||
os.close(fd)
|
||||
|
||||
fd = os.open(filepath, os.O_RDWR|os.O_DIRECT|os.O_TRUNC)
|
||||
|
||||
os.unlink(filepath)
|
||||
|
||||
size = resource.getpagesize()
|
||||
pattern = (b"mergerfs-o-direct" * ((size // 16) + 1))[:size]
|
||||
buf = mmap.mmap(-1, size)
|
||||
buf.write(pattern)
|
||||
buf.seek(0)
|
||||
|
||||
try:
|
||||
written = os.write(fd, buf)
|
||||
if written != size:
|
||||
print(f"O_DIRECT short write: wrote {written}, expected {size}", end="")
|
||||
return 1
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
data = aligned_read(fd, size)
|
||||
if len(data) != size:
|
||||
print(f"O_DIRECT short read: read {len(data)}, expected {size}", end="")
|
||||
return 1
|
||||
if data != pattern:
|
||||
print("O_DIRECT data mismatch", end="")
|
||||
return 1
|
||||
finally:
|
||||
buf.close()
|
||||
os.close(fd)
|
||||
|
||||
return 0
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -2,67 +2,67 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_open_after_unlink <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
try:
|
||||
merge_file = join(merge_base, "testfile")
|
||||
touch(merge_file, b"hello world")
|
||||
with mergerfs_mount() as (mount, _):
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
fd = os.open(merge_file, os.O_RDWR)
|
||||
|
||||
try:
|
||||
data = os.read(fd, 4096)
|
||||
if data != b"hello world":
|
||||
return fail("read before unlink: expected 'hello world', got {!r}".format(data))
|
||||
|
||||
os.unlink(merge_file)
|
||||
|
||||
fd2 = os.dup(fd)
|
||||
try:
|
||||
os.lseek(fd2, 0, os.SEEK_SET)
|
||||
data2 = os.read(fd2, 4096)
|
||||
if data2 != b"hello world":
|
||||
return fail("read after unlink via dup: expected 'hello world', got {!r}".format(data2))
|
||||
merge_file = join(merge_base, "testfile")
|
||||
touch(merge_file, b"hello world")
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
written = os.write(fd, b"modified")
|
||||
if written != len(b"modified"):
|
||||
return fail("write after unlink: wrote {}, expected {}".format(written, len(b"modified")))
|
||||
fd = os.open(merge_file, os.O_RDWR)
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
data3 = os.read(fd, 4096)
|
||||
if not data3.startswith(b"modified"):
|
||||
return fail("read back written data: got {!r} after writing 'modified'".format(data3))
|
||||
try:
|
||||
data = os.read(fd, 4096)
|
||||
if data != b"hello world":
|
||||
return fail("read before unlink: expected 'hello world', got {!r}".format(data))
|
||||
|
||||
os.unlink(merge_file)
|
||||
|
||||
fd2 = os.dup(fd)
|
||||
try:
|
||||
os.lseek(fd2, 0, os.SEEK_SET)
|
||||
data2 = os.read(fd2, 4096)
|
||||
if data2 != b"hello world":
|
||||
return fail("read after unlink via dup: expected 'hello world', got {!r}".format(data2))
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
written = os.write(fd, b"modified")
|
||||
if written != len(b"modified"):
|
||||
return fail("write after unlink: wrote {}, expected {}".format(written, len(b"modified")))
|
||||
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
data3 = os.read(fd, 4096)
|
||||
if not data3.startswith(b"modified"):
|
||||
return fail("read back written data: got {!r} after writing 'modified'".format(data3))
|
||||
finally:
|
||||
os.close(fd2)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
return 0
|
||||
except Exception as exc:
|
||||
return fail(str(exc))
|
||||
finally:
|
||||
os.close(fd2)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
return 0
|
||||
except Exception as exc:
|
||||
return fail(str(exc))
|
||||
finally:
|
||||
try:
|
||||
os.unlink(join(merge_base, "testfile"))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
cleanup_dir(merge_base)
|
||||
try:
|
||||
os.unlink(join(merge_base, "testfile"))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
+57
-58
@@ -8,6 +8,7 @@ from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_branches
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
@@ -50,75 +51,73 @@ def least_used_branches(branches, rel_dir):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_policy_lup <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
|
||||
try:
|
||||
orig_create = mergerfs_get_option(mount, "func.create")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
print("LUP create policy option unavailable", end="")
|
||||
return SKIP
|
||||
with mergerfs_mount(num_branches=2) as (mount, branches):
|
||||
try:
|
||||
orig_create = mergerfs_get_option(mount, "func.create")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
print("LUP create policy option unavailable", end="")
|
||||
return SKIP
|
||||
|
||||
try:
|
||||
orig_search = mergerfs_get_option(mount, "func.search")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
print("LUP search policy option unavailable", end="")
|
||||
return SKIP
|
||||
try:
|
||||
orig_search = mergerfs_get_option(mount, "func.search")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
print("LUP search policy option unavailable", end="")
|
||||
return SKIP
|
||||
|
||||
branches = mergerfs_branches(mount)
|
||||
if len(branches) < 2:
|
||||
print("LUP policy test requires at least 2 branches", end="")
|
||||
return SKIP
|
||||
if len(branches) < 2:
|
||||
print("LUP policy test requires at least 2 branches", end="")
|
||||
return SKIP
|
||||
|
||||
merge_base = temp_dir(mount)
|
||||
rel_base = os.path.relpath(merge_base, mount)
|
||||
merge_base = temp_dir(mount)
|
||||
rel_base = os.path.relpath(merge_base, mount)
|
||||
|
||||
try:
|
||||
for branch in branches:
|
||||
os.makedirs(join(branch, rel_base), exist_ok=True)
|
||||
try:
|
||||
for branch in branches:
|
||||
os.makedirs(join(branch, rel_base), exist_ok=True)
|
||||
|
||||
mergerfs_set_option(mount, "func.create", "lup")
|
||||
mergerfs_set_option(mount, "func.search", "lup")
|
||||
mergerfs_set_option(mount, "func.create", "lup")
|
||||
mergerfs_set_option(mount, "func.search", "lup")
|
||||
|
||||
path = join(merge_base, "lup_test_file")
|
||||
expected = least_used_branches(branches, rel_base)
|
||||
touch(path, b"lup create test")
|
||||
if not os.path.exists(path):
|
||||
return fail("LUP create: file not created")
|
||||
path = join(merge_base, "lup_test_file")
|
||||
expected = least_used_branches(branches, rel_base)
|
||||
touch(path, b"lup create test")
|
||||
if not os.path.exists(path):
|
||||
return fail("LUP create: file not created")
|
||||
|
||||
fullpath = os.getxattr(path, "user.mergerfs.fullpath").decode()
|
||||
if not fullpath:
|
||||
return fail("LUP create: no fullpath attribute")
|
||||
actual = branch_for_fullpath(branches, fullpath)
|
||||
if actual not in expected:
|
||||
return fail(f"LUP create: selected {actual}, expected one of {sorted(expected)}")
|
||||
fullpath = os.getxattr(path, "user.mergerfs.fullpath").decode()
|
||||
if not fullpath:
|
||||
return fail("LUP create: no fullpath attribute")
|
||||
actual = branch_for_fullpath(branches, fullpath)
|
||||
if actual not in expected:
|
||||
return fail(f"LUP create: selected {actual}, expected one of {sorted(expected)}")
|
||||
|
||||
path2 = join(merge_base, "lup_test_file2")
|
||||
for branch in branches:
|
||||
touch(join(branch, rel_base, "lup_test_file2"), b"lup search test")
|
||||
expected = least_used_branches(branches, rel_base)
|
||||
fullpath2 = os.getxattr(path2, "user.mergerfs.fullpath").decode()
|
||||
if not fullpath2:
|
||||
return fail("LUP search: no fullpath attribute")
|
||||
actual = branch_for_fullpath(branches, fullpath2)
|
||||
if actual not in expected:
|
||||
return fail(f"LUP search: selected {actual}, expected one of {sorted(expected)}")
|
||||
path2 = join(merge_base, "lup_test_file2")
|
||||
for branch in branches:
|
||||
touch(join(branch, rel_base, "lup_test_file2"), b"lup search test")
|
||||
expected = least_used_branches(branches, rel_base)
|
||||
fullpath2 = os.getxattr(path2, "user.mergerfs.fullpath").decode()
|
||||
if not fullpath2:
|
||||
return fail("LUP search: no fullpath attribute")
|
||||
actual = branch_for_fullpath(branches, fullpath2)
|
||||
if actual not in expected:
|
||||
return fail(f"LUP search: selected {actual}, expected one of {sorted(expected)}")
|
||||
|
||||
mergerfs_set_option(mount, "func.create", orig_create)
|
||||
mergerfs_set_option(mount, "func.search", orig_search)
|
||||
mergerfs_set_option(mount, "func.create", orig_create)
|
||||
mergerfs_set_option(mount, "func.search", orig_search)
|
||||
|
||||
return 0
|
||||
except Exception as exc:
|
||||
mergerfs_set_option(mount, "func.create", orig_create)
|
||||
mergerfs_set_option(mount, "func.search", orig_search)
|
||||
return fail(str(exc))
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
for branch in branches:
|
||||
cleanup_dir(join(branch, rel_base))
|
||||
return 0
|
||||
except Exception as exc:
|
||||
mergerfs_set_option(mount, "func.create", orig_create)
|
||||
mergerfs_set_option(mount, "func.search", orig_search)
|
||||
return fail(str(exc))
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
for branch in branches:
|
||||
cleanup_dir(join(branch, rel_base))
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+49
-49
@@ -10,70 +10,70 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_access
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_access <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
touch(merge_file, b"ok", 0o644)
|
||||
touch(native_file, b"ok", 0o644)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
err = compare_access("F_OK existing", merge_file, native_file, os.F_OK)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
touch(merge_file, b"ok", 0o644)
|
||||
touch(native_file, b"ok", 0o644)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
err = compare_access(
|
||||
"F_OK missing", merge_missing, native_missing, os.F_OK, errno.ENOENT
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_access("F_OK existing", merge_file, native_file, os.F_OK)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_access(
|
||||
"X_OK non-directory prefix",
|
||||
join(merge_notdir, "child"),
|
||||
join(native_notdir, "child"),
|
||||
os.X_OK,
|
||||
errno.ENOTDIR,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_access(
|
||||
"F_OK missing", merge_missing, native_missing, os.F_OK, errno.ENOENT
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
os.chmod(merge_file, 0)
|
||||
os.chmod(native_file, 0)
|
||||
|
||||
err = compare_access(
|
||||
"X_OK non-directory prefix",
|
||||
join(merge_notdir, "child"),
|
||||
join(native_notdir, "child"),
|
||||
os.X_OK,
|
||||
errno.ENOTDIR,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_access("R_OK unreadable", merge_file, native_file, os.R_OK)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
os.chmod(merge_file, 0)
|
||||
os.chmod(native_file, 0)
|
||||
os.chmod(merge_file, 0o644)
|
||||
os.chmod(native_file, 0o644)
|
||||
|
||||
err = compare_access("R_OK unreadable", merge_file, native_file, os.R_OK)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
os.chmod(merge_file, 0o644)
|
||||
os.chmod(native_file, 0o644)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+40
-42
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import ctypes
|
||||
import errno
|
||||
import fcntl
|
||||
import os
|
||||
@@ -11,6 +12,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -25,52 +27,48 @@ def fibmap(fd, block=0):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_bmap <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"x" * 8192)
|
||||
touch(native_file, b"x" * 8192)
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"ioctl FIBMAP block0",
|
||||
lambda: fibmap(mfd, 0),
|
||||
lambda: fibmap(nfd, 0),
|
||||
lambda a, b: (a >= 0) == (b >= 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"x" * 8192)
|
||||
touch(native_file, b"x" * 8192)
|
||||
err = compare_calls(
|
||||
"ioctl FIBMAP EBADF",
|
||||
lambda: fibmap(mfd, 0),
|
||||
lambda: fibmap(nfd, 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
# FIBMAP often requires CAP_SYS_RAWIO; compare errno parity if denied.
|
||||
err = compare_calls(
|
||||
"ioctl FIBMAP block0",
|
||||
lambda: fibmap(mfd, 0),
|
||||
lambda: fibmap(nfd, 0),
|
||||
lambda a, b: (a >= 0) == (b >= 0),
|
||||
)
|
||||
if err:
|
||||
# If both denied by privilege checks, compare_calls already passes.
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
# EBADF parity on closed fds.
|
||||
err = compare_calls(
|
||||
"ioctl FIBMAP EBADF",
|
||||
lambda: fibmap(mfd, 0),
|
||||
lambda: fibmap(nfd, 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+65
-64
@@ -11,6 +11,7 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import should_compare_inode
|
||||
from posix_parity import stat_cmp_basic
|
||||
from posix_parity import temp_dir
|
||||
@@ -22,80 +23,80 @@ def stat_cmp_basic_with_inode(lhs, rhs):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_chmod <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
stcmp = stat_cmp_basic_with_inode if should_compare_inode(mount) else stat_cmp_basic
|
||||
|
||||
mount = sys.argv[1]
|
||||
stcmp = stat_cmp_basic_with_inode if should_compare_inode(mount) else stat_cmp_basic
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
touch(merge_file, b"x", 0o644)
|
||||
touch(native_file, b"x", 0o644)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
|
||||
touch(merge_file, b"x", 0o644)
|
||||
touch(native_file, b"x", 0o644)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
err = compare_calls(
|
||||
"chmod success",
|
||||
lambda: os.chmod(merge_file, 0o600),
|
||||
lambda: os.chmod(native_file, 0o600),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chmod success",
|
||||
lambda: os.chmod(merge_file, 0o600),
|
||||
lambda: os.chmod(native_file, 0o600),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"chmod stat parity",
|
||||
lambda: os.lstat(merge_file),
|
||||
lambda: os.lstat(native_file),
|
||||
stcmp,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chmod stat parity",
|
||||
lambda: os.lstat(merge_file),
|
||||
lambda: os.lstat(native_file),
|
||||
stcmp,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"chmod ENOENT",
|
||||
lambda: os.chmod(merge_missing, 0o600),
|
||||
lambda: os.chmod(native_missing, 0o600),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chmod ENOENT",
|
||||
lambda: os.chmod(merge_missing, 0o600),
|
||||
lambda: os.chmod(native_missing, 0o600),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"chmod ENOTDIR",
|
||||
lambda: os.chmod(join(merge_notdir, "child"), 0o600),
|
||||
lambda: os.chmod(join(native_notdir, "child"), 0o600),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chmod ENOTDIR",
|
||||
lambda: os.chmod(join(merge_notdir, "child"), 0o600),
|
||||
lambda: os.chmod(join(native_notdir, "child"), 0o600),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
if os.geteuid() != 0:
|
||||
err = compare_calls(
|
||||
"chmod setuid parity",
|
||||
lambda: os.chmod(merge_file, stat.S_ISUID | 0o644),
|
||||
lambda: os.chmod(native_file, stat.S_ISUID | 0o644),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
else:
|
||||
_ = errno.EPERM
|
||||
|
||||
if os.geteuid() != 0:
|
||||
err = compare_calls(
|
||||
"chmod setuid parity",
|
||||
lambda: os.chmod(merge_file, stat.S_ISUID | 0o644),
|
||||
lambda: os.chmod(native_file, stat.S_ISUID | 0o644),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
else:
|
||||
_ = errno.EPERM
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+56
-55
@@ -9,75 +9,76 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_chown <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
uid = os.getuid()
|
||||
gid = os.getgid()
|
||||
|
||||
mount = sys.argv[1]
|
||||
uid = os.getuid()
|
||||
gid = os.getgid()
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
touch(merge_file, b"x", 0o644)
|
||||
touch(native_file, b"x", 0o644)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
|
||||
touch(merge_file, b"x", 0o644)
|
||||
touch(native_file, b"x", 0o644)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
err = compare_calls(
|
||||
"chown self",
|
||||
lambda: os.chown(merge_file, uid, gid),
|
||||
lambda: os.chown(native_file, uid, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chown self",
|
||||
lambda: os.chown(merge_file, uid, gid),
|
||||
lambda: os.chown(native_file, uid, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"chown ENOENT",
|
||||
lambda: os.chown(merge_missing, uid, gid),
|
||||
lambda: os.chown(native_missing, uid, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chown ENOENT",
|
||||
lambda: os.chown(merge_missing, uid, gid),
|
||||
lambda: os.chown(native_missing, uid, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"chown ENOTDIR",
|
||||
lambda: os.chown(join(merge_notdir, "child"), uid, gid),
|
||||
lambda: os.chown(join(native_notdir, "child"), uid, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"chown ENOTDIR",
|
||||
lambda: os.chown(join(merge_notdir, "child"), uid, gid),
|
||||
lambda: os.chown(join(native_notdir, "child"), uid, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
if os.geteuid() != 0:
|
||||
err = compare_calls(
|
||||
"chown EPERM foreign uid",
|
||||
lambda: os.chown(merge_file, 0, gid),
|
||||
lambda: os.chown(native_file, 0, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
if os.geteuid() != 0:
|
||||
err = compare_calls(
|
||||
"chown EPERM foreign uid",
|
||||
lambda: os.chown(merge_file, 0, gid),
|
||||
lambda: os.chown(native_file, 0, gid),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+159
-115
@@ -1,7 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
@@ -9,131 +11,173 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def _native_tmp():
|
||||
tests_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
parent = os.path.join(tests_dir, ".test_tmp")
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
return tempfile.mkdtemp(prefix="native_", dir=parent)
|
||||
|
||||
|
||||
class NativeTmp:
|
||||
def __enter__(self):
|
||||
self.path = _native_tmp()
|
||||
return self.path
|
||||
|
||||
def __exit__(self, *args):
|
||||
shutil.rmtree(self.path, ignore_errors=True)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_copy_file_range <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
if not hasattr(os, "copy_file_range"):
|
||||
return 0
|
||||
|
||||
if not hasattr(os, "copy_file_range"):
|
||||
return 0
|
||||
|
||||
mount = sys.argv[1]
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
merge_src = join(merge_base, "src")
|
||||
merge_dst = join(merge_base, "dst")
|
||||
native_src = join(native_base, "src")
|
||||
native_dst = join(native_base, "dst")
|
||||
|
||||
payload = b"0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
touch(merge_src, payload)
|
||||
touch(native_src, payload)
|
||||
touch(merge_dst, b"")
|
||||
touch(native_dst, b"")
|
||||
|
||||
msfd = os.open(merge_src, os.O_RDONLY)
|
||||
mdfd = os.open(merge_dst, os.O_WRONLY)
|
||||
nsfd = os.open(native_src, os.O_RDONLY)
|
||||
ndfd = os.open(native_dst, os.O_WRONLY)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"copy_file_range success",
|
||||
lambda: os.copy_file_range(msfd, mdfd, 16),
|
||||
lambda: os.copy_file_range(nsfd, ndfd, 16),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(msfd)
|
||||
os.close(mdfd)
|
||||
os.close(nsfd)
|
||||
os.close(ndfd)
|
||||
|
||||
with open(merge_dst, "rb") as mf, open(native_dst, "rb") as nf:
|
||||
mdata = mf.read()
|
||||
ndata = nf.read()
|
||||
if mdata != ndata:
|
||||
return fail(f"copy_file_range dst mismatch mergerfs={mdata!r} native={ndata!r}")
|
||||
|
||||
mdfd = os.open(merge_dst, os.O_WRONLY)
|
||||
ndfd = os.open(native_dst, os.O_WRONLY)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"copy_file_range EBADF src",
|
||||
lambda: os.copy_file_range(-1, mdfd, 1),
|
||||
lambda: os.copy_file_range(-1, ndfd, 1),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mdfd)
|
||||
os.close(ndfd)
|
||||
|
||||
# Exercise very large copy lengths to validate the 64-bit copy result
|
||||
# path when available. Use sparse files to keep disk usage low.
|
||||
# NOTE: 5GB+ length may be slow on some filesystems; targets
|
||||
# ENOSPC/EFBIG early if space is insufficient.
|
||||
large_len = (5 << 30) + 4096
|
||||
merge_large_src = join(merge_base, "src-large")
|
||||
merge_large_dst = join(merge_base, "dst-large")
|
||||
native_large_src = join(native_base, "src-large")
|
||||
native_large_dst = join(native_base, "dst-large")
|
||||
touch(merge_large_src, b"")
|
||||
touch(merge_large_dst, b"")
|
||||
touch(native_large_src, b"")
|
||||
touch(native_large_dst, b"")
|
||||
|
||||
msfd = os.open(merge_large_src, os.O_RDWR)
|
||||
mdfd = os.open(merge_large_dst, os.O_RDWR)
|
||||
nsfd = os.open(native_large_src, os.O_RDWR)
|
||||
ndfd = os.open(native_large_dst, os.O_RDWR)
|
||||
try:
|
||||
with NativeTmp() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
os.ftruncate(msfd, large_len)
|
||||
os.ftruncate(nsfd, large_len)
|
||||
os.pwrite(msfd, b"z", large_len - 1)
|
||||
os.pwrite(nsfd, b"z", large_len - 1)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.ENOSPC, errno.EFBIG):
|
||||
return 0
|
||||
raise
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
err = compare_calls(
|
||||
"copy_file_range large sparse",
|
||||
lambda: os.copy_file_range(msfd, mdfd, large_len),
|
||||
lambda: os.copy_file_range(nsfd, ndfd, large_len),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
merge_src = join(merge_base, "src")
|
||||
merge_dst = join(merge_base, "dst")
|
||||
native_src = join(native_base, "src")
|
||||
native_dst = join(native_base, "dst")
|
||||
|
||||
mstat = os.fstat(mdfd)
|
||||
nstat = os.fstat(ndfd)
|
||||
if mstat.st_size != nstat.st_size:
|
||||
return fail(
|
||||
"copy_file_range large sparse dst size mismatch "
|
||||
f"mergerfs={mstat.st_size} native={nstat.st_size}"
|
||||
)
|
||||
finally:
|
||||
os.close(msfd)
|
||||
os.close(mdfd)
|
||||
os.close(nsfd)
|
||||
os.close(ndfd)
|
||||
payload = b"0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
touch(merge_src, payload)
|
||||
touch(native_src, payload)
|
||||
touch(merge_dst, b"")
|
||||
touch(native_dst, b"")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
msfd = os.open(merge_src, os.O_RDONLY)
|
||||
mdfd = os.open(merge_dst, os.O_WRONLY)
|
||||
nsfd = os.open(native_src, os.O_RDONLY)
|
||||
ndfd = os.open(native_dst, os.O_WRONLY)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"copy_file_range success",
|
||||
lambda: os.copy_file_range(msfd, mdfd, 16),
|
||||
lambda: os.copy_file_range(nsfd, ndfd, 16),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(msfd)
|
||||
os.close(mdfd)
|
||||
os.close(nsfd)
|
||||
os.close(ndfd)
|
||||
|
||||
with open(merge_dst, "rb") as mf, open(native_dst, "rb") as nf:
|
||||
mdata = mf.read()
|
||||
ndata = nf.read()
|
||||
if mdata != ndata:
|
||||
return fail(f"copy_file_range dst mismatch mergerfs={mdata!r} native={ndata!r}")
|
||||
|
||||
mdfd = os.open(merge_dst, os.O_WRONLY)
|
||||
ndfd = os.open(native_dst, os.O_WRONLY)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"copy_file_range EBADF src",
|
||||
lambda: os.copy_file_range(-1, mdfd, 1),
|
||||
lambda: os.copy_file_range(-1, ndfd, 1),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mdfd)
|
||||
os.close(ndfd)
|
||||
|
||||
large_len = (5 << 30) + 4096
|
||||
merge_large_src = join(merge_base, "src-large")
|
||||
merge_large_dst = join(merge_base, "dst-large")
|
||||
native_large_src = join(native_base, "src-large")
|
||||
native_large_dst = join(native_base, "dst-large")
|
||||
touch(merge_large_src, b"")
|
||||
touch(merge_large_dst, b"")
|
||||
touch(native_large_src, b"")
|
||||
touch(native_large_dst, b"")
|
||||
|
||||
msfd = os.open(merge_large_src, os.O_RDWR)
|
||||
mdfd = os.open(merge_large_dst, os.O_RDWR)
|
||||
nsfd = os.open(native_large_src, os.O_RDWR)
|
||||
ndfd = os.open(native_large_dst, os.O_RDWR)
|
||||
try:
|
||||
try:
|
||||
os.ftruncate(msfd, large_len)
|
||||
os.ftruncate(nsfd, large_len)
|
||||
os.pwrite(msfd, b"z", large_len - 1)
|
||||
os.pwrite(nsfd, b"z", large_len - 1)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.ENOSPC, errno.EFBIG):
|
||||
return 0
|
||||
raise
|
||||
|
||||
total_m = 0
|
||||
while True:
|
||||
n = os.copy_file_range(msfd, mdfd, large_len)
|
||||
if n == 0:
|
||||
break
|
||||
total_m += n
|
||||
if total_m >= large_len:
|
||||
break
|
||||
|
||||
total_n = 0
|
||||
while True:
|
||||
n = os.copy_file_range(nsfd, ndfd, large_len)
|
||||
if n == 0:
|
||||
break
|
||||
total_n += n
|
||||
if total_n >= large_len:
|
||||
break
|
||||
|
||||
mstat = os.fstat(mdfd)
|
||||
nstat = os.fstat(ndfd)
|
||||
if mstat.st_size != nstat.st_size:
|
||||
return fail(
|
||||
"copy_file_range large sparse dst size mismatch "
|
||||
f"mergerfs={mstat.st_size} native={nstat.st_size}"
|
||||
)
|
||||
|
||||
def sha256_partial(fd, size):
|
||||
h = hashlib.sha256()
|
||||
pos = os.lseek(fd, 0, os.SEEK_SET)
|
||||
remaining = size
|
||||
while remaining > 0:
|
||||
chunk = os.read(fd, min(remaining, 65536))
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
remaining -= len(chunk)
|
||||
os.lseek(fd, pos, os.SEEK_SET)
|
||||
return h.digest()
|
||||
|
||||
if os.path.getsize(merge_large_dst) != os.path.getsize(native_large_dst):
|
||||
return fail("copy_file_range large: final sizes differ")
|
||||
|
||||
mhash = sha256_partial(mdfd, mstat.st_size)
|
||||
nhash = sha256_partial(ndfd, nstat.st_size)
|
||||
if mhash != nhash:
|
||||
return fail("copy_file_range large: content hash mismatch")
|
||||
finally:
|
||||
os.close(msfd)
|
||||
os.close(mdfd)
|
||||
os.close(nsfd)
|
||||
os.close(ndfd)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
raise SystemExit(main())
|
||||
@@ -10,90 +10,90 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_create_mknod <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_exist = join(merge_base, "exist")
|
||||
native_exist = join(native_base, "exist")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_file, merge_exist, merge_notdir])
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_exist = join(merge_base, "exist")
|
||||
native_exist = join(native_base, "exist")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
err = compare_calls(
|
||||
"create success",
|
||||
lambda: os.open(merge_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
lambda: os.open(native_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
close_fds=True,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
cleanup_paths([merge_file, merge_exist, merge_notdir])
|
||||
with open(merge_exist, "wb"):
|
||||
pass
|
||||
with open(native_exist, "wb"):
|
||||
pass
|
||||
|
||||
err = compare_calls(
|
||||
"create success",
|
||||
lambda: os.open(merge_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
lambda: os.open(native_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
close_fds=True,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"create EEXIST",
|
||||
lambda: os.open(merge_exist, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
lambda: os.open(native_exist, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
close_fds=True,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
with open(merge_exist, "wb"):
|
||||
pass
|
||||
with open(native_exist, "wb"):
|
||||
pass
|
||||
with open(merge_notdir, "wb"):
|
||||
pass
|
||||
with open(native_notdir, "wb"):
|
||||
pass
|
||||
|
||||
err = compare_calls(
|
||||
"create EEXIST",
|
||||
lambda: os.open(merge_exist, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
lambda: os.open(native_exist, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640),
|
||||
close_fds=True,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"create ENOTDIR",
|
||||
lambda: os.open(join(merge_notdir, "child"), os.O_CREAT | os.O_WRONLY, 0o640),
|
||||
lambda: os.open(join(native_notdir, "child"), os.O_CREAT | os.O_WRONLY, 0o640),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
with open(merge_notdir, "wb"):
|
||||
pass
|
||||
with open(native_notdir, "wb"):
|
||||
pass
|
||||
merge_node = join(merge_base, "mknod")
|
||||
native_node = join(native_base, "mknod")
|
||||
cleanup_paths([merge_node])
|
||||
|
||||
err = compare_calls(
|
||||
"create ENOTDIR",
|
||||
lambda: os.open(join(merge_notdir, "child"), os.O_CREAT | os.O_WRONLY, 0o640),
|
||||
lambda: os.open(join(native_notdir, "child"), os.O_CREAT | os.O_WRONLY, 0o640),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"mknod regular",
|
||||
lambda: os.mknod(merge_node, stat.S_IFREG | 0o600, 0),
|
||||
lambda: os.mknod(native_node, stat.S_IFREG | 0o600, 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
merge_node = join(merge_base, "mknod")
|
||||
native_node = join(native_base, "mknod")
|
||||
cleanup_paths([merge_node])
|
||||
err = compare_calls(
|
||||
"mknod EEXIST",
|
||||
lambda: os.mknod(merge_node, stat.S_IFREG | 0o600, 0),
|
||||
lambda: os.mknod(native_node, stat.S_IFREG | 0o600, 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"mknod regular",
|
||||
lambda: os.mknod(merge_node, stat.S_IFREG | 0o600, 0),
|
||||
lambda: os.mknod(native_node, stat.S_IFREG | 0o600, 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"mknod EEXIST",
|
||||
lambda: os.mknod(merge_node, stat.S_IFREG | 0o600, 0),
|
||||
lambda: os.mknod(native_node, stat.S_IFREG | 0o600, 0),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -8,76 +8,75 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_fsync_flush <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
touch(merge_file, b"abc")
|
||||
touch(native_file, b"abc")
|
||||
os.makedirs(merge_dir, exist_ok=True)
|
||||
os.makedirs(native_dir, exist_ok=True)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
os.lseek(mfd, 0, os.SEEK_END)
|
||||
os.lseek(nfd, 0, os.SEEK_END)
|
||||
os.write(mfd, b"-appended")
|
||||
os.write(nfd, b"-appended")
|
||||
|
||||
touch(merge_file, b"abc")
|
||||
touch(native_file, b"abc")
|
||||
os.makedirs(merge_dir, exist_ok=True)
|
||||
os.makedirs(native_dir, exist_ok=True)
|
||||
err = compare_calls("fsync file", lambda: os.fsync(mfd), lambda: os.fsync(nfd))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
os.lseek(mfd, 0, os.SEEK_END)
|
||||
os.lseek(nfd, 0, os.SEEK_END)
|
||||
os.write(mfd, b"-appended")
|
||||
os.write(nfd, b"-appended")
|
||||
mdirfd = os.open(merge_dir, os.O_RDONLY | os.O_DIRECTORY)
|
||||
ndirfd = os.open(native_dir, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
err = compare_calls("fsync dir", lambda: os.fsync(mdirfd), lambda: os.fsync(ndirfd))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mdirfd)
|
||||
os.close(ndirfd)
|
||||
|
||||
err = compare_calls("fsync file", lambda: os.fsync(mfd), lambda: os.fsync(nfd))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
bad_m = os.open(merge_file, os.O_RDONLY)
|
||||
bad_n = os.open(native_file, os.O_RDONLY)
|
||||
os.close(bad_m)
|
||||
os.close(bad_n)
|
||||
err = compare_calls("fsync EBADF", lambda: os.fsync(bad_m), lambda: os.fsync(bad_n))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
mdirfd = os.open(merge_dir, os.O_RDONLY | os.O_DIRECTORY)
|
||||
ndirfd = os.open(native_dir, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
err = compare_calls("fsync dir", lambda: os.fsync(mdirfd), lambda: os.fsync(ndirfd))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mdirfd)
|
||||
os.close(ndirfd)
|
||||
with open(merge_file, "rb") as mf, open(native_file, "rb") as nf:
|
||||
mdata = mf.read()
|
||||
ndata = nf.read()
|
||||
if mdata != ndata:
|
||||
return fail(f"flush content mismatch mergerfs={mdata!r} native={ndata!r}")
|
||||
|
||||
bad_m = os.open(merge_file, os.O_RDONLY)
|
||||
bad_n = os.open(native_file, os.O_RDONLY)
|
||||
os.close(bad_m)
|
||||
os.close(bad_n)
|
||||
err = compare_calls("fsync EBADF", lambda: os.fsync(bad_m), lambda: os.fsync(bad_n))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
# Close-to-flush parity check: both paths should preserve appended payload.
|
||||
with open(merge_file, "rb") as mf, open(native_file, "rb") as nf:
|
||||
mdata = mf.read()
|
||||
ndata = nf.read()
|
||||
if mdata != ndata:
|
||||
return fail(f"flush content mismatch mergerfs={mdata!r} native={ndata!r}")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -10,93 +10,92 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import should_compare_inode
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def st_cmp(lhs, rhs):
|
||||
return (
|
||||
def st_cmp(lhs, rhs, compare_ino=False):
|
||||
base = (
|
||||
stat.S_IFMT(lhs.st_mode) == stat.S_IFMT(rhs.st_mode)
|
||||
and stat.S_IMODE(lhs.st_mode) == stat.S_IMODE(rhs.st_mode)
|
||||
and lhs.st_size == rhs.st_size
|
||||
and lhs.st_nlink == rhs.st_nlink
|
||||
)
|
||||
|
||||
|
||||
def st_cmp_with_inode(lhs, rhs):
|
||||
return st_cmp(lhs, rhs) and lhs.st_ino == rhs.st_ino
|
||||
if stat.S_ISDIR(lhs.st_mode) or stat.S_ISDIR(rhs.st_mode):
|
||||
return base and (not compare_ino or lhs.st_ino == rhs.st_ino)
|
||||
return base and lhs.st_size == rhs.st_size and (not compare_ino or lhs.st_ino == rhs.st_ino)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_getattr_fgetattr <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
compare_ino = should_compare_inode(mount)
|
||||
|
||||
mount = sys.argv[1]
|
||||
stcmp = st_cmp_with_inode if should_compare_inode(mount) else st_cmp
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
cleanup_paths([merge_file, merge_dir, merge_notdir])
|
||||
|
||||
cleanup_paths([merge_file, merge_dir, merge_notdir])
|
||||
touch(merge_file, b"abc", 0o640)
|
||||
touch(native_file, b"abc", 0o640)
|
||||
os.makedirs(merge_dir, exist_ok=True)
|
||||
os.makedirs(native_dir, exist_ok=True)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
|
||||
touch(merge_file, b"abc", 0o640)
|
||||
touch(native_file, b"abc", 0o640)
|
||||
os.makedirs(merge_dir, exist_ok=True)
|
||||
os.makedirs(native_dir, exist_ok=True)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
err = compare_calls(
|
||||
"lstat regular", lambda: os.lstat(merge_file), lambda: os.lstat(native_file), lambda l, r: st_cmp(l, r, compare_ino)
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"lstat regular", lambda: os.lstat(merge_file), lambda: os.lstat(native_file), stcmp
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"lstat directory", lambda: os.lstat(merge_dir), lambda: os.lstat(native_dir), lambda l, r: st_cmp(l, r, compare_ino)
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"lstat directory", lambda: os.lstat(merge_dir), lambda: os.lstat(native_dir), stcmp
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls("fstat open fd", lambda: os.fstat(mfd), lambda: os.fstat(nfd), lambda l, r: st_cmp(l, r, compare_ino))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls("fstat open fd", lambda: os.fstat(mfd), lambda: os.fstat(nfd), stcmp)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
err = compare_calls("lstat ENOENT", lambda: os.lstat(merge_missing), lambda: os.lstat(native_missing))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("lstat ENOENT", lambda: os.lstat(merge_missing), lambda: os.lstat(native_missing))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"lstat ENOTDIR",
|
||||
lambda: os.lstat(join(merge_notdir, "child")),
|
||||
lambda: os.lstat(join(native_notdir, "child")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"lstat ENOTDIR",
|
||||
lambda: os.lstat(join(merge_notdir, "child")),
|
||||
lambda: os.lstat(join(native_notdir, "child")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+37
-37
@@ -11,6 +11,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -22,47 +23,46 @@ def fionread(fd):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_ioctl <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"hello-world")
|
||||
touch(native_file, b"hello-world")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls("ioctl FIONREAD", lambda: fionread(mfd), lambda: fionread(nfd), lambda a, b: a == b)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"hello-world")
|
||||
touch(native_file, b"hello-world")
|
||||
os.lseek(mfd, 5, os.SEEK_SET)
|
||||
os.lseek(nfd, 5, os.SEEK_SET)
|
||||
err = compare_calls(
|
||||
"ioctl FIONREAD after seek",
|
||||
lambda: fionread(mfd),
|
||||
lambda: fionread(nfd),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls("ioctl FIONREAD", lambda: fionread(mfd), lambda: fionread(nfd), lambda a, b: a == b)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
os.lseek(mfd, 5, os.SEEK_SET)
|
||||
os.lseek(nfd, 5, os.SEEK_SET)
|
||||
err = compare_calls(
|
||||
"ioctl FIONREAD after seek",
|
||||
lambda: fionread(mfd),
|
||||
lambda: fionread(nfd),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -9,67 +9,67 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_link_symlink <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_src = join(merge_base, "src")
|
||||
native_src = join(native_base, "src")
|
||||
merge_dst = join(merge_base, "dst")
|
||||
native_dst = join(native_base, "dst")
|
||||
merge_slnk = join(merge_base, "symlink")
|
||||
native_slnk = join(native_base, "symlink")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_src, merge_dst, merge_slnk])
|
||||
touch(merge_src, b"src", 0o644)
|
||||
touch(native_src, b"src", 0o644)
|
||||
|
||||
merge_src = join(merge_base, "src")
|
||||
native_src = join(native_base, "src")
|
||||
merge_dst = join(merge_base, "dst")
|
||||
native_dst = join(native_base, "dst")
|
||||
merge_slnk = join(merge_base, "symlink")
|
||||
native_slnk = join(native_base, "symlink")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
err = compare_calls("link success", lambda: os.link(merge_src, merge_dst), lambda: os.link(native_src, native_dst))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
cleanup_paths([merge_src, merge_dst, merge_slnk])
|
||||
touch(merge_src, b"src", 0o644)
|
||||
touch(native_src, b"src", 0o644)
|
||||
err = compare_calls(
|
||||
"link ENOENT source",
|
||||
lambda: os.link(merge_missing, join(merge_base, "dst2")),
|
||||
lambda: os.link(native_missing, join(native_base, "dst2")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("link success", lambda: os.link(merge_src, merge_dst), lambda: os.link(native_src, native_dst))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"symlink success",
|
||||
lambda: os.symlink("src", merge_slnk),
|
||||
lambda: os.symlink("src", native_slnk),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"link ENOENT source",
|
||||
lambda: os.link(merge_missing, join(merge_base, "dst2")),
|
||||
lambda: os.link(native_missing, join(native_base, "dst2")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"symlink EEXIST",
|
||||
lambda: os.symlink("src", merge_slnk),
|
||||
lambda: os.symlink("src", native_slnk),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"symlink success",
|
||||
lambda: os.symlink("src", merge_slnk),
|
||||
lambda: os.symlink("src", native_slnk),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"symlink EEXIST",
|
||||
lambda: os.symlink("src", merge_slnk),
|
||||
lambda: os.symlink("src", native_slnk),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+73
-75
@@ -12,12 +12,12 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def pack_flock(l_type, l_whence=0, l_start=0, l_len=0, l_pid=0):
|
||||
# struct flock (common glibc x86_64 layout)
|
||||
return struct.pack("hhqqi", l_type, l_whence, l_start, l_len, l_pid)
|
||||
|
||||
|
||||
@@ -51,91 +51,89 @@ def main():
|
||||
print(try_process_lock(sys.argv[2]))
|
||||
return 0
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_locking <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
mfd1 = os.open(merge_file, os.O_RDWR)
|
||||
mfd2 = os.open(merge_file, os.O_RDWR)
|
||||
nfd1 = os.open(native_file, os.O_RDWR)
|
||||
nfd2 = os.open(native_file, os.O_RDWR)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
try:
|
||||
err = compare_calls(
|
||||
"flock EX nonblock",
|
||||
lambda: fcntl.flock(mfd1, fcntl.LOCK_EX | fcntl.LOCK_NB),
|
||||
lambda: fcntl.flock(nfd1, fcntl.LOCK_EX | fcntl.LOCK_NB),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
mfd1 = os.open(merge_file, os.O_RDWR)
|
||||
mfd2 = os.open(merge_file, os.O_RDWR)
|
||||
nfd1 = os.open(native_file, os.O_RDWR)
|
||||
nfd2 = os.open(native_file, os.O_RDWR)
|
||||
err = compare_calls(
|
||||
"flock SH nonblock second fd",
|
||||
lambda: fcntl.flock(mfd2, fcntl.LOCK_SH | fcntl.LOCK_NB),
|
||||
lambda: fcntl.flock(nfd2, fcntl.LOCK_SH | fcntl.LOCK_NB),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
try:
|
||||
err = compare_calls(
|
||||
"flock EX nonblock",
|
||||
lambda: fcntl.flock(mfd1, fcntl.LOCK_EX | fcntl.LOCK_NB),
|
||||
lambda: fcntl.flock(nfd1, fcntl.LOCK_EX | fcntl.LOCK_NB),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("flock unlock", lambda: fcntl.flock(mfd1, fcntl.LOCK_UN), lambda: fcntl.flock(nfd1, fcntl.LOCK_UN))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("flock unlock second fd", lambda: fcntl.flock(mfd2, fcntl.LOCK_UN), lambda: fcntl.flock(nfd2, fcntl.LOCK_UN))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"flock SH nonblock second fd",
|
||||
lambda: fcntl.flock(mfd2, fcntl.LOCK_SH | fcntl.LOCK_NB),
|
||||
lambda: fcntl.flock(nfd2, fcntl.LOCK_SH | fcntl.LOCK_NB),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
wrlk = pack_flock(fcntl.F_WRLCK)
|
||||
unlck = pack_flock(fcntl.F_UNLCK)
|
||||
|
||||
err = compare_calls("flock unlock", lambda: fcntl.flock(mfd1, fcntl.LOCK_UN), lambda: fcntl.flock(nfd1, fcntl.LOCK_UN))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("flock unlock second fd", lambda: fcntl.flock(mfd2, fcntl.LOCK_UN), lambda: fcntl.flock(nfd2, fcntl.LOCK_UN))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("fcntl F_SETLK write lock", lambda: fcntl.fcntl(mfd1, fcntl.F_SETLK, wrlk), lambda: fcntl.fcntl(nfd1, fcntl.F_SETLK, wrlk))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
# POSIX record lock via fcntl
|
||||
wrlk = pack_flock(fcntl.F_WRLCK)
|
||||
unlck = pack_flock(fcntl.F_UNLCK)
|
||||
m_lock_errno = child_lock_errno(merge_file)
|
||||
n_lock_errno = child_lock_errno(native_file)
|
||||
if m_lock_errno != n_lock_errno:
|
||||
return fail(f"fcntl F_SETLK child contention errno mismatch mergerfs={m_lock_errno} native={n_lock_errno}")
|
||||
if m_lock_errno not in (errno.EACCES, errno.EAGAIN):
|
||||
return fail(f"fcntl F_SETLK child contention expected lock conflict got errno={m_lock_errno}")
|
||||
|
||||
err = compare_calls("fcntl F_SETLK write lock", lambda: fcntl.fcntl(mfd1, fcntl.F_SETLK, wrlk), lambda: fcntl.fcntl(nfd1, fcntl.F_SETLK, wrlk))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("fcntl F_SETLK unlock", lambda: fcntl.fcntl(mfd1, fcntl.F_SETLK, unlck), lambda: fcntl.fcntl(nfd1, fcntl.F_SETLK, unlck))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
m_lock_errno = child_lock_errno(merge_file)
|
||||
n_lock_errno = child_lock_errno(native_file)
|
||||
if m_lock_errno != n_lock_errno:
|
||||
return fail(f"fcntl F_SETLK child contention errno mismatch mergerfs={m_lock_errno} native={n_lock_errno}")
|
||||
if m_lock_errno not in (errno.EACCES, errno.EAGAIN):
|
||||
return fail(f"fcntl F_SETLK child contention expected lock conflict got errno={m_lock_errno}")
|
||||
bad_m = os.open(merge_file, os.O_RDONLY)
|
||||
bad_n = os.open(native_file, os.O_RDONLY)
|
||||
os.close(bad_m)
|
||||
os.close(bad_n)
|
||||
err = compare_calls(
|
||||
"fcntl EBADF",
|
||||
lambda: fcntl.fcntl(bad_m, fcntl.F_GETFD),
|
||||
lambda: fcntl.fcntl(bad_n, fcntl.F_GETFD),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd1)
|
||||
os.close(mfd2)
|
||||
os.close(nfd1)
|
||||
os.close(nfd2)
|
||||
|
||||
err = compare_calls("fcntl F_SETLK unlock", lambda: fcntl.fcntl(mfd1, fcntl.F_SETLK, unlck), lambda: fcntl.fcntl(nfd1, fcntl.F_SETLK, unlck))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
bad_m = os.open(merge_file, os.O_RDONLY)
|
||||
bad_n = os.open(native_file, os.O_RDONLY)
|
||||
os.close(bad_m)
|
||||
os.close(bad_n)
|
||||
err = compare_calls(
|
||||
"fcntl EBADF",
|
||||
lambda: fcntl.fcntl(bad_m, fcntl.F_GETFD),
|
||||
lambda: fcntl.fcntl(bad_n, fcntl.F_GETFD),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd1)
|
||||
os.close(mfd2)
|
||||
os.close(nfd1)
|
||||
os.close(nfd2)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -9,69 +9,69 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_mkdir_rmdir <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_dir, merge_notdir])
|
||||
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
err = compare_calls("mkdir success", lambda: os.mkdir(merge_dir, 0o755), lambda: os.mkdir(native_dir, 0o755))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
cleanup_paths([merge_dir, merge_notdir])
|
||||
err = compare_calls("mkdir EEXIST", lambda: os.mkdir(merge_dir, 0o755), lambda: os.mkdir(native_dir, 0o755))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("mkdir success", lambda: os.mkdir(merge_dir, 0o755), lambda: os.mkdir(native_dir, 0o755))
|
||||
if err:
|
||||
return fail(err)
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
err = compare_calls(
|
||||
"mkdir ENOTDIR",
|
||||
lambda: os.mkdir(join(merge_notdir, "child"), 0o755),
|
||||
lambda: os.mkdir(join(native_notdir, "child"), 0o755),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("mkdir EEXIST", lambda: os.mkdir(merge_dir, 0o755), lambda: os.mkdir(native_dir, 0o755))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("rmdir success", lambda: os.rmdir(merge_dir), lambda: os.rmdir(native_dir))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
err = compare_calls(
|
||||
"mkdir ENOTDIR",
|
||||
lambda: os.mkdir(join(merge_notdir, "child"), 0o755),
|
||||
lambda: os.mkdir(join(native_notdir, "child"), 0o755),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("rmdir ENOENT", lambda: os.rmdir(merge_dir), lambda: os.rmdir(native_dir))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("rmdir success", lambda: os.rmdir(merge_dir), lambda: os.rmdir(native_dir))
|
||||
if err:
|
||||
return fail(err)
|
||||
touch(join(merge_base, "nonempty/file"), b"x")
|
||||
touch(join(native_base, "nonempty/file"), b"x")
|
||||
err = compare_calls(
|
||||
"rmdir ENOTEMPTY",
|
||||
lambda: os.rmdir(join(merge_base, "nonempty")),
|
||||
lambda: os.rmdir(join(native_base, "nonempty")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("rmdir ENOENT", lambda: os.rmdir(merge_dir), lambda: os.rmdir(native_dir))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
touch(join(merge_base, "nonempty/file"), b"x")
|
||||
touch(join(native_base, "nonempty/file"), b"x")
|
||||
err = compare_calls(
|
||||
"rmdir ENOTEMPTY",
|
||||
lambda: os.rmdir(join(merge_base, "nonempty")),
|
||||
lambda: os.rmdir(join(native_base, "nonempty")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -9,79 +9,79 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_open_read_write <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
touch(merge_file, b"123456", 0o644)
|
||||
touch(native_file, b"123456", 0o644)
|
||||
os.makedirs(merge_dir, exist_ok=True)
|
||||
os.makedirs(native_dir, exist_ok=True)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
err = compare_calls("open ENOENT", lambda: os.open(merge_missing, os.O_RDONLY), lambda: os.open(native_missing, os.O_RDONLY))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
cleanup_paths([merge_file, merge_notdir])
|
||||
touch(merge_file, b"123456", 0o644)
|
||||
touch(native_file, b"123456", 0o644)
|
||||
os.makedirs(merge_dir, exist_ok=True)
|
||||
os.makedirs(native_dir, exist_ok=True)
|
||||
touch(merge_notdir, b"x", 0o644)
|
||||
touch(native_notdir, b"x", 0o644)
|
||||
err = compare_calls("open EISDIR", lambda: os.open(merge_dir, os.O_WRONLY), lambda: os.open(native_dir, os.O_WRONLY))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("open ENOENT", lambda: os.open(merge_missing, os.O_RDONLY), lambda: os.open(native_missing, os.O_RDONLY))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"open ENOTDIR",
|
||||
lambda: os.open(join(merge_notdir, "child"), os.O_RDONLY),
|
||||
lambda: os.open(join(native_notdir, "child"), os.O_RDONLY),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("open EISDIR", lambda: os.open(merge_dir, os.O_WRONLY), lambda: os.open(native_dir, os.O_WRONLY))
|
||||
if err:
|
||||
return fail(err)
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
os.lseek(mfd, 0, os.SEEK_SET)
|
||||
os.lseek(nfd, 0, os.SEEK_SET)
|
||||
m_data = os.read(mfd, 6)
|
||||
n_data = os.read(nfd, 6)
|
||||
if m_data != n_data:
|
||||
return fail(f"read parity mismatch mergerfs={m_data!r} native={n_data!r}")
|
||||
|
||||
err = compare_calls(
|
||||
"open ENOTDIR",
|
||||
lambda: os.open(join(merge_notdir, "child"), os.O_RDONLY),
|
||||
lambda: os.open(join(native_notdir, "child"), os.O_RDONLY),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
os.lseek(mfd, 0, os.SEEK_SET)
|
||||
os.lseek(nfd, 0, os.SEEK_SET)
|
||||
mw = os.write(mfd, b"abc")
|
||||
nw = os.write(nfd, b"abc")
|
||||
if mw != nw:
|
||||
return fail(f"write count mismatch mergerfs={mw} native={nw}")
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
os.lseek(mfd, 0, os.SEEK_SET)
|
||||
os.lseek(nfd, 0, os.SEEK_SET)
|
||||
m_data = os.read(mfd, 6)
|
||||
n_data = os.read(nfd, 6)
|
||||
if m_data != n_data:
|
||||
return fail(f"read parity mismatch mergerfs={m_data!r} native={n_data!r}")
|
||||
|
||||
os.lseek(mfd, 0, os.SEEK_SET)
|
||||
os.lseek(nfd, 0, os.SEEK_SET)
|
||||
mw = os.write(mfd, b"abc")
|
||||
nw = os.write(nfd, b"abc")
|
||||
if mw != nw:
|
||||
return fail(f"write count mismatch mergerfs={mw} native={nw}")
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+30
-31
@@ -8,6 +8,7 @@ import tempfile
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -23,41 +24,39 @@ def poll_mask(fd, timeout_ms=25):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_poll <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"abcdef")
|
||||
touch(native_file, b"abcdef")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
mmask = poll_mask(mfd)
|
||||
nmask = poll_mask(nfd)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"abcdef")
|
||||
touch(native_file, b"abcdef")
|
||||
if bool(mmask & select.POLLIN) != bool(nmask & select.POLLIN):
|
||||
return fail(f"poll POLLIN mismatch mergerfs_mask=0x{mmask:x} native_mask=0x{nmask:x}")
|
||||
if bool(mmask & select.POLLOUT) != bool(nmask & select.POLLOUT):
|
||||
return fail(f"poll POLLOUT mismatch mergerfs_mask=0x{mmask:x} native_mask=0x{nmask:x}")
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
mmask = poll_mask(mfd)
|
||||
nmask = poll_mask(nfd)
|
||||
|
||||
# For regular files POLLIN and POLLOUT are typically ready.
|
||||
if bool(mmask & select.POLLIN) != bool(nmask & select.POLLIN):
|
||||
return fail(f"poll POLLIN mismatch mergerfs_mask=0x{mmask:x} native_mask=0x{nmask:x}")
|
||||
if bool(mmask & select.POLLOUT) != bool(nmask & select.POLLOUT):
|
||||
return fail(f"poll POLLOUT mismatch mergerfs_mask=0x{mmask:x} native_mask=0x{nmask:x}")
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+53
-53
@@ -9,6 +9,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -111,68 +112,67 @@ def deleted_visibility_signature(dirpath, victim_path):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_readdir <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
for idx in range(8):
|
||||
touch(join(merge_dir, f"f{idx}"), b"x")
|
||||
touch(join(native_dir, f"f{idx}"), b"x")
|
||||
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
|
||||
for idx in range(8):
|
||||
touch(join(merge_dir, f"f{idx}"), b"x")
|
||||
touch(join(native_dir, f"f{idx}"), b"x")
|
||||
err = compare_calls(
|
||||
"opendir ENOENT",
|
||||
lambda: opendir_or_raise(join(merge_base, "missing")),
|
||||
lambda: opendir_or_raise(join(native_base, "missing")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
err = compare_calls(
|
||||
"opendir ENOTDIR",
|
||||
lambda: opendir_or_raise(join(merge_notdir, "child")),
|
||||
lambda: opendir_or_raise(join(native_notdir, "child")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"opendir ENOENT",
|
||||
lambda: opendir_or_raise(join(merge_base, "missing")),
|
||||
lambda: opendir_or_raise(join(native_base, "missing")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
merge_names = names_set(merge_dir)
|
||||
native_names = names_set(native_dir)
|
||||
if merge_names != native_names:
|
||||
return fail(f"readdir names mismatch mergerfs={sorted(merge_names)} native={sorted(native_names)}")
|
||||
|
||||
err = compare_calls(
|
||||
"opendir ENOTDIR",
|
||||
lambda: opendir_or_raise(join(merge_notdir, "child")),
|
||||
lambda: opendir_or_raise(join(native_notdir, "child")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
merge_consistent = seek_tell_consistent(merge_dir)
|
||||
native_consistent = seek_tell_consistent(native_dir)
|
||||
if merge_consistent != native_consistent:
|
||||
return fail(
|
||||
f"seekdir/telldir consistency mismatch mergerfs={merge_consistent} native={native_consistent}"
|
||||
)
|
||||
|
||||
merge_names = names_set(merge_dir)
|
||||
native_names = names_set(native_dir)
|
||||
if merge_names != native_names:
|
||||
return fail(f"readdir names mismatch mergerfs={sorted(merge_names)} native={sorted(native_names)}")
|
||||
touch(join(merge_dir, "victim"), b"x")
|
||||
touch(join(native_dir, "victim"), b"x")
|
||||
merge_sig = deleted_visibility_signature(merge_dir, join(merge_dir, "victim"))
|
||||
native_sig = deleted_visibility_signature(native_dir, join(native_dir, "victim"))
|
||||
if merge_sig != native_sig:
|
||||
return fail(f"deleted-entry visibility mismatch mergerfs={merge_sig} native={native_sig}")
|
||||
|
||||
merge_consistent = seek_tell_consistent(merge_dir)
|
||||
native_consistent = seek_tell_consistent(native_dir)
|
||||
if merge_consistent != native_consistent:
|
||||
return fail(
|
||||
f"seekdir/telldir consistency mismatch mergerfs={merge_consistent} native={native_consistent}"
|
||||
)
|
||||
|
||||
touch(join(merge_dir, "victim"), b"x")
|
||||
touch(join(native_dir, "victim"), b"x")
|
||||
merge_sig = deleted_visibility_signature(merge_dir, join(merge_dir, "victim"))
|
||||
native_sig = deleted_visibility_signature(native_dir, join(native_dir, "victim"))
|
||||
if merge_sig != native_sig:
|
||||
return fail(f"deleted-entry visibility mismatch mergerfs={merge_sig} native={native_sig}")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -8,6 +8,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_get_option
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import mergerfs_set_option
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
@@ -24,54 +25,51 @@ def list_and_stat(dirpath):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_readdir_plus <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
|
||||
for i in range(40):
|
||||
touch(join(merge_dir, f"f{i:03d}"), bytes(str(i), "ascii"))
|
||||
touch(join(native_dir, f"f{i:03d}"), bytes(str(i), "ascii"))
|
||||
|
||||
# Exercise mapping from readdir entries to immediate getattr calls.
|
||||
m_map = list_and_stat(merge_dir)
|
||||
n_map = list_and_stat(native_dir)
|
||||
if m_map != n_map:
|
||||
return fail("readdir+getattr map mismatch between mergerfs and native")
|
||||
|
||||
# Optionally toggle readdir policy and ensure semantic output stays stable.
|
||||
try:
|
||||
orig = mergerfs_get_option(mount, "func.readdir")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
|
||||
try:
|
||||
for mode in ("seq", "cosr:2:2", "cor:2:2"):
|
||||
mergerfs_set_option(mount, "func.readdir", mode)
|
||||
got = list_and_stat(merge_dir)
|
||||
if got != m_map:
|
||||
return fail(f"func.readdir={mode} changed readdir+getattr semantic output")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
mergerfs_set_option(mount, "func.readdir", orig)
|
||||
except OSError:
|
||||
pass
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
|
||||
for i in range(40):
|
||||
touch(join(merge_dir, f"f{i:03d}"), bytes(str(i), "ascii"))
|
||||
touch(join(native_dir, f"f{i:03d}"), bytes(str(i), "ascii"))
|
||||
|
||||
m_map = list_and_stat(merge_dir)
|
||||
n_map = list_and_stat(native_dir)
|
||||
if m_map != n_map:
|
||||
return fail("readdir+getattr map mismatch between mergerfs and native")
|
||||
|
||||
try:
|
||||
orig = mergerfs_get_option(mount, "func.readdir")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
|
||||
try:
|
||||
for mode in ("seq", "cosr:2:2", "cor:2:2"):
|
||||
mergerfs_set_option(mount, "func.readdir", mode)
|
||||
got = list_and_stat(merge_dir)
|
||||
if got != m_map:
|
||||
return fail(f"func.readdir={mode} changed readdir+getattr semantic output")
|
||||
except (PermissionError, FileNotFoundError, OSError):
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
mergerfs_set_option(mount, "func.readdir", orig)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+59
-61
@@ -7,81 +7,79 @@ import tempfile
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_release <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
touch(merge_file, b"abc")
|
||||
touch(native_file, b"abc")
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
mfd1 = os.open(merge_file, os.O_RDWR)
|
||||
nfd1 = os.open(native_file, os.O_RDWR)
|
||||
mfd2 = os.dup(mfd1)
|
||||
nfd2 = os.dup(nfd1)
|
||||
|
||||
touch(merge_file, b"abc")
|
||||
touch(native_file, b"abc")
|
||||
try:
|
||||
os.close(mfd1)
|
||||
os.close(nfd1)
|
||||
|
||||
# Open + dup so one close does not release the underlying open-file state.
|
||||
mfd1 = os.open(merge_file, os.O_RDWR)
|
||||
nfd1 = os.open(native_file, os.O_RDWR)
|
||||
mfd2 = os.dup(mfd1)
|
||||
nfd2 = os.dup(nfd1)
|
||||
mw = os.write(mfd2, b"X")
|
||||
nw = os.write(nfd2, b"X")
|
||||
if mw != nw:
|
||||
return fail(f"release dup write mismatch mergerfs={mw} native={nw}")
|
||||
|
||||
try:
|
||||
os.close(mfd1)
|
||||
os.close(nfd1)
|
||||
os.lseek(mfd2, 0, os.SEEK_SET)
|
||||
os.lseek(nfd2, 0, os.SEEK_SET)
|
||||
mr = os.read(mfd2, 4)
|
||||
nr = os.read(nfd2, 4)
|
||||
if mr != nr:
|
||||
return fail(f"release dup read mismatch mergerfs={mr!r} native={nr!r}")
|
||||
finally:
|
||||
os.close(mfd2)
|
||||
os.close(nfd2)
|
||||
|
||||
mw = os.write(mfd2, b"X")
|
||||
nw = os.write(nfd2, b"X")
|
||||
if mw != nw:
|
||||
return fail(f"release dup write mismatch mergerfs={mw} native={nw}")
|
||||
with open(merge_file, "rb") as mf, open(native_file, "rb") as nf:
|
||||
md = mf.read()
|
||||
nd = nf.read()
|
||||
if md != nd:
|
||||
return fail(f"release final data mismatch mergerfs={md!r} native={nd!r}")
|
||||
|
||||
os.lseek(mfd2, 0, os.SEEK_SET)
|
||||
os.lseek(nfd2, 0, os.SEEK_SET)
|
||||
mr = os.read(mfd2, 4)
|
||||
nr = os.read(nfd2, 4)
|
||||
if mr != nr:
|
||||
return fail(f"release dup read mismatch mergerfs={mr!r} native={nr!r}")
|
||||
finally:
|
||||
os.close(mfd2)
|
||||
os.close(nfd2)
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
try:
|
||||
os.close(mfd)
|
||||
m_err = 0
|
||||
except OSError as exc:
|
||||
m_err = exc.errno
|
||||
try:
|
||||
os.close(nfd)
|
||||
n_err = 0
|
||||
except OSError as exc:
|
||||
n_err = exc.errno
|
||||
if m_err != n_err:
|
||||
return fail(f"release EBADF mismatch mergerfs_errno={m_err} native_errno={n_err}")
|
||||
|
||||
with open(merge_file, "rb") as mf, open(native_file, "rb") as nf:
|
||||
md = mf.read()
|
||||
nd = nf.read()
|
||||
if md != nd:
|
||||
return fail(f"release final data mismatch mergerfs={md!r} native={nd!r}")
|
||||
|
||||
# Closing same fd twice should map to EBADF on both.
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
try:
|
||||
os.close(mfd)
|
||||
m_err = 0
|
||||
except OSError as exc:
|
||||
m_err = exc.errno
|
||||
try:
|
||||
os.close(nfd)
|
||||
n_err = 0
|
||||
except OSError as exc:
|
||||
n_err = exc.errno
|
||||
if m_err != n_err:
|
||||
return fail(f"release EBADF mismatch mergerfs_errno={m_err} native_errno={n_err}")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+22
-22
@@ -8,6 +8,7 @@ import tempfile
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -49,31 +50,30 @@ def close_and_dirfd_errno(path):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_releasedir <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
touch(join(merge_dir, "a"), b"x")
|
||||
touch(join(native_dir, "a"), b"x")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
m_err = close_and_dirfd_errno(merge_dir)
|
||||
n_err = close_and_dirfd_errno(native_dir)
|
||||
if m_err != n_err:
|
||||
return fail(f"releasedir EBADF parity mismatch mergerfs_errno={m_err} native_errno={n_err}")
|
||||
|
||||
merge_dir = join(merge_base, "dir")
|
||||
native_dir = join(native_base, "dir")
|
||||
touch(join(merge_dir, "a"), b"x")
|
||||
touch(join(native_dir, "a"), b"x")
|
||||
|
||||
m_err = close_and_dirfd_errno(merge_dir)
|
||||
n_err = close_and_dirfd_errno(native_dir)
|
||||
if m_err != n_err:
|
||||
return fail(f"releasedir EBADF parity mismatch mergerfs_errno={m_err} native_errno={n_err}")
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+29
-29
@@ -8,6 +8,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
|
||||
|
||||
@@ -22,38 +23,37 @@ def statvfs_cmp(lhs, rhs):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_statfs <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
err = compare_calls(
|
||||
"statvfs mount parity",
|
||||
lambda: os.statvfs(mount),
|
||||
lambda: os.statvfs(native),
|
||||
statvfs_cmp,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
err = compare_calls(
|
||||
"statvfs ENOENT",
|
||||
lambda: os.statvfs(join(merge_base, "missing")),
|
||||
lambda: os.statvfs(join(native_base, "missing")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"statvfs mount parity",
|
||||
lambda: os.statvfs(mount),
|
||||
lambda: os.statvfs(native),
|
||||
statvfs_cmp,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"statvfs ENOENT",
|
||||
lambda: os.statvfs(join(merge_base, "missing")),
|
||||
lambda: os.statvfs(join(native_base, "missing")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+85
-87
@@ -9,6 +9,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_fullpath
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import should_compare_inode
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
@@ -172,100 +173,97 @@ def expect_same_errno(name, mcall, ncall):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_statx <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
stcmp = cmp_statx_basic_with_inode if should_compare_inode(mount) else cmp_statx_basic
|
||||
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
try:
|
||||
merge_file = join(merge_base, "file")
|
||||
merge_link = join(merge_base, "link")
|
||||
with mergerfs_mount() as (mount, _):
|
||||
stcmp = cmp_statx_basic_with_inode if should_compare_inode(mount) else cmp_statx_basic
|
||||
|
||||
touch(merge_file, b"hello")
|
||||
merge_base = temp_dir(mount)
|
||||
|
||||
try:
|
||||
native_file = mergerfs_fullpath(merge_file)
|
||||
except OSError:
|
||||
return 0
|
||||
try:
|
||||
merge_file = join(merge_base, "file")
|
||||
merge_link = join(merge_base, "link")
|
||||
|
||||
try:
|
||||
os.unlink(merge_link)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.symlink("file", merge_link)
|
||||
touch(merge_file, b"hello")
|
||||
|
||||
try:
|
||||
native_link = mergerfs_fullpath(merge_link)
|
||||
except OSError:
|
||||
return 0
|
||||
try:
|
||||
native_file = mergerfs_fullpath(merge_file)
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
try:
|
||||
os.unlink(merge_link)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.symlink("file", merge_link)
|
||||
|
||||
try:
|
||||
native_link = mergerfs_fullpath(merge_link)
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
expected_native_link = os.path.join(os.path.dirname(native_file), "link")
|
||||
if not os.path.islink(native_link):
|
||||
if os.path.islink(expected_native_link):
|
||||
native_link = expected_native_link
|
||||
else:
|
||||
return 0
|
||||
|
||||
native_missing = os.path.join(os.path.dirname(native_file), "missing")
|
||||
|
||||
mst, nst, err = call_pair(
|
||||
"statx regular",
|
||||
lambda: statx_call(merge_file),
|
||||
lambda: statx_call(native_file),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
if not stcmp(mst, nst):
|
||||
return fail(
|
||||
"statx basic mismatch\n"
|
||||
f" mergerfs path: {merge_file}\n"
|
||||
f" native path: {native_file}\n"
|
||||
f" mergerfs statx: {statx_summary(mst)}\n"
|
||||
f" native statx: {statx_summary(nst)}"
|
||||
)
|
||||
|
||||
mst, nst, err = call_pair(
|
||||
"statx symlink nofollow",
|
||||
lambda: statx_call(merge_link, flags=AT_SYMLINK_NOFOLLOW),
|
||||
lambda: statx_call(native_link, flags=AT_SYMLINK_NOFOLLOW),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
if (mst.stx_mode & 0xF000) != (nst.stx_mode & 0xF000):
|
||||
return fail(
|
||||
"statx symlink type mismatch\n"
|
||||
f" mergerfs path: {merge_link}\n"
|
||||
f" native path: {native_link}\n"
|
||||
f" mergerfs statx: {statx_summary(mst)}\n"
|
||||
f" native statx: {statx_summary(nst)}"
|
||||
)
|
||||
|
||||
err = expect_same_errno(
|
||||
"statx ENOENT",
|
||||
lambda: statx_call(join(merge_base, "missing")),
|
||||
lambda: statx_call(native_missing),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = expect_same_errno(
|
||||
"statx ENOTDIR",
|
||||
lambda: statx_call(join(merge_file, "child")),
|
||||
lambda: statx_call(join(native_file, "child")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
# user.mergerfs.fullpath for a symlink may resolve to the target file path
|
||||
# depending on policy behavior. For AT_SYMLINK_NOFOLLOW comparisons we need
|
||||
# the underlying symlink path itself.
|
||||
expected_native_link = os.path.join(os.path.dirname(native_file), "link")
|
||||
if not os.path.islink(native_link):
|
||||
if os.path.islink(expected_native_link):
|
||||
native_link = expected_native_link
|
||||
else:
|
||||
return 0
|
||||
|
||||
native_missing = os.path.join(os.path.dirname(native_file), "missing")
|
||||
|
||||
mst, nst, err = call_pair(
|
||||
"statx regular",
|
||||
lambda: statx_call(merge_file),
|
||||
lambda: statx_call(native_file),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
if not stcmp(mst, nst):
|
||||
return fail(
|
||||
"statx basic mismatch\n"
|
||||
f" mergerfs path: {merge_file}\n"
|
||||
f" native path: {native_file}\n"
|
||||
f" mergerfs statx: {statx_summary(mst)}\n"
|
||||
f" native statx: {statx_summary(nst)}"
|
||||
)
|
||||
|
||||
mst, nst, err = call_pair(
|
||||
"statx symlink nofollow",
|
||||
lambda: statx_call(merge_link, flags=AT_SYMLINK_NOFOLLOW),
|
||||
lambda: statx_call(native_link, flags=AT_SYMLINK_NOFOLLOW),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
if (mst.stx_mode & 0xF000) != (nst.stx_mode & 0xF000):
|
||||
return fail(
|
||||
"statx symlink type mismatch\n"
|
||||
f" mergerfs path: {merge_link}\n"
|
||||
f" native path: {native_link}\n"
|
||||
f" mergerfs statx: {statx_summary(mst)}\n"
|
||||
f" native statx: {statx_summary(nst)}"
|
||||
)
|
||||
|
||||
err = expect_same_errno(
|
||||
"statx ENOENT",
|
||||
lambda: statx_call(join(merge_base, "missing")),
|
||||
lambda: statx_call(native_missing),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = expect_same_errno(
|
||||
"statx ENOTDIR",
|
||||
lambda: statx_call(join(merge_file, "child")),
|
||||
lambda: statx_call(join(native_file, "child")),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+30
-30
@@ -9,6 +9,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -28,40 +29,39 @@ def syncfs_fd(fd):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_syncfs <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls("syncfs success", lambda: syncfs_fd(mfd), lambda: syncfs_fd(nfd))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
err = compare_calls("syncfs EBADF", lambda: syncfs_fd(-1), lambda: syncfs_fd(-1))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDONLY)
|
||||
nfd = os.open(native_file, os.O_RDONLY)
|
||||
try:
|
||||
err = compare_calls("syncfs success", lambda: syncfs_fd(mfd), lambda: syncfs_fd(nfd))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
err = compare_calls("syncfs EBADF", lambda: syncfs_fd(-1), lambda: syncfs_fd(-1))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+38
-41
@@ -7,6 +7,7 @@ import tempfile
|
||||
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import join
|
||||
from posix_parity import temp_dir
|
||||
|
||||
@@ -25,55 +26,51 @@ def errno_of(call):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_tmpfile <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not hasattr(os, "O_TMPFILE"):
|
||||
return 0
|
||||
|
||||
mount = sys.argv[1]
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
merge_dir = merge_base
|
||||
native_dir = native_base
|
||||
|
||||
merge_dir = merge_base
|
||||
native_dir = native_base
|
||||
m_err = errno_of(lambda: open_tmpfile(merge_dir))
|
||||
n_err = errno_of(lambda: open_tmpfile(native_dir))
|
||||
if m_err == errno.EOPNOTSUPP:
|
||||
return 0
|
||||
if m_err != n_err:
|
||||
return fail(f"O_TMPFILE support mismatch mergerfs_errno={m_err} native_errno={n_err}")
|
||||
|
||||
m_err = errno_of(lambda: open_tmpfile(merge_dir))
|
||||
n_err = errno_of(lambda: open_tmpfile(native_dir))
|
||||
# mergerfs does not currently support O_TMPFILE, so EOPNOTSUPP is
|
||||
# the expected result even when the native filesystem supports it.
|
||||
if m_err == errno.EOPNOTSUPP:
|
||||
return 0
|
||||
if m_err != n_err:
|
||||
return fail(f"O_TMPFILE support mismatch mergerfs_errno={m_err} native_errno={n_err}")
|
||||
if m_err in (errno.EOPNOTSUPP, errno.EISDIR, errno.EINVAL, errno.ENOSYS):
|
||||
return 0
|
||||
|
||||
# If unsupported on both, treat as skip-pass.
|
||||
if m_err in (errno.EOPNOTSUPP, errno.EISDIR, errno.EINVAL, errno.ENOSYS):
|
||||
return 0
|
||||
mfd = open_tmpfile(merge_dir)
|
||||
try:
|
||||
nfd = open_tmpfile(native_dir)
|
||||
except OSError:
|
||||
os.close(mfd)
|
||||
return fail("O_TMPFILE: native open failed after merge succeeded")
|
||||
try:
|
||||
mw = os.write(mfd, b"tmpfile-data")
|
||||
nw = os.write(nfd, b"tmpfile-data")
|
||||
if mw != nw:
|
||||
return fail(f"O_TMPFILE write count mismatch mergerfs={mw} native={nw}")
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
mfd = open_tmpfile(merge_dir)
|
||||
try:
|
||||
nfd = open_tmpfile(native_dir)
|
||||
except OSError:
|
||||
os.close(mfd)
|
||||
return fail("O_TMPFILE: native open failed after merge succeeded")
|
||||
try:
|
||||
mw = os.write(mfd, b"tmpfile-data")
|
||||
nw = os.write(nfd, b"tmpfile-data")
|
||||
if mw != nw:
|
||||
return fail(f"O_TMPFILE write count mismatch mergerfs={mw} native={nw}")
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -9,57 +9,57 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_truncate_ftruncate <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_file])
|
||||
touch(merge_file, b"1234567890")
|
||||
touch(native_file, b"1234567890")
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
err = compare_calls("truncate shrink", lambda: os.truncate(merge_file, 3), lambda: os.truncate(native_file, 3))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
cleanup_paths([merge_file])
|
||||
touch(merge_file, b"1234567890")
|
||||
touch(native_file, b"1234567890")
|
||||
err = compare_calls("truncate ENOENT", lambda: os.truncate(merge_missing, 1), lambda: os.truncate(native_missing, 1))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("truncate shrink", lambda: os.truncate(merge_file, 3), lambda: os.truncate(native_file, 3))
|
||||
if err:
|
||||
return fail(err)
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
err = compare_calls("ftruncate grow", lambda: os.ftruncate(mfd, 16), lambda: os.ftruncate(nfd, 16))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
err = compare_calls("truncate ENOENT", lambda: os.truncate(merge_missing, 1), lambda: os.truncate(native_missing, 1))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("ftruncate EBADF", lambda: os.ftruncate(-1, 4), lambda: os.ftruncate(-1, 4))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
err = compare_calls("ftruncate grow", lambda: os.ftruncate(mfd, 16), lambda: os.ftruncate(nfd, 16))
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
err = compare_calls("ftruncate EBADF", lambda: os.ftruncate(-1, 4), lambda: os.ftruncate(-1, 4))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -9,70 +9,70 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_unlink_rename <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_a = join(merge_base, "a")
|
||||
native_a = join(native_base, "a")
|
||||
merge_b = join(merge_base, "b")
|
||||
native_b = join(native_base, "b")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_a, merge_b, merge_notdir])
|
||||
touch(merge_a, b"a")
|
||||
touch(native_a, b"a")
|
||||
|
||||
merge_a = join(merge_base, "a")
|
||||
native_a = join(native_base, "a")
|
||||
merge_b = join(merge_base, "b")
|
||||
native_b = join(native_base, "b")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
err = compare_calls("unlink success", lambda: os.unlink(merge_a), lambda: os.unlink(native_a))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
cleanup_paths([merge_a, merge_b, merge_notdir])
|
||||
touch(merge_a, b"a")
|
||||
touch(native_a, b"a")
|
||||
err = compare_calls("unlink ENOENT", lambda: os.unlink(merge_missing), lambda: os.unlink(native_missing))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("unlink success", lambda: os.unlink(merge_a), lambda: os.unlink(native_a))
|
||||
if err:
|
||||
return fail(err)
|
||||
touch(merge_a, b"a")
|
||||
touch(native_a, b"a")
|
||||
touch(merge_b, b"b")
|
||||
touch(native_b, b"b")
|
||||
|
||||
err = compare_calls("unlink ENOENT", lambda: os.unlink(merge_missing), lambda: os.unlink(native_missing))
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls("rename overwrite", lambda: os.rename(merge_a, merge_b), lambda: os.rename(native_a, native_b))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
touch(merge_a, b"a")
|
||||
touch(native_a, b"a")
|
||||
touch(merge_b, b"b")
|
||||
touch(native_b, b"b")
|
||||
err = compare_calls("rename ENOENT", lambda: os.rename(merge_missing, merge_a), lambda: os.rename(native_missing, native_a))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("rename overwrite", lambda: os.rename(merge_a, merge_b), lambda: os.rename(native_a, native_b))
|
||||
if err:
|
||||
return fail(err)
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
err = compare_calls(
|
||||
"rename ENOTDIR",
|
||||
lambda: os.rename(join(merge_notdir, "child"), merge_a),
|
||||
lambda: os.rename(join(native_notdir, "child"), native_a),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls("rename ENOENT", lambda: os.rename(merge_missing, merge_a), lambda: os.rename(native_missing, native_a))
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
err = compare_calls(
|
||||
"rename ENOTDIR",
|
||||
lambda: os.rename(join(merge_notdir, "child"), merge_a),
|
||||
lambda: os.rename(join(native_notdir, "child"), native_a),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+57
-57
@@ -10,76 +10,76 @@ from posix_parity import cleanup_paths
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_utimens <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
cleanup_paths([merge_file])
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
now = time.time()
|
||||
times = (now - 1000, now - 500)
|
||||
|
||||
cleanup_paths([merge_file])
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
err = compare_calls(
|
||||
"utime path success",
|
||||
lambda: os.utime(merge_file, times=times),
|
||||
lambda: os.utime(native_file, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
now = time.time()
|
||||
times = (now - 1000, now - 500)
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"utime fd success",
|
||||
lambda: os.utime(mfd, times=times),
|
||||
lambda: os.utime(nfd, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
|
||||
err = compare_calls(
|
||||
"utime path success",
|
||||
lambda: os.utime(merge_file, times=times),
|
||||
lambda: os.utime(native_file, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"utime ENOENT",
|
||||
lambda: os.utime(merge_missing, times=times),
|
||||
lambda: os.utime(native_missing, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
mfd = os.open(merge_file, os.O_RDWR)
|
||||
nfd = os.open(native_file, os.O_RDWR)
|
||||
try:
|
||||
err = compare_calls(
|
||||
"utime fd success",
|
||||
lambda: os.utime(mfd, times=times),
|
||||
lambda: os.utime(nfd, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
finally:
|
||||
os.close(mfd)
|
||||
os.close(nfd)
|
||||
err = compare_calls(
|
||||
"utime EBADF",
|
||||
lambda: os.utime(-1, times=times),
|
||||
lambda: os.utime(-1, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"utime ENOENT",
|
||||
lambda: os.utime(merge_missing, times=times),
|
||||
lambda: os.utime(native_missing, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"utime EBADF",
|
||||
lambda: os.utime(-1, times=times),
|
||||
lambda: os.utime(-1, times=times),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+122
-121
@@ -9,6 +9,7 @@ from posix_parity import cleanup_dir
|
||||
from posix_parity import compare_calls
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -31,135 +32,135 @@ def has_attr(name):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_xattr <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
mount = sys.argv[1]
|
||||
xname = "user.posix_parity"
|
||||
xvalue = b"parity-check"
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
|
||||
err = compare_calls(
|
||||
"setxattr success",
|
||||
lambda: os.setxattr(merge_file, xname, xvalue),
|
||||
lambda: os.setxattr(native_file, xname, xvalue),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
os.setxattr(merge_file, xname, xvalue)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs setxattr", exc)
|
||||
try:
|
||||
os.setxattr(native_file, xname, xvalue)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native setxattr", exc)
|
||||
return fail(err)
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
err = compare_calls(
|
||||
"getxattr success",
|
||||
lambda: os.getxattr(merge_file, xname),
|
||||
lambda: os.getxattr(native_file, xname),
|
||||
lambda lhs, rhs: lhs == rhs,
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.getxattr(merge_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs getxattr", exc)
|
||||
try:
|
||||
os.getxattr(native_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native getxattr", exc)
|
||||
return fail(err)
|
||||
merge_file = join(merge_base, "file")
|
||||
native_file = join(native_base, "file")
|
||||
merge_missing = join(merge_base, "missing")
|
||||
native_missing = join(native_base, "missing")
|
||||
merge_notdir = join(merge_base, "notdir")
|
||||
native_notdir = join(native_base, "notdir")
|
||||
|
||||
err = compare_calls(
|
||||
"listxattr includes key",
|
||||
lambda: os.listxattr(merge_file),
|
||||
lambda: os.listxattr(native_file),
|
||||
has_attr(xname),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
m_list = os.listxattr(merge_file)
|
||||
err += f" | mergerfs list={m_list!r}"
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs listxattr", exc)
|
||||
try:
|
||||
n_list = os.listxattr(native_file)
|
||||
err += f" | native list={n_list!r}"
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native listxattr", exc)
|
||||
return fail(err)
|
||||
touch(merge_file, b"x")
|
||||
touch(native_file, b"x")
|
||||
touch(merge_notdir, b"x")
|
||||
touch(native_notdir, b"x")
|
||||
|
||||
err = compare_calls(
|
||||
"removexattr success",
|
||||
lambda: os.removexattr(merge_file, xname),
|
||||
lambda: os.removexattr(native_file, xname),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.removexattr(merge_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs removexattr", exc)
|
||||
try:
|
||||
os.removexattr(native_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native removexattr", exc)
|
||||
return fail(err)
|
||||
xname = "user.posix_parity"
|
||||
xvalue = b"parity-check"
|
||||
|
||||
err = compare_calls(
|
||||
"getxattr missing attr",
|
||||
lambda: os.getxattr(merge_file, xname),
|
||||
lambda: os.getxattr(native_file, xname),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.getxattr(merge_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs getxattr missing", exc)
|
||||
try:
|
||||
os.getxattr(native_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native getxattr missing", exc)
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"setxattr success",
|
||||
lambda: os.setxattr(merge_file, xname, xvalue),
|
||||
lambda: os.setxattr(native_file, xname, xvalue),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.setxattr(merge_file, xname, xvalue)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs setxattr", exc)
|
||||
try:
|
||||
os.setxattr(native_file, xname, xvalue)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native setxattr", exc)
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"setxattr ENOENT",
|
||||
lambda: os.setxattr(merge_missing, xname, xvalue),
|
||||
lambda: os.setxattr(native_missing, xname, xvalue),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"getxattr success",
|
||||
lambda: os.getxattr(merge_file, xname),
|
||||
lambda: os.getxattr(native_file, xname),
|
||||
lambda lhs, rhs: lhs == rhs,
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.getxattr(merge_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs getxattr", exc)
|
||||
try:
|
||||
os.getxattr(native_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native getxattr", exc)
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"getxattr ENOTDIR",
|
||||
lambda: os.getxattr(join(merge_notdir, "child"), xname),
|
||||
lambda: os.getxattr(join(native_notdir, "child"), xname),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare_calls(
|
||||
"listxattr includes key",
|
||||
lambda: os.listxattr(merge_file),
|
||||
lambda: os.listxattr(native_file),
|
||||
has_attr(xname),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
m_list = os.listxattr(merge_file)
|
||||
err += f" | mergerfs list={m_list!r}"
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs listxattr", exc)
|
||||
try:
|
||||
n_list = os.listxattr(native_file)
|
||||
err += f" | native list={n_list!r}"
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native listxattr", exc)
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
err = compare_calls(
|
||||
"removexattr success",
|
||||
lambda: os.removexattr(merge_file, xname),
|
||||
lambda: os.removexattr(native_file, xname),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.removexattr(merge_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs removexattr", exc)
|
||||
try:
|
||||
os.removexattr(native_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native removexattr", exc)
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"getxattr missing attr",
|
||||
lambda: os.getxattr(merge_file, xname),
|
||||
lambda: os.getxattr(native_file, xname),
|
||||
)
|
||||
if err:
|
||||
try:
|
||||
os.getxattr(merge_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("mergerfs getxattr missing", exc)
|
||||
try:
|
||||
os.getxattr(native_file, xname)
|
||||
except OSError as exc:
|
||||
err += " | " + format_oserror("native getxattr missing", exc)
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"setxattr ENOENT",
|
||||
lambda: os.setxattr(merge_missing, xname, xvalue),
|
||||
lambda: os.setxattr(native_missing, xname, xvalue),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare_calls(
|
||||
"getxattr ENOTDIR",
|
||||
lambda: os.getxattr(join(merge_notdir, "child"), xname),
|
||||
lambda: os.getxattr(join(native_notdir, "child"), xname),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+121
-121
@@ -8,6 +8,7 @@ import tempfile
|
||||
from posix_parity import cleanup_dir
|
||||
from posix_parity import fail
|
||||
from posix_parity import join
|
||||
from posix_parity import mergerfs_mount
|
||||
from posix_parity import temp_dir
|
||||
from posix_parity import touch
|
||||
|
||||
@@ -88,155 +89,154 @@ def lremovexattr(path, name):
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: TEST_posix_xattr_matrix <mountpoint>", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
with mergerfs_mount() as (mount, _):
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
|
||||
mount = sys.argv[1]
|
||||
root_m = merge_base
|
||||
root_n = native_base
|
||||
|
||||
with tempfile.TemporaryDirectory() as native:
|
||||
merge_base = temp_dir(mount)
|
||||
try:
|
||||
native_base = join(native, os.path.basename(merge_base))
|
||||
os.makedirs(native_base, exist_ok=True)
|
||||
file_m = join(root_m, "file")
|
||||
file_n = join(root_n, "file")
|
||||
dir_m = join(root_m, "dir")
|
||||
dir_n = join(root_n, "dir")
|
||||
link_m = join(root_m, "link")
|
||||
link_n = join(root_n, "link")
|
||||
|
||||
root_m = merge_base
|
||||
root_n = native_base
|
||||
touch(file_m, b"x")
|
||||
touch(file_n, b"x")
|
||||
os.makedirs(dir_m, exist_ok=True)
|
||||
os.makedirs(dir_n, exist_ok=True)
|
||||
try:
|
||||
os.unlink(link_m)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
os.unlink(link_n)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.symlink("file", link_m)
|
||||
os.symlink("file", link_n)
|
||||
|
||||
file_m = join(root_m, "file")
|
||||
file_n = join(root_n, "file")
|
||||
dir_m = join(root_m, "dir")
|
||||
dir_n = join(root_n, "dir")
|
||||
link_m = join(root_m, "link")
|
||||
link_n = join(root_n, "link")
|
||||
objs = [
|
||||
("file", file_m, file_n),
|
||||
("dir", dir_m, dir_n),
|
||||
]
|
||||
|
||||
touch(file_m, b"x")
|
||||
touch(file_n, b"x")
|
||||
os.makedirs(dir_m, exist_ok=True)
|
||||
os.makedirs(dir_n, exist_ok=True)
|
||||
try:
|
||||
os.unlink(link_m)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
os.unlink(link_n)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.symlink("file", link_m)
|
||||
os.symlink("file", link_n)
|
||||
xbase = "user.xattr_matrix"
|
||||
|
||||
objs = [
|
||||
("file", file_m, file_n),
|
||||
("dir", dir_m, dir_n),
|
||||
]
|
||||
for obj_name, path_m, path_n in objs:
|
||||
xname = f"{xbase}.{obj_name}"
|
||||
|
||||
xbase = "user.xattr_matrix"
|
||||
err = compare(
|
||||
f"setxattr default {obj_name}",
|
||||
lambda p=path_m, n=xname: os.setxattr(p, n, b"v1"),
|
||||
lambda p=path_n, n=xname: os.setxattr(p, n, b"v1"),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
for obj_name, path_m, path_n in objs:
|
||||
xname = f"{xbase}.{obj_name}"
|
||||
err = getxattr_cmp(path_m, path_n, xname)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
f"setxattr default {obj_name}",
|
||||
lambda p=path_m, n=xname: os.setxattr(p, n, b"v1"),
|
||||
lambda p=path_n, n=xname: os.setxattr(p, n, b"v1"),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = list_has_cmp(path_m, path_n, xname)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = getxattr_cmp(path_m, path_n, xname)
|
||||
if err:
|
||||
return fail(err)
|
||||
if hasattr(os, "XATTR_CREATE"):
|
||||
err = compare(
|
||||
f"setxattr CREATE existing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.setxattr(p, n, b"v2", os.XATTR_CREATE),
|
||||
lambda p=path_n, n=xname: os.setxattr(p, n, b"v2", os.XATTR_CREATE),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = list_has_cmp(path_m, path_n, xname)
|
||||
if err:
|
||||
return fail(err)
|
||||
if hasattr(os, "XATTR_REPLACE"):
|
||||
err = compare(
|
||||
f"setxattr REPLACE existing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.setxattr(p, n, b"v3", os.XATTR_REPLACE),
|
||||
lambda p=path_n, n=xname: os.setxattr(p, n, b"v3", os.XATTR_REPLACE),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
if hasattr(os, "XATTR_CREATE"):
|
||||
err = getxattr_cmp(path_m, path_n, xname)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
missing_name = f"{xname}.missing"
|
||||
if hasattr(os, "XATTR_REPLACE"):
|
||||
err = compare(
|
||||
f"setxattr REPLACE missing {obj_name}",
|
||||
lambda p=path_m, n=missing_name: os.setxattr(p, n, b"v", os.XATTR_REPLACE),
|
||||
lambda p=path_n, n=missing_name: os.setxattr(p, n, b"v", os.XATTR_REPLACE),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
f"removexattr existing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.removexattr(p, n),
|
||||
lambda p=path_n, n=xname: os.removexattr(p, n),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
f"removexattr missing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.removexattr(p, n),
|
||||
lambda p=path_n, n=xname: os.removexattr(p, n),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
lxname = f"{xbase}.link"
|
||||
err = compare(
|
||||
f"setxattr CREATE existing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.setxattr(p, n, b"v2", os.XATTR_CREATE),
|
||||
lambda p=path_n, n=xname: os.setxattr(p, n, b"v2", os.XATTR_CREATE),
|
||||
"lsetxattr symlink default",
|
||||
lambda: lsetxattr(link_m, lxname, b"lv1"),
|
||||
lambda: lsetxattr(link_n, lxname, b"lv1"),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
if hasattr(os, "XATTR_REPLACE"):
|
||||
err = compare(
|
||||
f"setxattr REPLACE existing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.setxattr(p, n, b"v3", os.XATTR_REPLACE),
|
||||
lambda p=path_n, n=xname: os.setxattr(p, n, b"v3", os.XATTR_REPLACE),
|
||||
"lgetxattr symlink",
|
||||
lambda: lgetxattr(link_m, lxname),
|
||||
lambda: lgetxattr(link_n, lxname),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = getxattr_cmp(path_m, path_n, xname)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
missing_name = f"{xname}.missing"
|
||||
if hasattr(os, "XATTR_REPLACE"):
|
||||
err = compare(
|
||||
f"setxattr REPLACE missing {obj_name}",
|
||||
lambda p=path_m, n=missing_name: os.setxattr(p, n, b"v", os.XATTR_REPLACE),
|
||||
lambda p=path_n, n=missing_name: os.setxattr(p, n, b"v", os.XATTR_REPLACE),
|
||||
"llistxattr has symlink key",
|
||||
lambda: llistxattr(link_m),
|
||||
lambda: llistxattr(link_n),
|
||||
lambda a, b: ((lxname in a) == (lxname in b)),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
f"removexattr existing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.removexattr(p, n),
|
||||
lambda p=path_n, n=xname: os.removexattr(p, n),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
err = compare(
|
||||
"lremovexattr symlink",
|
||||
lambda: lremovexattr(link_m, lxname),
|
||||
lambda: lremovexattr(link_n, lxname),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
f"removexattr missing {obj_name}",
|
||||
lambda p=path_m, n=xname: os.removexattr(p, n),
|
||||
lambda p=path_n, n=xname: os.removexattr(p, n),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
lxname = f"{xbase}.link"
|
||||
err = compare(
|
||||
"lsetxattr symlink default",
|
||||
lambda: lsetxattr(link_m, lxname, b"lv1"),
|
||||
lambda: lsetxattr(link_n, lxname, b"lv1"),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
"lgetxattr symlink",
|
||||
lambda: lgetxattr(link_m, lxname),
|
||||
lambda: lgetxattr(link_n, lxname),
|
||||
lambda a, b: a == b,
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
"llistxattr has symlink key",
|
||||
lambda: llistxattr(link_m),
|
||||
lambda: llistxattr(link_n),
|
||||
lambda a, b: ((lxname in a) == (lxname in b)),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
err = compare(
|
||||
"lremovexattr symlink",
|
||||
lambda: lremovexattr(link_m, lxname),
|
||||
lambda: lremovexattr(link_n, lxname),
|
||||
)
|
||||
if err:
|
||||
return fail(err)
|
||||
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
return 0
|
||||
finally:
|
||||
cleanup_dir(merge_base)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), end="")
|
||||
return 77
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user