Compare commits

...
13 Commits
Author SHA1 Message Date
Antonio SJ Musumeci aa0b4141d6 mutex_debug.hpp 2025-08-23 13:59:38 -05:00
trapexit 90b3585347 Update readme to match index of mkdocs (#1515) 2025-08-22 20:53:22 -05:00
trapexit d7549f8d53 Fix string manipulation functions after refactor (#1514) 2025-08-22 19:42:31 -05:00
trapexit 6ac7f389c5 Misc changes (#1513)
* Start moving to string_view where possible
* Rework readdir object to use atomic load/store
2025-08-21 22:42:58 -05:00
trapexit 2101ef5bcc Updates to docs, add nonraid, more intro to fs details (#1512) 2025-08-21 15:09:04 -05:00
Max.-F. HelmandMax-F-Helm dd1bea52cd Allow empty lines in config ini files (#1509)
* Allow empty lines in config files

* Add example comment to ini example in docs/quickstart.md

---------

Co-authored-by: Max-F-Helm <m7t24zklr@relay.firefox.com>
2025-08-13 20:00:05 -05:00
trapexit df19f99976 Fix constructing string from attrval setxattr config (#1508) 2025-08-11 19:50:12 -05:00
trapexit 24e778b95a Rework calculation of branch for release make targets (#1507) 2025-08-10 18:39:18 -05:00
trapexit a8a7b34cf3 Add Debian 13 build targets (#1506) 2025-08-10 17:50:06 -05:00
trapexit 6ae9641bcc Misc documentation updates (#1505) 2025-08-10 17:23:55 -05:00
trapexit 4439c2a030 Add setting of oom_score_adj (#1504)
To limit risk of killing by OOM Killer.
2025-08-07 23:58:06 -05:00
trapexit 9872710720 More makefile tweaks, build log, smaller git clones (#1503) 2025-08-06 12:24:25 -05:00
trapexit b18aba86b6 Build improvements (#1501)
* Remove python dependency
* Remove ecfd
* Improve makefiles
* Make preload.so more 32bit compatible
2025-08-04 23:39:52 -05:00
142 changed files with 2883 additions and 1936 deletions
+5 -1
View File
@@ -14,12 +14,16 @@
# build artifacts
mergerfs
obj/
build/
src/version.hpp
VERSION
# Debian files
debian/files
debian/changelog
debian/debhelper-build-stamp
debian/mergerfs.debhelper.log
debian/mergerfs.substvars
# RPM files
rpmbuild/
+192 -125
View File
@@ -12,109 +12,131 @@
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
GIT = git
TAR = tar
MKDIR = mkdir
TOUCH = touch
CP = cp
RM = rm
LN = ln
FIND = find
INSTALL = install
MKTEMP = mktemp
STRIP ?= strip
SED = sed
GIT2DEBCL = ./buildtools/git2debcl
PKGCONFIG = pkg-config
CP ?= cp
FAKEROOT ?= fakeroot
FIND ?= find
GIT ?= git
GIT2DEBCL ?= buildtools/git2debcl
INSTALL ?= install
LN ?= ln
MKDIR ?= mkdir
MKTEMP ?= mktemp
MV ?= mv
RM ?= rm
SED ?= sed
STRIP ?= strip
TAR ?= tar
TOUCH ?= touch
GIT_REPO = 0
BUILDDIR := build
GITREF = $(shell git describe --exact-match --tags HEAD 2>/dev/null || git symbolic-ref --short HEAD 2>/dev/null || git rev-parse --short HEAD)
ifndef GIT_REPO
ifneq ($(shell $(GIT) --version 2> /dev/null),)
ifeq ($(shell test -e .git; echo $$?),0)
GIT_REPO = 1
endif
endif
USE_XATTR = 1
UGID_USE_RWLOCK = 0
ifeq ($(NDEBUG),1)
OPT_FLAGS := -O2 -DNDEBUG
else
OPT_FLAGS := -O0 -DDEBUG -g
endif
ifeq ($(STATIC),1)
USE_XATTR ?= 1
UGID_USE_RWLOCK ?= 0
ifdef NDEBUG
OPT_FLAGS := -O2 -DNDEBUG
else
OPT_FLAGS := -O0 -g -fno-omit-frame-pointer -DDEBUG
endif
ifdef STATIC
STATIC_FLAGS := -static
else
STATIC_FLAGS :=
endif
ifeq ($(LTO),1)
ifdef LTO
LTO_FLAGS := -flto
else
LTO_FLAGS :=
endif
SRC = $(wildcard src/*.cpp)
OBJS = $(SRC:src/%.cpp=build/.src/%.o)
DEPS = $(SRC:src/%.cpp=build/.src/%.d)
SRC := $(wildcard src/*.cpp)
OBJS := $(SRC:src/%.cpp=build/.src/%.o)
DEPS := $(SRC:src/%.cpp=build/.src/%.d)
TESTS = $(wildcard tests/*.cpp)
TESTS_OBJS = $(filter-out build/.src/mergerfs.o,$(OBJS))
TESTS := $(wildcard tests/*.cpp)
TESTS_OBJS := $(filter-out build/.src/mergerfs.o,$(OBJS))
TESTS_OBJS += $(TESTS:tests/%.cpp=build/.tests/%.o)
TESTS_DEPS = $(TESTS:tests/%.cpp=build/.tests/%.d)
TESTS_DEPS := $(TESTS:tests/%.cpp=build/.tests/%.d)
TESTS_DEPS += $(DEPS)
MANPAGE = mergerfs.1
CFLAGS ?= ${OPT_FLAGS}
CFLAGS := ${CFLAGS} \
-Wall \
-Wno-unused-result
CXXFLAGS ?= ${OPT_FLAGS}
CXXFLAGS := \
${CXXFLAGS} \
-std=c++17 \
$(STATIC_FLAGS) \
$(LTO_FLAGS) \
-Isrc \
-Wall \
-Wno-unused-result \
-MMD
FUSE_FLAGS = \
-Ilibfuse/include \
-D_FILE_OFFSET_BITS=64
MFS_FLAGS = \
-DUSE_XATTR=$(USE_XATTR) \
-DUGID_USE_RWLOCK=$(UGID_USE_RWLOCK)
TESTS_FLAGS = \
-Isrc \
-DTESTS
MANPAGE := mergerfs.1
CPPFLAGS ?=
override CPPFLAGS += \
-D_FILE_OFFSET_BITS=64
CFLAGS ?= \
$(OPT_FLAGS) \
-Wall \
-Wno-unused-result \
-pipe
override CFLAGS += \
-MMD \
-MP
CXXFLAGS ?= \
$(OPT_FLAGS) \
$(STATIC_FLAGS) \
$(LTO_FLAGS) \
-Wall \
-Wpessimizing-move \
-Werror=pessimizing-move \
-Wredundant-move \
-Werror=redundant-move \
-Wno-unused-result \
-pipe
override CXXFLAGS += \
-std=c++17 \
-MMD \
-MP
override INC_FLAGS := \
-Isrc \
-Ilibfuse/include
override MFS_FLAGS := \
-DUSE_XATTR=$(USE_XATTR) \
-DUGID_USE_RWLOCK=$(UGID_USE_RWLOCK)
override TESTS_FLAGS := \
-Isrc \
-DTESTS
LDFLAGS := \
${LDFLAGS} \
-pthread \
-lrt \
-lstdc++fs
LIBFUSE := libfuse/$(BUILDDIR)/libfuse.a
LDFLAGS ?=
LDLIBS := \
-lrt \
-latomic \
-pthread \
-lstdc++fs
override LDLIBS += \
$(LIBFUSE)
# https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
DESTDIR =
PREFIX = /usr/local
EXEC_PREFIX = $(PREFIX)
DATAROOTDIR = $(PREFIX)/share
DATADIR = $(DATAROOTDIR)
BINDIR = $(EXEC_PREFIX)/bin
SBINDIR = $(EXEC_PREFIX)/sbin
LIBDIR = $(EXEC_PREFIX)/lib
MANDIR = $(DATAROOTDIR)/man
MAN1DIR = $(MANDIR)/man1
DESTDIR ?=
PREFIX ?= /usr/local
EXEC_PREFIX ?= $(PREFIX)
DATAROOTDIR ?= $(PREFIX)/share
DATADIR ?= $(DATAROOTDIR)
BINDIR ?= $(EXEC_PREFIX)/bin
SBINDIR ?= $(EXEC_PREFIX)/sbin
LIBDIR ?= $(EXEC_PREFIX)/lib
MANDIR ?= $(DATAROOTDIR)/man
MAN1DIR ?= $(MANDIR)/man1
INSTALLBINDIR ?= $(DESTDIR)$(BINDIR)
INSTALLSBINDIR ?= $(DESTDIR)$(SBINDIR)
INSTALLLIBDIR ?= $(DESTDIR)$(LIBDIR)/mergerfs
INSTALLMAN1DIR ?= $(DESTDIR)$(MAN1DIR)
INSTALLBINDIR = $(DESTDIR)$(BINDIR)
INSTALLSBINDIR = $(DESTDIR)$(SBINDIR)
INSTALLLIBDIR = $(DESTDIR)$(LIBDIR)/mergerfs
INSTALLMAN1DIR = $(DESTDIR)$(MAN1DIR)
.PHONY: all
all: mergerfs
all: libfuse $(BUILDDIR)/mergerfs $(BUILDDIR)/fsck.mergerfs $(BUILDDIR)/mergerfs.collect-info
.PHONY: help
help:
@@ -123,82 +145,91 @@ help:
@echo "make STATIC=1 - build static binary"
@echo "make LTO=1 - build with link time optimization"
objects: version build/stamp
$(MAKE) $(OBJS)
tests-objects:
$(MAKE) $(TESTS_OBJS)
$(BUILDDIR)/mergerfs: $(LIBFUSE) src/version.hpp $(OBJS)
$(CXX) $(CXXFLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(OBJS) -o $@ $(LDFLAGS) $(LDLIBS)
build/mergerfs: libfuse objects
$(CXX) $(CXXFLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(OBJS) -o $@ libfuse/build/libfuse.a $(LDFLAGS)
$(BUILDDIR)/fsck.mergerfs:
$(LN) -sf "mergerfs" $@
build/tests: build/mergerfs tests-objects
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(TESTS_OBJS) -o $@ libfuse/build/libfuse.a $(LDFLAGS)
$(BUILDDIR)/mergerfs.collect-info:
$(LN) -sf "mergerfs" $@
mergerfs: build/mergerfs
ln -fs "mergerfs" "build/fsck.mergerfs"
ln -fs "mergerfs" "build/mergerfs.collect-info"
$(BUILDDIR)/tests: $(BUILDDIR)/mergerfs $(TESTS_OBJS)
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) $(TESTS_OBJS) -o $@ $(LDFLAGS) $(LDLIBS)
tests: build/tests
.PHONY: libfuse
$(LIBFUSE):
libfuse:
$(MAKE) NDEBUG=$(NDEBUG) -C libfuse
tests: $(BUILDDIR)/tests
changelog:
ifeq ($(GIT_REPO),1)
ifdef GIT_REPO
$(GIT2DEBCL) --name mergerfs > ChangeLog
else
@echo "WARNING: need git repo to generate ChangeLog"
endif
.PHONY: version
version:
version: src/version.hpp
src/version.hpp:
./buildtools/update-version
build/stamp:
$(MKDIR) -p build/.src build/.tests
$(BUILDDIR)/stamp:
$(MKDIR) -p $(BUILDDIR)/.src $(BUILDDIR)/.tests
$(TOUCH) $@
build/.src/%.o: src/%.cpp
$(CXX) $(CXXFLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@
$(BUILDDIR)/.src/%.o: src/%.cpp $(BUILDDIR)/stamp
$(CXX) $(CXXFLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@
build/.tests/%.o: tests/%.cpp
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(FUSE_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@
$(BUILDDIR)/.tests/%.o: tests/%.cpp
$(CXX) $(CXXFLAGS) $(TESTS_FLAGS) $(INC_FLAGS) $(MFS_FLAGS) $(CPPFLAGS) -c $< -o $@
build/preload.so: build/stamp tools/preload.c
$(CC) -shared -fPIC $(CFLAGS) $(CPPFLAGS) -o $@ tools/preload.c
$(BUILDDIR)/preload.so: $(BUILDDIR)/stamp tools/preload.c
$(CC) -shared -fPIC $(CFLAGS) -o $@ tools/preload.c
preload: build/preload.so
preload: $(BUILDDIR)/preload.so
.PHONY: clean
clean: rpm-clean
$(RM) -rf build
$(RM) -rf $(BUILDDIR)
$(FIND) . -name "*~" -delete
$(MAKE) -C libfuse clean
distclean: clean
ifeq ($(GIT_REPO),1)
ifdef GIT_REPO
$(GIT) clean -xfd
endif
.PHONY: install
install: install-base install-mount-tools install-preload install-man
install-base: build/mergerfs
.PHONY: install-base
install-base: all
$(MKDIR) -p "$(INSTALLBINDIR)"
$(INSTALL) -v -m 0755 "build/mergerfs" "$(INSTALLBINDIR)/mergerfs"
ln -s "mergerfs" "${INSTALLBINDIR}/fsck.mergerfs"
ln -s "mergerfs" "${INSTALLBINDIR}/mergerfs.collect-info"
$(INSTALL) -v -m 0755 "$(BUILDDIR)/mergerfs" "$(INSTALLBINDIR)/mergerfs"
$(LN) -fs "mergerfs" "${INSTALLBINDIR}/fsck.mergerfs"
$(LN) -fs "mergerfs" "${INSTALLBINDIR}/mergerfs.collect-info"
.PHONY: install-mount-tools
install-mount-tools: install-base
$(MKDIR) -p "$(INSTALLBINDIR)"
$(MAKE) -C libfuse install
.PHONY: install-man
install-man: man/$(MANPAGE)
$(MKDIR) -p "$(INSTALLMAN1DIR)"
$(INSTALL) -v -m 0644 "man/$(MANPAGE)" "$(INSTALLMAN1DIR)/$(MANPAGE)"
.PHONY: install-preload
install-preload: preload
$(MKDIR) -p "$(INSTALLLIBDIR)"
$(INSTALL) -v -m 444 "build/preload.so" "$(INSTALLLIBDIR)/preload.so"
$(INSTALL) -v -m 444 "$(BUILDDIR)/preload.so" "$(INSTALLLIBDIR)/preload.so"
.PHONY: install-strip
install-strip: install-base
$(STRIP) "$(INSTALLBINDIR)/mergerfs"
@@ -225,29 +256,34 @@ tarball: changelog version
$(TAR) --exclude=.git -cz -C $(TMPDIR) -f $(FILENAME).tar.gz $(FILENAME)
$(RM) -rf $(TMPDIR)
.PHONY: debian-changelog
debian-changelog:
ifeq ($(GIT_REPO),1)
ifdef GIT_REPO
$(GIT2DEBCL) --name mergerfs > debian/changelog
else
cp ChangeLog debian/changelog
$(CP) -v ChangeLog debian/changelog
endif
.PHONY: signed-deb
signed-deb:
$(MAKE) distclean
$(MAKE) debian-changelog
fakeroot dpkg-buildpackage -nc
$(FAKEROOT) dpkg-buildpackage -nc
.PHONY: deb
deb:
$(MAKE) distclean
$(MAKE) debian-changelog
fakeroot dpkg-buildpackage -nc -uc -us
mkdir -p ./build/pkgs/
mv -v ../mergerfs*deb ./build/pkgs/
$(MAKE) version
$(FAKEROOT) dpkg-buildpackage -nc -uc -us
$(MKDIR) -p ./$(BUILDDIR)/pkgs/
$(MV) -v ../mergerfs*deb ./$(BUILDDIR)/pkgs/
.PHONY: rpm-clean
rpm-clean:
$(RM) -rf rpmbuild
.PHONY: rpm
rpm: tarball
$(eval VERSION := $(shell cat VERSION))
$(eval VERSION := $(subst -,_,$(VERSION)))
@@ -262,37 +298,68 @@ rpm: tarball
install-build-pkgs:
./buildtools/install-build-pkgs
.PHONY: libfuse
libfuse:
$(MAKE) NDEBUG=$(NDEBUG) -C libfuse
.PHONY: install-build-tools
install-build-tools:
./bulidtools/install-build-tools
.PHONY: release release-amd64
.PHONY: release
release:
./buildtools/build-release \
--target=all \
--cleanup \
--branch=$(shell git branch --show-current)
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-sample
release-sample:
./buildtools/build-release \
--target=debian.12.amd64 \
--cleanup \
--branch=$(shell git branch --show-current)
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-amd64
release-amd64:
./buildtools/build-release \
--target=amd64 \
--cleanup \
--branch=$(shell git branch --show-current)
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-arm64
release-arm64:
./buildtools/build-release \
--target=arm64 \
--cleanup \
--branch=$(shell git branch --show-current)
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-riscv64
release-riscv64:
./buildtools/build-release \
--target=riscv64 \
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-armhf
release-armhf:
./buildtools/build-release \
--target=armhf \
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-static
release-static:
./buildtools/build-release \
--target=static \
--cleanup \
--branch=$(shell git branch --show-current)
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: release-tarball
release-tarball:
./buildtools/build-release \
--target=tarball \
$(if $(CLEANUP),--cleanup) \
--branch=$(GITREF)
.PHONY: tags
tags:
rm -fv TAGS
find . -name "*.c" -print | etags --append -
+36 -26
View File
@@ -4,23 +4,30 @@
[FUSE](https://en.wikipedia.org/wiki/Filesystem_in_Userspace) based
[union filesystem](https://en.wikipedia.org/wiki/Union_mount) geared
towards simplifying storage and management of files across numerous
commodity storage devices. It is similar to **mhddfs**, **unionfs**,
and **aufs**.
commodity storage devices. It is similar to [**mhddfs**, **unionfs**,
**aufs**, **DrivePool**,
etc.](https://trapexit.github.io/mergerfs/latest/project_comparisons/)
## Features
* Logically combine numerous filesystems/paths into a single
mount point
mount point (JBOFS: Just a Bunch of FileSystems)
* Combine paths of the same or different filesystems
* Ability to add or remove filesystems/paths without impacting the
rest of the data
* Unaffected by individual filesystem failure
* Configurable file selection and creation placement
* File IO
[passthrough](https://trapexit.github.io/mergerfs/latest/config/passthrough/)
for near native IO performance (where supported)
* Works with filesystems of any size
* Works with filesystems of almost any type
* Works with filesystems of [almost any
type](https://trapexit.github.io/mergerfs/latest/faq/compatibility_and_integration/#what-filesystems-can-be-used-as-branches)
* Ignore read-only filesystems when creating files
* Hard link copy-on-write / CoW
* Runtime configurable
* Hard link [copy-on-write /
CoW](https://trapexit.github.io/mergerfs/latest/config/link_cow/)
* [Runtime configurable](https://trapexit.github.io/mergerfs/latest/runtime_interface/)
* Support for extended attributes (xattrs)
* Support for file attributes (chattr)
* Support for POSIX ACLs
@@ -30,16 +37,17 @@ and **aufs**.
* Read/write overlay on top of read-only filesystem like OverlayFS
* File whiteout
* RAID like parity calculation
* RAID like parity calculation (see [SnapRAID](https://www.snapraid.it))
* Redundancy
* Splitting of files across branches
## How it works
mergerfs logically merges multiple filesystem paths together. It acts
as a proxy to the underlying filesystem paths. Combining the behaviors
of some functions and being a selector for others.
mergerfs logically merges multiple filesystem paths together. Not
block devices, not filesystem mounts, just paths. It acts as a proxy
to the underlying filesystem paths. Combining the behaviors of some
functions and being a selector for others.
When the contents of a directory are requested mergerfs combines the
list of files from each directory, deduplicating entries, and returns
@@ -52,7 +60,7 @@ For functions which change attributes or remove the file the behavior
may be applied to all instances found.
Read more about [policies
here](https://trapexit.github.io/mergerfs/config/functions_categories_policies/).
here](https://trapexit.github.io/mergerfs/latest/config/functions_categories_policies/).
### Visualization
@@ -63,24 +71,26 @@ A + B = C
| | |
+-- /dir1 +-- /dir1 +-- /dir1
| | | | | |
| +-- file1 | +-- file2 | +-- file1
| | +-- file3 | +-- file2
+-- /dir2 | | +-- file3
| | +-- /dir3 |
| +-- file4 | +-- /dir2
| +-- file5 | |
+-- file6 | +-- file4
|
+-- /dir3
| |
| +-- file5
|
+-- file6
| +-- file1 | | | +-- file1
| | +-- file2 | +-- file2
| | +-- file3 | +-- file3
| | |
+-- /dir2 | +-- /dir2
| | | | |
| *-- file4 | | +-- file4
| | |
| +-- /dir3 +-- /dir3
| | | | |
| | +-- file5 | +-- file5
| | |
+-- file6 | +-- file6
+-- file7 +-- file7 +-- file7
```
## QuickStart
https://trapexit.github.io/mergerfs/quickstart/
https://trapexit.github.io/mergerfs/latest/quickstart/
## Documentation
@@ -90,7 +100,7 @@ https://trapexit.github.io/mergerfs
## Support
https://trapexit.github.io/mergerfs/support/
https://trapexit.github.io/mergerfs/latest/support/
## Sponsorship and Donations
+7 -1
View File
@@ -2,8 +2,14 @@
BRANCH="${1:-master}"
SRCDIR="/tmp/mergerfs"
REPO_URL="https://github.com/trapexit/mergerfs"
git clone https://github.com/trapexit/mergerfs "${SRCDIR}" -b "${BRANCH}"
git \
clone \
--single-branch \
--branch="${BRANCH}" \
"${REPO_URL}" \
"${SRCDIR}"
cd "${SRCDIR}"
+20 -3
View File
@@ -20,8 +20,23 @@ def build(containerfile,
f'--build-arg=BRANCH={branch}',
f'--build-arg=BUILD_TIMESTAMP={timestamp}',
'buildtools/']
# TODO: Capture output and write to log
print(args)
subprocess.run(args)
rv = subprocess.run(args)
os.makedirs(pkgdirpath,exist_ok=True)
report_filepath = os.path.join(pkgdirpath,"build-report.txt")
with open(report_filepath,"a+") as f:
build = os.path.basename(containerfile)
f.write(build + ": ")
f.write(f"branch={branch}; ")
f.write(f"timestamp={timestamp}; ")
f.write("rv=")
if rv.returncode == 0:
f.write("success;")
else:
f.write("fail;")
f.write("\n")
def setup():
@@ -82,18 +97,20 @@ def main():
containerfiles.append(f'{basepath}/{args.target}')
elif args.target == 'all':
for root,dirnames,filenames in os.walk(basepath):
filenames.sort()
for filename in filenames:
if should_skip(filename):
continue
continue
containerfile = f'buildtools/containerfiles/{filename}'
containerfiles.append(containerfile)
else:
for root,dirnames,filenames in os.walk(basepath):
filenames.sort()
for filename in filenames:
if args.target not in filename:
continue
if should_skip(filename):
continue
continue
containerfile = f'buildtools/containerfiles/{filename}'
containerfiles.append(containerfile)
@@ -1,4 +1,4 @@
FROM --platform=linux/arm64 debian:10 as build
FROM --platform=linux/amd64 debian:trixie as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
@@ -0,0 +1,9 @@
FROM --platform=linux/arm64 debian:trixie as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
ARG BRANCH=master
RUN /tmp/build-mergerfs $BRANCH
FROM scratch
COPY --from=build /build/ /
@@ -0,0 +1,9 @@
FROM --platform=linux/armhf debian:trixie as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
ARG BRANCH=master
RUN /tmp/build-mergerfs $BRANCH
FROM scratch
COPY --from=build /build/ /
@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 debian:10 as build
FROM --platform=linux/i386 debian:trixie as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
@@ -1,4 +1,4 @@
FROM --platform=linux/armhf debian:10 as build
FROM --platform=linux/amd64 fedora:43 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
@@ -1,4 +1,4 @@
FROM --platform=linux/i386 debian:10 as build
FROM --platform=linux/arm64 fedora:43 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 quay.io/centos/centos:10 as build
FROM --platform=linux/amd64 quay.io/rockylinux/rockylinux:10 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
@@ -1,4 +1,4 @@
FROM --platform=linux/arm64 quay.io/centos/centos:10 as build
FROM --platform=linux/arm64 quay.io/rockylinux/rockylinux:10 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
@@ -0,0 +1,9 @@
FROM --platform=linux/riscv64 quay.io/rockylinux/rockylinux:10 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
ARG BRANCH=master
RUN /tmp/build-mergerfs $BRANCH
FROM scratch
COPY --from=build /build/ /
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 rockylinux:8 as build
FROM --platform=linux/amd64 quay.io/rockylinux/rockylinux:8 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/arm64 rockylinux:8 as build
FROM --platform=linux/arm64 quay.io/rockylinux/rockylinux:8 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/armhf rockylinux:8 as build
FROM --platform=linux/armhf quay.io/rockylinux/rockylinux:8 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/i386 rockylinux:8 as build
FROM --platform=linux/i386 quay.io/rockylinux/rockylinux:8 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 rockylinux:9 as build
FROM --platform=linux/amd64 quay.io/rockylinux/rockylinux:9 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/arm64 rockylinux:9 as build
FROM --platform=linux/arm64 quay.io/rockylinux/rockylinux:9 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/armhf rockylinux:9 as build
FROM --platform=linux/armhf quay.io/rockylinux/rockylinux:9 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/i386 rockylinux:9 as build
FROM --platform=linux/i386 quay.io/rockylinux/rockylinux:9 as build
COPY install-build-pkgs build-mergerfs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
+1 -1
View File
@@ -3,7 +3,7 @@ FROM --platform=linux/amd64 alpine:latest as build
COPY install-build-pkgs /tmp/
RUN /tmp/install-build-pkgs
ARG BRANCH=master
RUN git clone https://github.com/trapexit/mergerfs /tmp/mergerfs -b "${BRANCH}"
RUN git clone --single-branch --depth=1 https://github.com/trapexit/mergerfs /tmp/mergerfs --branch="${BRANCH}"
WORKDIR /tmp/mergerfs
RUN make NDEBUG=1 LTO=1 STATIC=1 DESTDIR=/tmp -j$(nproc) install
RUN mkdir /build
+1 -1
View File
@@ -3,7 +3,7 @@ FROM --platform=linux/arm64 alpine:latest as build
COPY install-build-pkgs /tmp/
RUN /tmp/install-build-pkgs
ARG BRANCH=master
RUN git clone https://github.com/trapexit/mergerfs /tmp/mergerfs -b "${BRANCH}"
RUN git clone --single-branch --depth=1 https://github.com/trapexit/mergerfs /tmp/mergerfs --branch="${BRANCH}"
WORKDIR /tmp/mergerfs
RUN make NDEBUG=1 LTO=1 STATIC=1 DESTDIR=/tmp -j$(nproc) install
RUN mkdir /build
+1 -1
View File
@@ -3,7 +3,7 @@ FROM --platform=linux/armhf alpine:latest as build
COPY install-build-pkgs /tmp/
RUN /tmp/install-build-pkgs
ARG BRANCH=master
RUN git clone https://github.com/trapexit/mergerfs /tmp/mergerfs -b "${BRANCH}"
RUN git clone --single-branch --depth=1 https://github.com/trapexit/mergerfs /tmp/mergerfs --branch="${BRANCH}"
WORKDIR /tmp/mergerfs
RUN make NDEBUG=1 LTO=1 STATIC=1 DESTDIR=/tmp -j$(nproc) install
RUN mkdir /build
+1 -1
View File
@@ -3,7 +3,7 @@ FROM --platform=linux/i386 alpine:latest as build
COPY install-build-pkgs /tmp/
RUN /tmp/install-build-pkgs
ARG BRANCH=master
RUN git clone https://github.com/trapexit/mergerfs /tmp/mergerfs -b "${BRANCH}"
RUN git clone --single-branch --depth=1 https://github.com/trapexit/mergerfs /tmp/mergerfs --branch="${BRANCH}"
WORKDIR /tmp/mergerfs
RUN make NDEBUG=1 LTO=1 STATIC=1 DESTDIR=/tmp -j$(nproc) install
RUN mkdir /build
+13
View File
@@ -0,0 +1,13 @@
ARG BUILD_TIMESTAMP=0
FROM --platform=linux/riscv64 alpine:latest as build
COPY install-build-pkgs /tmp/
RUN /tmp/install-build-pkgs
ARG BRANCH=master
RUN git clone --single-branch --depth=1 https://github.com/trapexit/mergerfs /tmp/mergerfs --branch="${BRANCH}"
WORKDIR /tmp/mergerfs
RUN make NDEBUG=1 LTO=1 STATIC=1 DESTDIR=/tmp -j$(nproc) install
RUN mkdir /build
RUN tar cvfz /build/mergerfs-static-linux_riscv64.tar.gz --directory=/tmp usr sbin
FROM scratch
COPY --from=build /build/ /
+3 -3
View File
@@ -1,9 +1,9 @@
FROM debian:buster as tarball
ARG BUILD_TIMESTAMP=0
FROM alpine:latest as tarball
COPY install-build-pkgs /tmp/
RUN /tmp/install-build-pkgs
ARG BUILD_TIMESTAMP=0
ARG BRANCH=master
RUN git clone https://github.com/trapexit/mergerfs /tmp/mergerfs -b "${BRANCH}"
RUN git clone --single-branch --depth=1 https://github.com/trapexit/mergerfs /tmp/mergerfs --branch="${BRANCH}"
RUN cd /tmp/mergerfs && make version tarball
RUN mkdir /build
RUN cp -v /tmp/mergerfs/*.tar.gz /build/
+164 -74
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/bin/sh
# Copyright (c) 2016, Antonio SJ Musumeci <trapexit@spawn.link>
@@ -14,99 +14,189 @@
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import sys
import subprocess
import argparse
# Default values
NAME=""
VERSION="::guess::"
DISTRO="::guess::"
CODENAME="::guess::"
URGENCY="medium"
def call(args):
return subprocess.Popen(args,stdout=subprocess.PIPE).communicate()[0].decode()
# Function to display usage
usage() {
cat << EOF
Usage: $0 --name NAME [OPTIONS]
def git_tags():
args = ["git", "tag", '-l']
tags = call(args)
tags = [[int(X) for X in tag.split(".")] for tag in tags.split()]
tags.sort()
tags.reverse()
tags = [".".join([str(X) for X in tag]) for tag in tags]
return tags
Generated debian/changelog from git log
def git_log(fromtag,totag):
args = ['git','log','--no-merges','--oneline',fromtag+'...'+totag]
return call(args).strip().split('\n')
Options:
--name NAME Name of package (required)
--version VERSION Place in git history to include upto (default: ::guess::)
--distro DISTRO Distribution name (default: ::guess::)
--codename NAME Distribution codename (default: ::guess::)
--urgency URGENCY Urgency (default: medium)
-h, --help Show this help message
EOF
exit 1
}
def git_author_and_time(tag):
args = ['git','log','-1','--format=-- %an <%ae> %cD',tag]
return call(args).strip()
# Parse command line arguments
while [ $# -gt 0 ]; do
case "$1" in
--name)
NAME="$2"
shift 2
;;
--version)
VERSION="$2"
shift 2
;;
--distro)
DISTRO="$2"
shift 2
;;
--codename)
CODENAME="$2"
shift 2
;;
--urgency)
URGENCY="$2"
shift 2
;;
-h|--help)
usage
;;
*)
echo "Unknown option: $1" >&2
usage
;;
esac
done
def git_version():
args = ['git','describe','--always','--tags','--dirty']
return call(args).strip()
# Check required arguments
if [ -z "$NAME" ]; then
echo "Error: --name is required" >&2
usage
fi
def guess_distro():
try:
args = ['lsb_release','-i','-s']
return call(args).strip().lower()
except:
return 'unknown'
# Function to get git tags sorted in reverse order
git_tags() {
git tag -l | sort -rV
}
# Function to get git log between two tags
git_log() {
local fromtag="$1"
local totag="$2"
git log --no-merges --oneline "${fromtag}...${totag}" 2>/dev/null
}
def guess_codename():
try:
args = ['lsb_release','-c','-s']
return call(args).strip().lower()
except:
return 'unknown'
# Function to get author and time for a tag
git_author_and_time() {
local tag="$1"
git log -1 --format="-- %an <%ae> %cD" "$tag"
}
# Function to get git version
git_version() {
git describe --always --tags --dirty 2>/dev/null || echo "unknown"
}
def main():
parser = argparse.ArgumentParser(description='Generated debian/changelog from git log')
parser.add_argument('--name',type=str,help='Name of package',required=True)
parser.add_argument('--version',type=str,help='Place in git history to include upto',default='::guess::')
parser.add_argument('--distro',type=str,help='Distribution name',default='::guess::')
parser.add_argument('--codename',type=str,help='Distribution codename',default='::guess::')
parser.add_argument('--urgency',type=str,help='Urgency',default='medium')
# Function to guess distribution
guess_distro() {
if command -v lsb_release >/dev/null 2>&1; then
lsb_release -i -s 2>/dev/null | tr '[:upper:]' '[:lower:]'
else
echo "unknown"
fi
}
args = parser.parse_args()
# Function to guess codename
guess_codename() {
if command -v lsb_release >/dev/null 2>&1; then
lsb_release -c -s 2>/dev/null | tr '[:upper:]' '[:lower:]'
else
echo "unknown"
fi
}
if args.distro == '::guess::':
args.distro = guess_distro()
# Handle guessing of values
if [ "$DISTRO" = "::guess::" ]; then
DISTRO=$(guess_distro)
fi
if args.codename == '::guess::':
args.codename = guess_codename()
if [ "$CODENAME" = "::guess::" ]; then
CODENAME=$(guess_codename)
fi
versuffix = "~"+args.distro+"-"+args.codename
if [ "$VERSION" = "::guess::" ]; then
VERSION=$(git_version)
fi
if args.version == '::guess::':
args.version = git_version()
# Version suffix
VERSUFFIX="~${DISTRO}-${CODENAME}"
tags = git_tags()
if args.version in tags:
idx = tags.index(args.version)
tags = tags[idx:]
tags = list(zip(tags,tags))
else:
tags = list(zip(tags,tags))
tags.insert(0,(args.version,'HEAD'))
# Get all tags
TAGS=$(git_tags)
for i in range(0,len(tags)):
tags[i] = (tags[i][0] + versuffix,tags[i][1])
# Create temporary file for tag pairs
TMPFILE=$(mktemp)
trap 'rm -f "$TMPFILE"' EXIT
tag = tags[0]
for prev in tags[1:]:
lines = git_log(tag[1],prev[1])
if lines == ['']:
tag = prev
# Check if version is in tags
if echo "$TAGS" | grep -q "^${VERSION}$"; then
# If version is a tag, start from that tag
echo "$TAGS" | awk -v ver="$VERSION" 'BEGIN{found=0} {if($0==ver)found=1; if(found)print}' > "$TMPFILE"
else
# If version is not a tag, use HEAD as starting point
echo "$VERSION HEAD" > "$TMPFILE"
echo "$TAGS" >> "$TMPFILE"
fi
# Process tags and generate changelog
PREV_TAG=""
PREV_REF=""
FIRST=1
while IFS= read -r line; do
if [ -z "$line" ]; then
continue
fi
# Handle the special case of VERSION HEAD
if [ "$line" = "$VERSION HEAD" ]; then
TAG="$VERSION"
REF="HEAD"
else
TAG="$line"
REF="$line"
fi
if [ -n "$PREV_TAG" ]; then
# Get log entries between current and previous tag
LOG_ENTRIES=$(git_log "$REF" "$PREV_REF")
# Skip if no changes
if [ -z "$LOG_ENTRIES" ] || [ "$LOG_ENTRIES" = "" ]; then
PREV_TAG="$TAG"
PREV_REF="$REF"
continue
fi
print('{0} ({1}) {2}; urgency={3}\n'.format(args.name,tag[0],args.codename,args.urgency))
# Print package header
printf "%s (%s%s) %s; urgency=%s\n\n" "$NAME" "$PREV_TAG" "$VERSUFFIX" "$CODENAME" "$URGENCY"
for line in lines:
print(" * " + line)
# Print each log entry
echo "$LOG_ENTRIES" | while IFS= read -r entry; do
if [ -n "$entry" ]; then
printf " * %s\n" "$entry"
fi
done
authorandtime = git_author_and_time(tag[1])
print(' {0}\n'.format(authorandtime))
# Print author and time
AUTHOR_TIME=$(git_author_and_time "$PREV_REF")
printf "\n %s\n\n" "$AUTHOR_TIME"
fi
tag = prev
if __name__ == "__main__":
main()
PREV_TAG="$TAG"
PREV_REF="$REF"
done < "$TMPFILE"
+7 -16
View File
@@ -13,37 +13,28 @@ if [ -e /usr/bin/apt-get ]; then
debhelper \
lsb-release \
fakeroot
apt-get -qy --force-yes --no-install-suggests --no-install-recommends install python
apt-get -qy --force-yes --no-install-suggests --no-install-recommends install python3
elif [ -e /usr/bin/dnf ]; then
dnf -y update
dnf -y install \
git rpm-build gcc-c++ make which python3
git rpm-build gcc-c++ make which libatomic
dnf -y install lsb_release || true
elif [ -e /usr/bin/yum ]; then
yum -y update
yum -y install \
git rpm-build gcc-c++ make which \
python python-argparse
git rpm-build gcc-c++ make which
yum -y install lsb_release || true
yum -y install centos-release-scl
yum -y install devtoolset-9-gcc\*
elif [ -e /usr/bin/zypper ]; then
zypper update -y
zypper install -y \
git rpm-build gcc-c++ make which \
python
git rpm-build gcc-c++ make which \
lsb_release
elif [ -e /sbin/apk ]; then
apk add \
abuild git gcc g++ make \
linux-headers
linux-headers lsb-release
elif [ -e /usr/sbin/pkg ]; then
pkg install \
git gmake gcc
fi
if [ ! -e /usr/bin/python ]; then
if [ -e /usr/bin/python3 ]; then
ln -s /usr/bin/python3 /usr/bin/python
elif [ -e /usr/bin/python2 ]; then
ln -s /usr/bin/python2 /usr/bin/python
fi
fi
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
if [ -e /usr/bin/apt-get ]; then
export DEBIAN_FRONTEND=noninteractive
apt-get -qy update
apt-get -qy --force-yes --no-install-suggests --no-install-recommends install \
qemu-user-static \
podman
elif [ -e /usr/bin/dnf ]; then
dnf -y update
dnf -y install \
qemu-user-static \
podman
elif [ -e /sbin/apk ]; then
apk add \
python3 \
qemu-user-static \
podman
fi
+3 -1
View File
@@ -9,10 +9,12 @@ if [ "${VERSION}" = "" ]; then
VERSION="unknown"
fi
echo "version = ${VERSION}"
echo -n "${VERSION}" > VERSION
grep -q \"${VERSION}\" src/version.hpp
grep -sq \"${VERSION}\" src/version.hpp
RV=$?
if [ $RV -ne 0 ]; then
echo "Writing 'src/version.hpp'"
echo "#pragma once" > src/version.hpp
echo "static const char MERGERFS_VERSION[] = \"${VERSION}\";" >> src/version.hpp
fi
+5 -5
View File
@@ -4,10 +4,10 @@
export DH_VERBOSE=1
%:
dh $@ --parallel --max-parallel=6
override_dh_strip:
dh_strip --no-automatic-dbgsym
dh $@
override_dh_auto_install:
$(MAKE) DESTDIR=$(CURDIR)/debian/mergerfs PREFIX=/usr install
$(MAKE) \
DESTDIR=$(CURDIR)/debian/mergerfs \
PREFIX=/usr \
install
+74 -65
View File
@@ -1,4 +1,11 @@
VERSION = 2.9.7-mergerfs_2.30.0
AR ?= ar
CHMOD ?= chmod
CHOWN ?= chown
INSTALL ?= install
MKDIR ?= mkdir
RM ?= rm
STRIP ?= strip
TOUCH ?= touch
OS := $(shell uname -s)
ifeq ($(OS),Linux)
@@ -12,7 +19,7 @@ endif
ifeq ($(NDEBUG),1)
OPT_FLAGS := -O2 -DNDEBUG
else
OPT_FLAGS := -O0 -DDEBUG -g
OPT_FLAGS := -O0 -g -DDEBUG -fno-omit-frame-pointer
endif
ifeq ($(LTO),1)
@@ -21,21 +28,22 @@ else
LTO_FLAGS :=
endif
DESTDIR =
PREFIX = /usr/local
EXEC_PREFIX = $(PREFIX)
DATAROOTDIR = $(PREFIX)/share
DATADIR = $(DATAROOTDIR)
BINDIR = $(EXEC_PREFIX)/bin
SBINDIR = /sbin
MANDIR = $(DATAROOTDIR)/man
MAN1DIR = $(MANDIR)/man1
# https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
DESTDIR ?=
PREFIX ?= /usr/local
EXEC_PREFIX ?= $(PREFIX)
DATAROOTDIR ?= $(PREFIX)/share
DATADIR ?= $(DATAROOTDIR)
BINDIR ?= $(EXEC_PREFIX)/bin
SBINDIR ?= /sbin
MANDIR ?= $(DATAROOTDIR)/man
MAN1DIR ?= $(MANDIR)/man1
INSTALLBINDIR = $(DESTDIR)$(BINDIR)
INSTALLSBINDIR = $(DESTDIR)$(SBINDIR)
INSTALLMAN1DIR = $(DESTDIR)$(MAN1DIR)
INSTALLBINDIR ?= $(DESTDIR)$(BINDIR)
INSTALLSBINDIR ?= $(DESTDIR)$(SBINDIR)
INSTALLMAN1DIR ?= $(DESTDIR)$(MAN1DIR)
AR ?= ar
BUILDDIR := build
SRC_C = \
lib/buffer.c \
@@ -48,7 +56,7 @@ SRC_C = \
lib/fuse_signals.c \
lib/helper.c \
lib/mount.c
SRC_CPP = \
SRC_CXX = \
lib/fuse.cpp \
lib/cpu.cpp \
lib/fuse_config.cpp \
@@ -59,90 +67,91 @@ SRC_CPP = \
lib/node.cpp \
lib/maintenance_thread.cpp
OBJS_C = $(SRC_C:lib/%.c=build/%.o)
OBJS_CPP = $(SRC_CPP:lib/%.cpp=build/%.o)
DEPS_C = $(SRC_C:lib/%.c=build/%.d)
DEPS_CPP = $(SRC_CPP:lib/%.cpp=build/%.d)
OBJS_C = $(SRC_C:lib/%.c=$(BUILDDIR)/%.o)
OBJS_CXX = $(SRC_CXX:lib/%.cpp=$(BUILDDIR)/%.o)
DEPS_C = $(SRC_C:lib/%.c=$(BUILDDIR)/%.d)
DEPS_CXX = $(SRC_CXX:lib/%.cpp=$(BUILDDIR)/%.d)
CPPFLAGS ?=
CFLAGS ?= \
$(OPT_FLAGS)
CFLAGS := \
${CFLAGS} \
$(OPT_FLAGS) \
$(LTO_FLAGS) \
-Wall \
-pipe
override CFLAGS += \
-std=gnu99 \
-Wall \
-pipe \
-MMD
-MMD \
-MP
CXXFLAGS ?= \
$(OPT_FLAGS)
CXXFLAGS := \
${CXXFLAGS} \
$(OPT_FLAGS) \
$(LTO_FLAGS) \
-std=c++17 \
-Wall \
-pipe \
-MMD
FUSERMOUNT_DIR = $(BINDIR)
FUSE_FLAGS = \
-pipe
override CXXFLAGS += \
-std=c++17 \
-MMD \
-MP
FUSERMOUNT_DIR := $(BINDIR)
FUSE_FLAGS := \
-Iinclude \
-Ibuild \
-I$(BUILDDIR) \
-D_REENTRANT \
-D_FILE_OFFSET_BITS=64 \
-DFUSERMOUNT_DIR=\"$(FUSERMOUNT_DIR)\"
LDFLAGS := \
${LDFLAGS} \
LDFLAGS ?=
LDLIBS := \
-lrt \
-latomic \
-pthread
all: build/libfuse.a $(UTILS)
all: $(BUILDDIR)/libfuse.a $(UTILS)
build/config.h: build/stamp
ecfd/build | tee build/config.h
$(BUILDDIR)/stamp:
$(MKDIR) -p $(BUILDDIR)
$(TOUCH) $@
build/stamp:
mkdir -p build
touch $@
objects: build/config.h
$(MAKE) $(OBJS_C) $(OBJS_CPP)
build/libfuse.a: objects
${AR} rcs build/libfuse.a $(OBJS_C) $(OBJS_CPP)
$(BUILDDIR)/libfuse.a: $(OBJS_C) $(OBJS_CXX)
$(AR) rcs $(BUILDDIR)/libfuse.a $(OBJS_C) $(OBJS_CXX)
.PHONY: utils
utils: mergerfs-fusermount mount.mergerfs
build/mergerfs-fusermount: build/config.h util/fusermount.c lib/mount_util.c
$(CC) $(CFLAGS) $(FUSE_FLAGS) -Ilib -o build/mergerfs-fusermount util/fusermount.c lib/mount_util.c
$(BUILDDIR)/mergerfs-fusermount: util/fusermount.c lib/mount_util.c
$(CC) $(CFLAGS) $(FUSE_FLAGS) -Ilib -o $(BUILDDIR)/mergerfs-fusermount util/fusermount.c lib/mount_util.c
mergerfs-fusermount: build/mergerfs-fusermount
mergerfs-fusermount: $(BUILDDIR)/mergerfs-fusermount
build/mount.mergerfs: build/libfuse.a util/mount.mergerfs.c
$(CC) $(CFLAGS) $(FUSE_FLAGS) -o build/mount.mergerfs util/mount.mergerfs.c build/libfuse.a $(LDFLAGS)
$(BUILDDIR)/mount.mergerfs: $(BUILDDIR)/libfuse.a util/mount.mergerfs.c
$(CC) $(CFLAGS) $(FUSE_FLAGS) -o $(BUILDDIR)/mount.mergerfs util/mount.mergerfs.c $(BUILDDIR)/libfuse.a $(LDFLAGS)
mount.mergerfs: build/mount.mergerfs
mount.mergerfs: $(BUILDDIR)/mount.mergerfs
build/%.o: lib/%.c
$(BUILDDIR)/%.o: lib/%.c | $(BUILDDIR)/stamp
$(CC) $(CFLAGS) $(FUSE_FLAGS) -c $< -o $@
build/%.o: lib/%.cpp
$(BUILDDIR)/%.o: lib/%.cpp | $(BUILDDIR)/stamp
$(CXX) $(CXXFLAGS) $(FUSE_FLAGS) -c $< -o $@
.PHONY: clean
clean:
rm -rf build
$(RM) -rf $(BUILDDIR)
.PHONY: distclean
distclean: clean
.PHONY: strip
strip:
strip --strip-all build/mount.mergerfs
strip --strip-all build/mergerfs-fusermount
$(STRIP) --strip-all $(BUILDDIR)/mount.mergerfs
$(STRIP) --strip-all $(BUILDDIR)/mergerfs-fusermount
install-utils: mergerfs-fusermount mount.mergerfs strip
install -D build/mergerfs-fusermount "$(INSTALLBINDIR)/mergerfs-fusermount"
install -D build/mount.mergerfs "$(INSTALLSBINDIR)/mount.mergerfs"
chown root:root "$(INSTALLBINDIR)/mergerfs-fusermount"
chmod u+s "$(INSTALLBINDIR)/mergerfs-fusermount"
$(INSTALL) -D $(BUILDDIR)/mergerfs-fusermount "$(INSTALLBINDIR)/mergerfs-fusermount"
$(INSTALL) -D $(BUILDDIR)/mount.mergerfs "$(INSTALLSBINDIR)/mount.mergerfs"
$(CHOWN) root:root "$(INSTALLBINDIR)/mergerfs-fusermount"
$(CHMOD) u+s "$(INSTALLBINDIR)/mergerfs-fusermount"
install: $(INSTALLUTILS)
.PHONY: objects strip utils install install-utils
-include $(DEPS_C) $(DEPS_CPP)
-include $(DEPS_C) $(DEPS_CXX)
-16
View File
@@ -1,16 +0,0 @@
# ecfd: embeddable C feature discovery
This is a simple script and set of tests to help with what would traditionally be the `configure` step of most C/C++. It is intended to be simple to be embedded into a project and only depends on the C compiler and /bin/sh.
## USAGE
First copy `ecfd` into your project. Next simply call the build script which will run the tests and output a `config.` style file to `stdout`. Compile errors will output to `stderr` which can be used to understand why they aren't working.
```
$ ./ecfd/build > include/config.h
```
## TESTS
Each test is in a single module. The build script will try to compile each module and on success will execute the program. If anything is printed to `stdout` it will be appended to the `#define` printed to the build script's `stdout`. If the compilation fails nothing will be printed. Look at the tests directory for examples.
-32
View File
@@ -1,32 +0,0 @@
#!/bin/sh
PWD=$(pwd)
BASEPATH="${PWD}/${0%/*}"
CC=${CC:-cc}
OUTPUT="/dev/null"
echo "#ifndef CONFIG_H_INCLUDED"
echo "#define CONFIG_H_INCLUDED"
echo
IFS=
for file in "${BASEPATH}/tests/"*.c
do
binary="${file%.c}"
basename=$(basename ${binary})
${CC} -o "${binary}" "${file}" 1>&2
if [ $? -eq 0 ]; then
STDOUT=$(${binary})
if [ "${STDOUT}" != "" ]; then
echo "#define ${basename} ${STDOUT}"
else
echo "#define ${basename}"
fi
rm -f "${binary}"
fi
done
echo
echo "#endif"
-10
View File
@@ -1,10 +0,0 @@
#include <sys/types.h>
#include <unistd.h>
int
main(void)
{
(void)fork;
return 0;
}
-7
View File
@@ -1,7 +0,0 @@
#include <malloc.h>
int
main()
{
malloc_trim(0);
}
@@ -1,14 +0,0 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
int
main(int argc,
char *argv[])
{
struct stat st;
(void)st.st_atim;
return 0;
}
-11
View File
@@ -1,11 +0,0 @@
#include <fcntl.h>
#include <sys/stat.h>
int
main(int argc,
char *argv[])
{
(void)utimensat;
return 0;
}
-4
View File
@@ -1,4 +0,0 @@
#pragma once
#define STATIC_ASSERT(condition) \
((void)sizeof(char[1 - 2*!(condition)]))
+2 -1
View File
@@ -103,7 +103,8 @@ _mutex_lock(pthread_mutex_t *mutex_,
case ETIMEDOUT:
cnt++;
fmt::println(stderr,
"NOTICE: pthread_mutex_timedlock expired - count={}; ({}:{}:{})",
"NOTICE: pthread_mutex_timedlock expired - addr={}; count={}; ({}:{}:{})",
(void*)mutex_,
cnt,
file_,
func_,
+6 -7
View File
@@ -1,17 +1,16 @@
#pragma once
#include "config.h"
#ifdef HAVE_STRUCT_STAT_ST_ATIM
#ifdef __linux__
#define ST_ATIM_NSEC(ST) ((ST)->st_atim.tv_nsec)
#define ST_CTIM_NSEC(ST) ((ST)->st_ctim.tv_nsec)
#define ST_MTIM_NSEC(ST) ((ST)->st_mtim.tv_nsec)
#elif defined HAVE_STRUCT_STAT_ST_ATIMESPEC
#elif defined __FreeBSD__
#define ST_ATIM_NSEC(ST) ((ST)->st_atimespec.tv_nsec)
#define ST_CTIM_NSEC(ST) ((ST)->st_ctimespec.tv_nsec)
#define ST_MTIM_NSEC(ST) ((ST)->st_mtimespec.tv_nsec)
#else
#define ST_ATIM_NSEC(ST) 0
#define ST_CTIM_NSEC(ST) 0
#define ST_MTIM_NSEC(ST) 0
#error "Unsupported platform for st_{a,c,m}time nanosecond"
/* #define ST_ATIM_NSEC(ST) 0 */
/* #define ST_CTIM_NSEC(ST) 0 */
/* #define ST_MTIM_NSEC(ST) 0 */
#endif
-1
View File
@@ -8,7 +8,6 @@
#define _GNU_SOURCE
#include "config.h"
#include "fuse_i.h"
#include "fuse_lowlevel.h"
#include <string.h>
+507 -507
View File
File diff suppressed because it is too large Load Diff
+9 -14
View File
@@ -11,18 +11,18 @@
#define _GNU_SOURCE
#endif
#include "syslog.hpp"
#include "crc32b.h"
#include "khash.h"
#include "kvec.h"
#include "mutex.hpp"
#include "node.hpp"
#include "config.h"
#include "fuse_dirents.h"
#include "fuse_i.h"
#include "fuse_kernel.h"
#include "fuse_lowlevel.h"
#include "fuse_misc.h"
#include "fuse_opt.h"
#include "fuse_pollhandle.h"
#include "fuse_msgbuf.hpp"
@@ -51,11 +51,10 @@
#include <sys/param.h>
#include <sys/time.h>
#include <sys/uio.h>
#include <syslog.h>
#include <time.h>
#include <unistd.h>
#ifdef HAVE_MALLOC_TRIM
#ifdef __GLIBC__
#include <malloc.h>
#endif
@@ -1848,7 +1847,6 @@ fuse_lib_setattr(fuse_req_t req,
f->fs->op.truncate(path,arg->size) :
f->fs->op.ftruncate(fh,arg->size));
#ifdef HAVE_UTIMENSAT
if(!err && (arg->valid & (FATTR_ATIME | FATTR_MTIME)))
{
struct timespec tv[2];
@@ -1872,9 +1870,7 @@ fuse_lib_setattr(fuse_req_t req,
f->fs->op.utimens(path,tv) :
f->fs->op.futimens(fh,tv));
}
else
#endif
if(!err && ((arg->valid & (FATTR_ATIME|FATTR_MTIME)) == (FATTR_ATIME|FATTR_MTIME)))
else if(!err && ((arg->valid & (FATTR_ATIME|FATTR_MTIME)) == (FATTR_ATIME|FATTR_MTIME)))
{
struct timespec tv[2];
tv[0].tv_sec = arg->atime;
@@ -3971,7 +3967,7 @@ static
void
fuse_malloc_trim(void)
{
#ifdef HAVE_MALLOC_TRIM
#ifdef __GLIBC__
malloc_trim(1024 * 1024);
#endif
}
@@ -4003,9 +3999,8 @@ fuse_invalidate_all_nodes()
}
mutex_unlock(&f->lock);
syslog(LOG_INFO,
"invalidating %ld file entries",
names.size());
SysLog::info("invalidating {} file entries",
names.size());
for(auto &name : names)
{
fuse_lowlevel_notify_inval_entry(f->se->ch,
@@ -4018,7 +4013,7 @@ fuse_invalidate_all_nodes()
void
fuse_gc()
{
syslog(LOG_INFO,"running thorough garbage collection");
SysLog::info("running thorough garbage collection");
node_gc();
msgbuf_gc();
fuse_malloc_trim();
@@ -4027,7 +4022,7 @@ fuse_gc()
void
fuse_gc1()
{
syslog(LOG_INFO,"running basic garbage collection");
SysLog::info("running basic garbage collection");
node_gc1();
msgbuf_gc_10percent();
fuse_malloc_trim();
-1
View File
@@ -13,7 +13,6 @@
#include "fuse_i.h"
#include "fuse_kernel.h"
#include "fuse_lowlevel.h"
#include "fuse_misc.h"
#include "fuse_config.hpp"
#include "fuse_msgbuf.hpp"
+3 -4
View File
@@ -13,14 +13,13 @@
#include "mutex.hpp"
#include "lfmp.h"
#include "config.h"
#include "debug.hpp"
#include "fuse_i.h"
#include "fuse_kernel.h"
#include "fuse_opt.h"
#include "fuse_misc.h"
#include "fuse_pollhandle.h"
#include "fuse_msgbuf.hpp"
#include "fuse_opt.h"
#include "fuse_pollhandle.h"
#include "stat_utils.h"
#include <stdio.h>
#include <stdlib.h>
-49
View File
@@ -1,49 +0,0 @@
/*
FUSE: Filesystem in Userspace
Copyright (C) 2001-2007 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU LGPLv2.
See the file COPYING.LIB
*/
#pragma once
#include "config.h"
#include <pthread.h>
/*
Versioned symbols cannot be used in some cases because it
- confuse the dynamic linker in uClibc
- not supported on MacOSX (in MachO binary format)
*/
#if (!defined(__UCLIBC__) && !defined(__APPLE__))
#define FUSE_SYMVER(x) __asm__(x)
#else
#define FUSE_SYMVER(x)
#endif
#ifdef HAVE_STRUCT_STAT_ST_ATIM
/* Linux */
#define ST_ATIM_NSEC(stbuf) ((stbuf)->st_atim.tv_nsec)
#define ST_CTIM_NSEC(stbuf) ((stbuf)->st_ctim.tv_nsec)
#define ST_MTIM_NSEC(stbuf) ((stbuf)->st_mtim.tv_nsec)
#define ST_ATIM_NSEC_SET(stbuf,val) ((stbuf)->st_atim.tv_nsec = (val))
#define ST_CTIM_NSEC_SET(stbuf,val) ((stbuf)->st_ctim.tv_nsec = (val))
#define ST_MTIM_NSEC_SET(stbuf,val) ((stbuf)->st_mtim.tv_nsec = (val))
#elif defined(HAVE_STRUCT_STAT_ST_ATIMESPEC)
/* FreeBSD */
#define ST_ATIM_NSEC(stbuf) ((stbuf)->st_atimespec.tv_nsec)
#define ST_CTIM_NSEC(stbuf) ((stbuf)->st_ctimespec.tv_nsec)
#define ST_MTIM_NSEC(stbuf) ((stbuf)->st_mtimespec.tv_nsec)
#define ST_ATIM_NSEC_SET(stbuf,val) ((stbuf)->st_atimespec.tv_nsec = (val))
#define ST_CTIM_NSEC_SET(stbuf,val) ((stbuf)->st_ctimespec.tv_nsec = (val))
#define ST_MTIM_NSEC_SET(stbuf,val) ((stbuf)->st_mtimespec.tv_nsec = (val))
#else
#define ST_ATIM_NSEC(stbuf) 0
#define ST_CTIM_NSEC(stbuf) 0
#define ST_MTIM_NSEC(stbuf) 0
#define ST_ATIM_NSEC_SET(stbuf,val) do { } while (0)
#define ST_CTIM_NSEC_SET(stbuf,val) do { } while (0)
#define ST_MTIM_NSEC_SET(stbuf,val) do { } while (0)
#endif
-1
View File
@@ -7,7 +7,6 @@
*/
#include "fuse_opt.h"
#include "fuse_misc.h"
#include <stdio.h>
#include <stdlib.h>
-1
View File
@@ -7,7 +7,6 @@
*/
#include "fuse_i.h"
#include "fuse_misc.h"
#include "fuse_kernel.h"
#include <assert.h>
-2
View File
@@ -6,9 +6,7 @@
See the file COPYING.LIB.
*/
#include "config.h"
#include "fuse_i.h"
#include "fuse_misc.h"
#include "fuse_opt.h"
#include "fuse_lowlevel.h"
-6
View File
@@ -6,9 +6,7 @@
See the file COPYING.LIB.
*/
#include "config.h"
#include "fuse_i.h"
#include "fuse_misc.h"
#include "fuse_opt.h"
#include "mount_util.h"
@@ -41,10 +39,6 @@
#define FUSERMOUNT_PROG "fusermount"
#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
#ifndef HAVE_FORK
#define fork() vfork()
#endif
#ifndef MS_DIRSYNC
#define MS_DIRSYNC 128
#endif
+3 -4
View File
@@ -1,6 +1,6 @@
#include "pin_threads.hpp"
#include <syslog.h>
#include "syslog.hpp"
void
PinThreads::R1L(const CPU::ThreadIdVec threads_)
@@ -211,7 +211,6 @@ PinThreads::pin(const CPU::ThreadIdVec read_threads_,
if(type_ == "R1PPSP")
return PinThreads::R1PPSP(read_threads_,process_threads_);
syslog(LOG_WARNING,
"Invalid pin-threads type, ignoring: %s",
type_.c_str());
SysLog::warning("Invalid pin-threads type, ignoring: {}",
type_);
}
+1 -2
View File
@@ -8,7 +8,6 @@
/* This program does the mounting and unmounting of FUSE filesystems */
#define _GNU_SOURCE /* for clone */
#include <config.h>
#include "mount_util.h"
#include <stdio.h>
@@ -30,7 +29,7 @@
#include <sys/utsname.h>
#include <sched.h>
#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
#define FUSE_COMMFD_ENV "_FUSE_COMMFD"
#define FUSE_DEV_OLD "/proc/fs/fuse/dev"
#define FUSE_DEV_NEW "/dev/fuse"
+3 -3
View File
@@ -13,9 +13,9 @@ BuildRequires: gcc-c++
# rpmbuild driven by the Makefile uses git to generate a version number
BuildRequires: git
Requires: fuse
%global debug_package %{nil}
%undefine _debuginfo_subpackages
%global _enable_debug_packages 0
%prep
%setup -q
@@ -26,7 +26,7 @@ management of files across numerous commodity storage devices. It is
similar to mhddfs, unionfs, and aufs.
%build
make %{?_smp_mflags}
make %{?_smp_mflags} CFLAGS="%{optflags}" CXXFLAGS="%{optflags}" LDFLAGS="%{__global_ldflags}"
%install
make install PREFIX=%{_prefix} DESTDIR=%{buildroot}
+5 -3
View File
@@ -3,6 +3,11 @@
* `fuse_msg_size=UINT|SIZE`
* Defaults to `1M`
* Performance improvements often peak at about `4M`
* If the value is a `SIZE` in the form of `xB`, `xK`, `xM`, `xG`, or
`xT` then it will choose the closest multiple of the system page
size
* If the value is just an integer without a size multipler suffix then
it is treated as a page count
FUSE applications communicate with the kernel over a special character
device: `/dev/fuse`. A large portion of the overhead associated with
@@ -42,8 +47,5 @@ On kernels before v4.20 the option has no effect. On kernels between
v4.20 and v6.13 the max value is 256. On kernels >= v6.13 the maximum
value is 65535.
Since page size can differ between systems mergerfs can take a value in
bytes and will convert it to the proper number of pages (rounded up).
NOTE: If you intend to enable `cache.files` you should also set
[readahead](readahead.md) to match `fuse_msg_size`.
@@ -1,4 +1,4 @@
# Usage Patterns
# Extended Usage Patterns
## tiered cache
@@ -2,16 +2,20 @@
## What should mergerfs NOT be used for?
- databases: Even if the database stored data in separate files
* databases: Even if the database stored data in separate files
(mergerfs wouldn't offer much otherwise) the higher latency of the
indirection will really harm performance. If it is a lightly used
sqlite3 database then it should be fine.
- VM images: For the same reasons as databases. VM images are accessed
* VM images: For the same reasons as databases. VM images are accessed
very aggressively and mergerfs will introduce a lot of extra latency.
- As replacement for RAID: mergerfs is just for pooling branches. If
* As replacement for RAID: mergerfs is just for pooling branches. If
you need that kind of device performance aggregation or high
availability you should stick with RAID. However, it is fine to put
a filesystem which is on a RAID setup in mergerfs.
**However, if using [passthrough](../config/passthrough.md) the above
situations are less likely to be a concern. Best to do testing for
your specific use case.**
## It's mentioned that there are some security issues with mhddfs. What are they? How does mergerfs address them?
@@ -23,6 +23,19 @@ directory in the mergerfs pool that includes all the paths you need if
you want links and rename to work.
## Do reflink, FICLONE, or FICLONERANGE work?
Unfortunately not. FUSE, the technology mergerfs is based on, does not
support the `clone_file_range` feature needed for it to work. mergerfs
won't even know such a request is made. The kernel will simply return
an error back to the application making the request.
Should FUSE gain the ability mergerfs will be updated to support it.
That said: mergerfs itself will attempt to use `FICLONE` when copying
data between files in the limited cases where it copies/moves files.
## How does mergerfs handle moving and copying of files?
This is a _very_ common mistaken assumption regarding how filesystems
@@ -73,15 +86,8 @@ final destination. That temporary file name will typically be random
and have no indication of the type of file being written. At best
something could be done on rename.
## Does FICLONE or FICLONERANGE work?
Unfortunately not. FUSE, the technology mergerfs is based on, does not
support the `clone_file_range` feature needed for it to work. mergerfs
won't even know such a request is made. The kernel will simply return
an error back to the application making the request.
Should FUSE gain the ability mergerfs will be updated to support it.
Additional reading: [Intro to Filesystems:
Workflows](../intro_to_filesystems.md#workflows)
## Why does the total available space in mergerfs not equal outside?
@@ -172,20 +178,26 @@ removed to simplify the codebase.
mergerfs is a multithreaded application in order to handle requests
from the kernel concurrently. Each FUSE message has a header with
certain details about the request include the process ID (pid) of the
requesting application, the process' effective user id (uid), and
certain details about the request including the process ID (pid) of
the requesting application, the process' effective user id (uid), and
group id (gid). To ensure proper POSIX filesystem behavior and
security mergerfs must change its identity to match that of the
requester when performing the core filesystem function on the
underlying filesystem. On most Unix/POSIX based system a process and
all its threads are under the same uid and gid. However, on Linux each
thread may have its own credentials. This allows mergerfs to be
multithreaded and for each thread to change to the credentials
(seteuid,setegid) as required by the incoming message it is
handling. However, on FreeBSD this is not possible at the moment
(though there has been
[discussions](https://wiki.freebsd.org/Per-Thread%20Credentials) and
requester when performing the certain functions on the underlying
filesystem. As required by standards most Unix/POSIX based systems a
process and all its threads are under the same uid and gid. However,
on Linux each thread **may** have its own credentials. This allows
mergerfs to be multithreaded and for each thread to change to the
credentials as required by the incoming message it is
handling. However, currently on FreeBSD this is not possible (though
there has been
[discussions](https://wiki.freebsd.org/Per-Thread%20Credentials)) and
as such must change the credentials of the whole application when
actioning messages. mergerfs does optimize this behavior by only
changing credentials and locking the thread to do so if the process is
currently not the same as what is necessary by the incoming request.
currently not the same as what is necessary by the incoming
request. As a result of this design FreeBSD may experience more
contention and therefore lower performance than Linux.
Additionally, mergerfs [utilizes a cache for supplemental
groups](../known_issues_bugs.md#supplemental-user-groups) due the the
high cost of querying that information.
@@ -1,5 +1,14 @@
# Usage and Functionality
## What happens when file paths overlap?
It depends on the situation and the configuration of mergerfs. The
user chooses the algorithm (policy) on how to behave when a particular
filesystem function is requested. Which file to select or act on.
See how [policies](../config/functions_categories_policies.md) work.
## Can mergerfs be used with filesystems which already have data?
Yes. mergerfs is really just a proxy and does **NOT** interfere with
+26 -20
View File
@@ -4,23 +4,27 @@
[FUSE](https://en.wikipedia.org/wiki/Filesystem_in_Userspace) based
[union filesystem](https://en.wikipedia.org/wiki/Union_mount) geared
towards simplifying storage and management of files across numerous
commodity storage devices. It is similar to **mhddfs**, **unionfs**,
and **aufs**.
commodity storage devices. It is similar to [**mhddfs**, **unionfs**,
**aufs**, **DrivePool**, etc.](project_comparisons.md)
## Features
* Logically combine numerous filesystems/paths into a single
mount point
mount point (JBOFS: Just a Bunch of FileSystems)
* Combine paths of the same or different filesystems
* Ability to add or remove filesystems/paths without impacting the
rest of the data
* Unaffected by individual filesystem failure
* Configurable file selection and creation placement
* File IO [passthrough](config/passthrough.md) for near native IO
performance (where supported)
* Works with filesystems of any size
* Works with filesystems of almost any type
* Works with filesystems of [almost any
type](faq/compatibility_and_integration.md#what-filesystems-can-be-used-as-branches)
* Ignore read-only filesystems when creating files
* Hard link copy-on-write / CoW
* Runtime configurable
* Hard link [copy-on-write / CoW](config/link_cow.md)
* [Runtime configurable](runtime_interface.md)
* Support for extended attributes (xattrs)
* Support for file attributes (chattr)
* Support for POSIX ACLs
@@ -30,7 +34,7 @@ and **aufs**.
* Read/write overlay on top of read-only filesystem like OverlayFS
* File whiteout
* RAID like parity calculation
* RAID like parity calculation (see [SnapRAID](https://www.snapraid.it))
* Redundancy
* Splitting of files across branches
@@ -64,21 +68,23 @@ A + B = C
| | |
+-- /dir1 +-- /dir1 +-- /dir1
| | | | | |
| +-- file1 | +-- file2 | +-- file1
| | +-- file3 | +-- file2
+-- /dir2 | | +-- file3
| | +-- /dir3 |
| +-- file4 | +-- /dir2
| +-- file5 | |
+-- file6 | +-- file4
|
+-- /dir3
| |
| +-- file5
|
+-- file6
| +-- file1 | | | +-- file1
| | +-- file2 | +-- file2
| | +-- file3 | +-- file3
| | |
+-- /dir2 | +-- /dir2
| | | | |
| *-- file4 | | +-- file4
| | |
| +-- /dir3 +-- /dir3
| | | | |
| | +-- file5 | +-- file5
| | |
+-- file6 | +-- file6
+-- file7 +-- file7 +-- file7
```
## Getting Started
Head to the [quick start guide](quickstart.md).
+331 -17
View File
@@ -1,6 +1,6 @@
# Intro to Filesystems
WORK IN PROGRESS
**!! WORK IN PROGRESS !!**
mergerfs is a union filesystem that manages other filesystems. To
understand how mergerfs works and what the features do and mean you
@@ -10,38 +10,352 @@ filesystems. This section of the documentation is provide a primer for
those needing that knowledge.
## terminology
## Terminology
* [filesystem](https://en.wikipedia.org/wiki/File_system): A method,
data structures, and API used to organize and store files on
computer storage devices. Often in a tree like structure / hierarchy.
* [file](https://en.wikipedia.org/wiki/Computer_file): A named
collection of data stored as a unique object. Filesystems can have
numerous file types and "file" can be used to reference any of those
types or a "regular" file.
* regular file: A file as commonly understood as a named collection of data.
* [directory](https://en.wikipedia.org/wiki/Directory_(computing)): A
container which holds files and, typically, other directories. In
Unix/Linux/POSIX filesystems a directory is technically also a file.
* root filesystem: The primary filesystem for which other filesystems
within a system are mounted.
* [root directory](https://en.wikipedia.org/wiki/Root_directory): The
top-level directory in POSIX and POSIX-like filesystems. Denoted by
"/".
* [path](https://en.wikipedia.org/wiki/Path_(computing)): The location
of a file within a filesystem hierarchy.
* [inode](https://en.wikipedia.org/wiki/Inode): A unique integer
identifier for a file. Like a primary key. Also used to refer to the
underlying data structure which stores metadata about the file.
* [link](https://en.wikipedia.org/wiki/Hard_link): A reference to an
inode in the form of a name. An inode may have 0 to N links.
* [symlink](https://en.wikipedia.org/wiki/Symbolic_link): A file type
which stores data that may, but is not required to, be a path to
another file.
* device id: A runtime, unique identification value assigned to a
filesystem.
* [mount
point](https://en.wikipedia.org/wiki/Mount_(computing)#MOUNT-POINT):
The directory where a filesystem is "mounted" or presented within a
directory hierarchy.
* [permissions](https://en.wikipedia.org/wiki/File-system_permissions):
Access rights for files.
* [owner](https://en.wikipedia.org/wiki/User_identifier): System user
which has rights to a file.
* [file descriptor](https://en.wikipedia.org/wiki/File_descriptor): A
handle used by software, provided by the operating system, to
reference open files.
## files
## Files
### types
In POSIX filesystems there are several types of files. Interacting
with them is often similar but they offer different capabilities.
* regular:
* directory:
* symlink:
* fifo:
* unix domain socket:
### Types
* regular: The typical file type. An array of data which can be
randomly accessed.
* directory: A container which holds other files including other
directories allowing for a tree like layout.
* symlink: A special file which can store a limited amount of data,
typically a file path, which may be transparently interpreted by
some filesystem functions as a path. symlinks do *not* need to be a
valid path or a path at all. They can be used as a key/value like
storage mechanism for instance.
* character device: A special file typically representing some
physical or logical hardware device offering, typically, an
unbuffered byte stream based interface.
* block device: A special file which represents a physical or logical
hardware device (typically storage) allowing for buffered,
fixed-size blocks data transfers.
* fifo: Also referred to as a "named pipe." A FIFO, First In, First
Out, is a special file which acts as a non-persistent,
unidirectional conduit for data.
* unix domain socket: Also referred to as a "local socket" or "IPC
socket" a "UDS" works similar to a FIFO but bidirectional and
offering more features similar to TCP/IP or UDP/IP but local to the
system.
### inodes
### names
The inode in practical terms **is** the file within a POSIX
filesystem. It is, typically, a unique identifier within a filesystem
similar to a primary key, represented as a 32 or 64bit integer, which
is what the typical user imagines as the file and its metadata
(ownership, permissions, type, internal filesystem details, etc.).
"typically unique" because some filesystems, such as FUSE based ones,
may reuse an inode value for multiple logical files despite them being
separate entities in the traditional sense. From the user's
perspective multiple files may have the same inode but as mentioned
above the inode is the true reference for a file and the "file" people
interact with is really a link (see below.)
### links / names
To make inodes more useful and accessible POSIX filesystems provide
"links". A link (or sometimes referred to as "hardlink") is a file
name which references an inode. A single inode can be referenced by
multiple links (except for directories typically.) If the number of
links to an inode reach zero, and no programs have the inode actively
in use, the filesystem will logically remove the inode/file.
When a regular file is first created think of it as the filesystem
first creating the underlying structure, assigning an inode, then
creating a link (name) for it in one shot. When a file is "link"ed to
in the future a new "link" / "name" is being created and referencing
the source file which will then cause the "nlink" value within the
inode, which holds a reference count of the number of links/names that
inode has, to be increased by 1. This is why deleting a file is
referred to as an "unlink". Technically you are requesting the removal
of the "link" and once all the links to a file are removed/unlinked,
and the file is not in active use, the filesystem will then logically
remove the file.
Knowing the details above the API for POSIX filesystems is a little
odd in that the programmer does not first create an inode then a
link. Instead that is done together with no way to do the former alone
despite being able to add additional links afterwards. Modern Linux
filesystems do allow for this using the
[O_TMPFILE](https://man7.org/linux/man-pages/man2/openat.2.html)
flag. Since there is no link on creation should the program which
created the file then release the handle on that file the filesystem
would see it has no references and remove it. To keep the temporary
file around the program must explicitly create a link.
### permissions
### owership
File permissions are a security mechanism and apply to all files and
directories. They, in combination with ownership information, dictate
who is allowed to read, write, and execute a file (as well as some
other niche allowances and behaviors.)
While there are more advanced and complicated forms of permissioning
the traditional POSIX permissions come in the form of 3 sets of 3
permissions (the read, write, and execute.) One set for the individual
user owner of the file, one for the group owner, and one for "others".
Execute in POSIX systems indicates that a regular file is a piece of
software and can be run (like a Windows ".exe" file) and for
directories means you are allowed to list the contents. A directory
can be marked as "read" and not "executable" to allow users to access
content within the directory but not list it in a soft form of
security through obscurity. You must know the path and name of the
file to read it and can not simply search for it.
## functions
### ownership
Each file has 2 owner values. The individual user identifier and the
group identifier. These values are the raw integer value known to the
kernel and not the human readable user name or group name most people
typically use. In fact the kernel has no awareness of the human
readable identifiers. Those are managed entirely by userspace. This is
why if you mount a filesystem on a different system with different
users tools like `ls` may report different user names or only show
integer values for owners because the value stored in the inode is
"1000" and the name associated with integer 1000 on system A is
different from system B (or may not exist at all.)
## workflows
### timestamps
### creating a file
* mtime: The time the file *data* was last modified. Can be explicitly modified.
* ctime: The time the file *metadata* was last modified. Can not be modified.
* atime: The time the file was last accessed. Note that Linux
generally uses a heuristic to update atime so that it does not in
fact have to write to the filesystem every time a file is
accessed. Can be explicitly modified.
* btime: The "birth" time of the file. Only available in certain
filesystems and via a newer system call. Can not be modified.
### reading or writing to a file
### copying a file
## Functions
### moving a file
* open():
* creat():
* close():
* opendir():
* readdir():
* closedir():
* mkdir():
* link():
* unlink():
* symlink():
* rename():
* read():
* write():
* lseek():
* tell():
* stat():
## Workflows
Filesystems tend to be
[misunderstood](faq/technical_behavior_and_limitations.md#how-does-mergerfs-handle-moving-and-copying-of-files). A
number of behaviors users are familiar with are actually high level
concepts which are made up of numerous steps/functions. As a result
high level context that users expect to be available to mergerfs when
certain actions are taken and decisions made are in fact not
available. Below are explanations of a number of those situations.
### Creating a file
[Creating](https://www.man7.org/linux/man-pages/man3/creat.3p.html) a
file can range from a single call to a semi-complex procedure
depending on the requirements and expectations of the calling
application.
Most important to understand is that when a file is created there is
very limited information needed / provided. The file path, some flags
to control a bit how the file is created and if it is to be for
reading and/or writing, and the permissions. mergerfs knows a bit more
details like which user id is making the request and from what process
but that is all the data immediately available. There is no size. The
initial size is always zero. Any data that may be written to the file
would come in the future via other function calls. The source of that
data is completely unknown.
Keep in mind that while there exists this ability to create a file in
a single call to the operating system that call has numerous
aspects which my be exploitable by malicious software.
If software wishes to create a file in more secure manner it must do
some form of the following.
1. Attempt to open the directory where you wish to create the file.
2. If that succeeds check the directory to ensure it is the directory
you expect it to be. Checking ownership, permissions, perhaps
existence of other files, the filesystem, etc.
3. Using the handle to the directory, attempt to open the file using a
flag which make the open fail should a file name/link already exist
with that same name. Also set the permissions to the absolute minimum
possible to ensure no interference from others.
4. If it succeeds then continue on with whatever you had planned. If
it fails try to unlink the unexpected file and try again at step 3. If
it fails multiple times then something is suspicious and the program
should stop trying to create the file and report an error.
The above can be made even more involved as you will see below.
### Reading or writing to a file
To write:
1. Assuming the file is already created/opened...
2. Allocate memory to hold a portion of the data.
3. Fill the memory with the data to write from wherever it may come
from. Another file, network, programmatically generated, etc.
4. Call `write()` function on the open file pointing to the memory to
write and how much to write.
5. The kernel will take the request and attempt it. On success it will
report back to the program how much was written from 1 byte to the
amount requested. A write of fewer than the requested amount is
called a `short write.`
6. If a short write occurred the program must then adjust the data to
ensure it doesn't write the same data again and lower the amount to
write by the previously written amount and attempt another
write. Continuing this till there is nothing left to write. A short
write could also mean that the filesystem has no more available
space and following write requests may fail completely.
7. Go back to step 2 or 3 and do it all again with any additional data
to write.
8. If at any time there is an error it may be necessary to query the
size of the file or other details to know how to continue depending
on the needs of the program.
Reading is similar and `short reads` are also possible.
### Copying a file
The "proper" way to copy a file is quite involved.
1. Open the source file. Do so with certain flags to ensure it is the
type of file you expect. You may also need to perform a similar
"open directory, check things out, then open file" workflow as
mentioned above for security purposes.
2. `stat` the file to get its metadata such as the file size,
timestamps, etc.
3. Securely open a file as described previously. The name of the file
should not be that of the destination but a temporary file name
with some amount of randomness. Often also making it "hidden" by
prepending a "." to it.
4. In a loop read data from the source file and write it to the
temporary file as described above. There are some platform specific
ways to copy the data which may be tried first such as
FICLONE/reflink which takes advantage of CoW (copy-on-write)
filesystem features. Falling back to the read then write loop if
necessary.
5. Request a list of all the extended attribute keys from the source
file and one by one query the value. For each key value pair
attempt to write it to the destination file.
6. Request file attributes from the source file and write them to the
destination file but first filtering out flags such as "immutable"
as that would make it impossible to make further changes.
7. Using the details captured in step 2 change the ownership to the
destination file to match the source.
8. Using the details captured in step 2 change the permissions (mode)
of the destination file to match the source.
9. Using the details captured in step 2 change the mtime and atime of
the destination to match the source.
10. Query the source file's metadata again as in step 2. Using those
details as well as those from step 2 check the source file to see
if it may have changed since the copy procedure started. If it had
determine the changes and go back to step 4. Perhaps unlinking the
temporary file and going back to step 2.
11. Ensure the data and metadata are flushed to the storage device by
calling appropriate sync functions.
12. `rename` the file from the temporary name to the destination name.
13. If the file had a special flag such as the immutable attribute set
then set such attribute.
14. Close the destination and source files.
There can be additional steps too depending on the platform, available
features, and safety concerns. For instance it might be possible to
put a lock or lease on the source file to limit possibility of
changes while the copy occurs.
### Moving a file
The moving of a file is similar to copying a file.
1. Attempt a `rename` between source and destination. If it succeeds,
done. Note that the rename itself may require setup similar to
securely opening or creating a file. IE... requiring the opening of
the source and destination directories, confirming they are what is
expected, then issuing the actual rename.
2. If it fails... perform a copy of the file as described above.
3. After the rename and close'ing of the copy `unlink` the source
file.
### Summary
As seen in the above descriptions these high level concepts users
perceive are in fact involved sets of steps with loops and conditional
behaviors. All leveraging low level functions which alone have very
little context. For instance: there is no practical way of knowing a
file is being copied or from where. There is no way to decide at
creation time what to do based on size because its size doesn't exist
yet. Even making a decision based on the destination file name is
complicated by the fact that "well written" software will create
randomly named temporary files and then rename the file. Only at
`rename` would you know the name as the end user chose and at that
point it may be too late.
+73 -33
View File
@@ -2,6 +2,23 @@
## mergerfs
### FreeBSD version
* FreeBSD doesn't have per thread credentials meaning threads must
block to change credentials as required by numerous filesystem
functions. This impacts performance.
* FreeBSD's FUSE implementation is lacking many features of Linux.
* passthrough
* statx
* lazy umount
* oom_score_adj
* fuse_msg_size
* kernel symlink caching
* kernel readdir caching
* writeback caching
* ...
### Supplemental user groups
#### Supplemental group caching
@@ -74,6 +91,55 @@ on [page caching](config/cache.md).
## 3rd Party Software
### SQLite3, Plex, Jellyfin do not work with mergerfs
It does. If you're trying to put the software's config / metadata /
database on mergerfs you can't set [cache.files=off](config/cache.md)
(unless you use Linux v6.6 or above and
[direct-io-allow-mmap](config/options.md) is enabled) because they are
using **sqlite3** with **mmap** enabled and have failed to properly
handle the situation where **mmap** may not be available.
That said it is recommended that config and runtime files be stored on
SSDs on a regular filesystem for performance reasons. See [What should
mergerfs NOT be used
for?](faq/recommendations_and_warnings.md#what-should-mergerfs-not-be-used-for).
Other software that leverages **sqlite3** which require **mmap**
includes Radarr, Sonarr, and Lidarr. However, many programs use
**sqlite3** and do not require **mmap**.
It is recommended that you reach out to the developers of the software
you are having troubles with and ask them to add a fallback to regular
file IO when **mmap** is unavailable. It is not only more compatible
but also can be more performant in certain situations.
If the issue is that quick scans do not seem to pick up media then be
sure to set `func.getattr=newest`. That said a full scan will pick up
all media and it will put less load on the host to use time based
library scans or to configure downloading software to trigger a scan
when files are added to the pool. See [Does inotify and fanotify
work?](faq/compatibility_and_integration.md#does-inotify-and-fanotify-work)
for more details.
### backup software
Some software, like borgbackup and others, leverage inodes as one of a
number of metrics to notice changes in files. Depending on how [inode
generation is configured](config/inodecalc.md) it is possible that the
same file may have different inodes one mount to another and as such
interfere with the expected stability of the value. In such cases you
may need to disable the use of the inode in the software if possible
or change the [inodecalc](config/inodecalc.md) setting if possible.
Generally speaking, in modern setups where there are many non-fully
complient POSIX filesystem in use, reliance on inode to have a
specific meaning is not always reasonable. FUSE, for instance, has no
requirement that inode values need to be unique or that two files that
happen to be the same underlying file have the same inode.
### NFS clients returning ESTALE / Stale file handle
NFS generally does not like out of band changes. Take a look at the
@@ -81,36 +147,6 @@ section on NFS in the [remote-filesystems](remote_filesystems.md) for
more details.
### SQLite3, Plex, Jellyfin do not work with mergerfs
It does. If you're trying to put the software's config / metadata /
database on mergerfs you can't set
[cache.files=off](config/cache.md) (unless you use Linux v6.6 or
above) because they are using **sqlite3** with **mmap** enabled.
That said it is recommended that config and runtime files be stored on
SSDs on a regular filesystem for performance reasons. See [What should
mergerfs NOT be used for?](faq/recommendations_and_warnings.md).
Other software that leverages **sqlite3** which require **mmap**
includes Radarr, Sonarr, and Lidarr. That said many programs use
**sqlite3** and do not require **mmap**.
It is recommended that you reach out to the developers of the software
you are having troubles with and asking them to add a fallback to
regular file IO when **mmap** is unavailable. It is not only more
compatible and resilient but also can be more performant in certain
situations.
If the issue is that quick scanning doesn't seem to pick up media then
be sure to set `func.getattr=newest`. That said a full scan will pick
up all media and it will put less load on the host to use time based
library scans or to configure downloading software to trigger a scan
when files are added to the pool. See [Does inotify and fanotify
work?](faq/compatibility_and_integration.md#does-inotify-and-fanotify-work)
for more details.
### my 32bit software has problems
Some software have problems with 64bit inode values. The symptoms can
@@ -118,6 +154,7 @@ include EOVERFLOW errors when trying to list files. You can address
this by setting `inodecalc` to one of the 32bit based algos as
described in the relevant section.
### Moving files and directories fails with Samba
Workaround: Copy the file/directory and then remove the original
@@ -169,7 +206,7 @@ the [mergerfs-tools](https://github.com/trapexit/mergerfs-tools) tool
## FUSE and Linux kernel
There have been a number of kernel issues / bugs over the years which
mergerfs has run into. Here is a list of them for reference and
mergerfs users have run into. Here is a list of them for reference and
posterity.
@@ -225,14 +262,17 @@ lookup which should work across any kernel version.
### Truncated files
This was a bug with `mmap` and `FUSE` on 32bit platforms. Should be fixed in all LTS releases.
This was a bug with `mmap` and `FUSE` on 32bit platforms. Should be
fixed in all LTS releases.
* [https://marc.info/?l=linux-fsdevel&m=155550785230874&w=2](https://marc.info/?l=linux-fsdevel&m=155550785230874&w=2)
### Crashing on OpenVZ
There was a bug in the OpenVZ kernel with regard to how it handles `ioctl` calls. It was making invalid requests which would lead to crashes due to mergerfs not expecting them.
There was a bug in the OpenVZ kernel with regard to how it handles
`ioctl` calls. It was making invalid requests which would lead to
crashes due to mergerfs not expecting them.
* [https://bugs.openvz.org/browse/OVZ-7145](https://bugs.openvz.org/browse/OVZ-7145)
* [https://www.mail-archive.com/devel@openvz.org/msg37096.html](https://www.mail-archive.com/devel@openvz.org/msg37096.html)
+1 -1
View File
@@ -43,7 +43,7 @@ before changing them to understand how functionality will change.
* disable `async_read`
* use [symlinkify](config/symlinkify.md) if your data is largely
static and read-only
* use [tiered cache](usage_patterns.md) devices
* use [tiered cache](extended_usage_patterns.md) devices
* use LVM and LVM cache to place a SSD in front of your HDDs
+84 -1
View File
@@ -141,6 +141,21 @@ For semi-static data mergerfs + [SnapRAID](http://www.snapraid.it)
provides a similar, but not real-time, solution.
## NonRAID
* [https://github.com/qvr/nonraid](https://github.com/qvr/nonraid)
[NonRAID](https://github.com/qvr/nonraid#how-it-works) is a fork of
UnRAID's storage array kernel driver. Effectively it is like RAID5 at
the block device layer without treating the collection of devices as a
single device. Each device gets a virtual block device associated with
it which can be formatted with the filesystem of the user's choice and
will be individually accessible. Similar to SnapRAID but real-time.
Given each device has its own filesystem like a traditional setup it can
be used with mergerfs for those looking for a unified view.
## ZFS
* [https://en.wikipedia.org/wiki/ZFS](https://en.wikipedia.org/wiki/ZFS)
@@ -163,7 +178,7 @@ AnyRAID](https://hexos.com/blog/introducing-zfs-anyraid-sponsored-by-eshtek)
is a feature being developed for ZFS which is intended to provide more
flexibility in ZFS pools. Allowing a mix of capacity disks to have
greater capacity than traditional RAID and would allow for partial
upgrades while keeping live redundency.
upgrades while keeping live redundancy.
This ZFS feature, as of mid-2025, is extremely early in its development
and there are no timelines or estimates for when it may be released.
@@ -226,3 +241,71 @@ mergerfs has the feature [symlinkify](config/symlinkify.md) which
provides a similar behavior but is more flexible in that it is not
read-only. That said there can still be some software that won't like
that kind of setup.
## rclone union
rclone's [union](https://rclone.org/union) backend allows you to
create a union of multiple rclone backends and was inspired by
[mergerfs](https://rclone.org/union/#behavior-policies). Given rclone
knows more about the underlying backend than mergerfs could it can be
more efficient than creating a similar union with `rclone mount` and
mergerfs.
However, it is not uncommon to see users setup rclone mounts and
combine them with local or other remote filesystems using mergerfs
given the differing feature sets and focuses of the two projects.
## distributed filesystems
* AFS
* Ceph/CephFS
* GlusterFS
* LizardFS
* MooseFS
* etc.
Distributed remote filesystems come in many forms. Some offering POSIX
filesystem compliance and some not. Some providing remote block
devices or object stores on which a POSIX or POSIX-like filesystem is
built on top of. Some which are effectively distributed union
filesystems with duplication.
These filesystems almost always require a significant amount of
compute to run well and are typically deployed on their own
hardware. Often in an "orchestrators" + "workers" configuration across
numerous nodes. This limits their usefulness for casual and homelab
users. There could also be issues with network congestion and general
performance if using a single network and that network is slower than
the storage devices.
While possible to use a distributed filesystem to combine storage
devices (and typically to provide redundancy) it will require a more
complicated setup and more compute resources than mergerfs (while also
offering a different set of capabilities.)
## 9P
[9P, the Plan 9 Filesystem
Protocol,](https://en.wikipedia.org/wiki/9P_(protocol)) is a protocol
developed for the Plan 9 operating system to help expand on the Unix
idea that everything should be a file. The protocol made its way to
other systems and is still widely used. As such 9P is not directly
comparable to mergerfs but more so to FUSE which mergerfs uses. FUSE
is also a filesystem protocol (though designed for kernel <->
userspace communication rather than over a network). FUSE, even more
than the
[9P2000.L](https://github.com/chaos/diod/blob/master/protocol.md)
variant of 9P, is focused primarily on supporting Linux filesystem
features.
mergerfs leverages FUSE but could have in theory leveraged 9P with a
reduction in features.
While 9P has [extensive
usage](https://docs.kernel.org/filesystems/9p.html) in certain
situations its use in modern userland Linux systems is limited. FUSE
has largely replaced use cases that may have been implemented with 9P
servers in the past.
+1
View File
@@ -95,6 +95,7 @@ cache.files=off
category.create=pfrd
func.getattr=newest
dropcacheonclose=false
# comments and empty lines are supported
```
#### /etc/mergerfs/branches/media/
+31
View File
@@ -0,0 +1,31 @@
# Resource Usage and Management
## Usage
* threads
* configurable number of [threads](config/threads.md)
* reading from kernel
* processing messages from kernel
* readdir concurrency
* memory
* 1MB+ pre reader thread + inflight processing for messages
depending on [fuse_msg_size](config/fuse_msg_size.md)
* buffers allocated temporarily for reading directories
* [gidcache](faq/technical_behavior_and_limitations.md#how-does-mergerfs-handle-credentials)
* FUSE nodes
* noforget forgotten nodes
## Management
* To limit the risk of the Linux kernel's OOM Killer targeting
mergerfs it sets its
[oom_score_adj](https://man7.org/linux/man-pages/man5/proc_pid_oom_score_adj.5.html)
value to -990.
* mergerfs increases [its available file descriptor and file size
limit.](https://www.man7.org/linux/man-pages/man3/setrlimit.3p.html)
* mergerfs lowers its [scheduling
priority](https://man7.org/linux/man-pages/man3/setpriority.3p.html)
to -10 ([by default](config/options.md))
* The [readahead](config/readahead.md) values of mergerfs itself and
managed filesystems can be modified.
+2 -1
View File
@@ -22,7 +22,8 @@ directly.](mailto:support@spawn.link)**
* [Information about the broader problem along with any attempted
solutions.](https://xyproblem.info)
* Solution already ruled out and why.
* The details from the output of the `mergerfs.collect-info` tool.
* The details from the output of the
[mergerfs.collect-info](tooling.md#mergerfscollect-info) tool.
* Alternatively:
* Version of mergerfs: `mergerfs --version`
* mergerfs settings / arguments: from fstab, systemd unit, command
+78 -23
View File
@@ -1,17 +1,64 @@
# Tooling
## mergerfs.collect-info
A tool included in recent releases of `mergerfs` which collects
details about your system and configuration to help providing
[support](support.md).
```text
$ mergerfs.collect-info
* Please have mergerfs mounted before running this tool.
* Upload the following file to your GitHub ticket or put on https://pastebin.com when requesting support.
* /tmp/mergerfs.info.txt
```
## fsck.mergerfs
A tool to help diagnose and solve mergerfs pool issues. Primarily
related to mismatched permissions and ownership.
```text
$ fsck.mergerfs --help
fsck.mergerfs: A tool to help diagnose and solve mergerfs pool issues
USAGE: fsck.mergerfs [OPTIONS] path
POSITIONALS:
path TEXT:DIR REQUIRED mergerfs path
OPTIONS:
-h, --help Print this help message and exit
--fix TEXT:{none,manual,newest,largest} [none]
Will attempt to 'fix' the problem by chown+chmod or copying files
based on a selected file.
* none: Do nothing. Just print details.
* manual: User selects source file.
* newest: Use file with most recent mtime.
* largest: Use file with largest size.
--check-size BOOLEAN [false]
Considers file size in calculating differences
--copy-file BOOLEAN [false]
Copy file rather than chown/chmod to fix
```
## preload.so
EXPERIMENTAL
For some time there has been work to enable passthrough IO in
FUSE. Passthrough IO would allow for near native performance with
regards to reads and writes (at the expense of certain mergerfs
features.) In Linux v6.9 that feature made its way into the kernel
however in a somewhat limited form which is incompatible with aspects
of how mergerfs currently functions. While work will continue to
support passthrough IO in mergerfs this library was created to offer
similar functionality in a more limited way.
For some time there has been work to enable
[passthrough](config/passthrough.md) IO in FUSE. Passthrough IO would
allow for near native performance with regards to reads and writes (at
the expense of certain mergerfs features.) In Linux v6.9 that feature
made its way into the kernel however in a somewhat limited form which
is incompatible with aspects of how mergerfs functions and took longer
to implement as a result. This preloadable library was created as an
alternative to the FUSE [passthrough](config/passthrough.md)
integration.
`/usr/lib/mergerfs/preload.so`
@@ -24,11 +71,13 @@ on, reopens the file on the underlying filesystem and returns that
instead. Meaning that you will get native read/write performance
because mergerfs is no longer part of the workflow. Keep in mind that
this also means certain mergerfs features that work by interrupting
the read/write workflow, such as `moveonenospc`, will no longer work.
the read/write workflow, such as
[moveonenospc](config/moveonenospc.md), will no longer work.
Also, understand that this will only work on dynamically linked
software. Anything statically compiled will not work. Many GoLang and
Rust apps are statically compiled.
software (a that dynamically linked with the same general libc version
as the software being used with it.) Anything statically compiled will
not work. Many GoLang and Rust apps are statically compiled.
The library will not interfere with non-mergerfs filesystems. The
library is written to always fallback to returning the mergerfs opened
@@ -41,6 +90,7 @@ Thank you to
[nohajc](https://github.com/nohajc/mergerfs-io-passthrough) for
prototyping the idea.
### casual usage
```sh
@@ -49,7 +99,8 @@ LD_PRELOAD=/usr/lib/mergerfs/preload.so touch /mnt/mergerfs/filename
Or run `export LD_PRELOAD=/usr/lib/mergerfs/preload.so` in your shell
or place it in your shell config file to have it be picked up by all
software ran from your shell.
software ran from your shell. For instance, in `bash`, add the export
to `.bashrc`.
### Docker and Podman usage
@@ -104,14 +155,18 @@ Environment=LD_PRELOAD=/usr/lib/mergerfs/preload.so
## Misc
- https://github.com/trapexit/mergerfs-tools
- mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime
- mergerfs.fsck: Provides permissions and ownership auditing and the ability to fix them
- mergerfs.dedup: Will help identify and optionally remove duplicate files
- mergerfs.dup: Ensure there are at least N copies of a file across the pool
- mergerfs.balance: Rebalance files across filesystems by moving them from the most filled to the least filled
- mergerfs.consolidate: move files within a single mergerfs directory to the filesystem with most free space
- https://github.com/trapexit/scorch
- scorch: A tool to help discover silent corruption of files and keep track of files
- https://github.com/trapexit/bbf
- bbf (bad block finder): a tool to scan for and 'fix' hard drive bad blocks and find the files using those blocks
* https://github.com/trapexit/mergerfs-tools
* **Keep in mind these tools are more to provide examples of custom
behaviors that can be build on top of mergerfs. They may not
have all the features you are looking for.**
* mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime
* mergerfs.fsck: Provides permissions and ownership auditing and
the ability to fix them (should use `fsck.mergerfs` instead)
* mergerfs.dedup: Will help identify and optionally remove duplicate files
* mergerfs.dup: Ensure there are at least N copies of a file across the pool
* mergerfs.balance: Rebalance files across filesystems by moving them from the most filled to the least filled
* mergerfs.consolidate: move files within a single mergerfs directory to the filesystem with most free space
* https://github.com/trapexit/scorch
* scorch: A tool to help discover silent corruption of files and keep track of files
* https://github.com/trapexit/bbf
* bbf (bad block finder): a tool to scan for and 'fix' hard drive bad blocks and find the files using those blocks
+2 -1
View File
@@ -95,6 +95,7 @@ nav:
- config/export-support.md
- config/kernel-permissions-check.md
- error_handling_and_logging.md
- resource_usage.md
- runtime_interface.md
- remote_filesystems.md
- tips_notes.md
@@ -103,7 +104,7 @@ nav:
- benchmarking.md
- performance.md
- tooling.md
- usage_patterns.md
- extended_usage_patterns.md
- FAQ:
- faq/why_isnt_it_working.md
- faq/reliability_and_scalability.md
+1 -1
View File
@@ -38,7 +38,7 @@ Branch::Branch(const u64 &default_minfreespace_)
}
int
Branch::from_string(const std::string &str_)
Branch::from_string(const std::string_view str_)
{
return -EINVAL;
}
+1 -1
View File
@@ -57,7 +57,7 @@ public:
bool ro_or_nc(void) const;
public:
int from_string(const std::string &str) final;
int from_string(const std::string_view str) final;
std::string to_string(void) const final;
public:
+14 -14
View File
@@ -74,9 +74,9 @@ namespace l
{
static
void
split(const std::string &s_,
std::string *instr_,
std::string *values_)
split(const std::string_view s_,
std::string *instr_,
std::string *values_)
{
u64 offset;
@@ -222,8 +222,8 @@ namespace l
static
int
set(const std::string &str_,
Branches::Impl *branches_)
set(const std::string_view str_,
Branches::Impl *branches_)
{
int rv;
StrVec paths;
@@ -244,8 +244,8 @@ namespace l
static
int
add_begin(const std::string &str_,
Branches::Impl *branches_)
add_begin(const std::string_view str_,
Branches::Impl *branches_)
{
int rv;
std::vector<std::string> paths;
@@ -268,8 +268,8 @@ namespace l
static
int
add_end(const std::string &str_,
Branches::Impl *branches_)
add_end(const std::string_view str_,
Branches::Impl *branches_)
{
int rv;
StrVec paths;
@@ -310,8 +310,8 @@ namespace l
static
int
erase_fnmatch(const std::string &str_,
Branches::Impl *branches_)
erase_fnmatch(const std::string_view str_,
Branches::Impl *branches_)
{
StrVec patterns;
@@ -336,7 +336,7 @@ namespace l
}
int
Branches::Impl::from_string(const std::string &s_)
Branches::Impl::from_string(const std::string_view s_)
{
std::string instr;
std::string values;
@@ -403,7 +403,7 @@ Branches::Impl::to_paths() const
}
int
Branches::from_string(const std::string &str_)
Branches::from_string(const std::string_view str_)
{
int rv;
Branches::Ptr impl;
@@ -462,7 +462,7 @@ SrcMounts::SrcMounts(Branches &b_)
}
int
SrcMounts::from_string(const std::string &s_)
SrcMounts::from_string(const std::string_view s_)
{
// return _branches.from_string(s_);
return 0;
+10 -3
View File
@@ -46,7 +46,7 @@ public:
Impl(const u64 &default_minfreespace);
public:
int from_string(const std::string &str) final;
int from_string(const std::string_view str) final;
std::string to_string(void) const final;
public:
@@ -73,13 +73,20 @@ public:
{}
public:
int from_string(const std::string &str) final;
int from_string(const std::string_view str) final;
std::string to_string(void) const final;
public:
operator Ptr() const { std::lock_guard<std::mutex> lg(_mutex); return _impl; }
Ptr operator->() const { std::lock_guard<std::mutex> lg(_mutex); return _impl; }
public:
Impl::iterator begin() { return _impl->begin(); }
Impl::iterator end() { return _impl->end(); }
Impl::const_iterator begin() const { return _impl->begin(); }
Impl::const_iterator end() const { return _impl->end(); }
public:
void find_and_set_mode_ro();
};
@@ -90,7 +97,7 @@ public:
SrcMounts(Branches &b);
public:
int from_string(const std::string &str) final;
int from_string(const std::string_view str) final;
std::string to_string(void) const final;
private:
+1 -1
View File
@@ -23,7 +23,7 @@
#include <string>
int
Category::Base::from_string(const std::string &s_)
Category::Base::from_string(const std::string_view s_)
{
int rv;
+1 -1
View File
@@ -27,7 +27,7 @@ namespace Category
class Base : public ToFromString
{
public:
int from_string(const std::string &s) final;
int from_string(const std::string_view s) final;
std::string to_string() const final;
protected:
+1 -1
View File
@@ -381,7 +381,7 @@ Config::from_stream(std::istream &istrm_,
while(std::getline(istrm_,line,'\n'))
{
line = str::trim(line);
if(!line.empty() && (line[0] == '#'))
if(line.empty() || (line[0] == '#'))
continue;
str::splitkv(line,'=',&key,&val);
+2 -1
View File
@@ -56,6 +56,7 @@
typedef ToFromWrapper<bool> ConfigBOOL;
typedef ToFromWrapper<uint64_t> ConfigUINT64;
typedef ToFromWrapper<int64_t> ConfigS64;
typedef ToFromWrapper<int> ConfigINT;
typedef ToFromWrapper<std::string> ConfigSTR;
typedef ToFromWrapper<std::filesystem::path> ConfigPath;
@@ -161,7 +162,7 @@ public:
StatFS statfs;
StatFSIgnore statfs_ignore;
ConfigBOOL symlinkify;
ConfigUINT64 symlinkify_timeout;
ConfigS64 symlinkify_timeout;
ConfigINT fuse_read_thread_count;
ConfigINT fuse_process_thread_count;
ConfigINT fuse_process_thread_queue_depth;
+1 -1
View File
@@ -45,7 +45,7 @@ CacheFiles::to_string() const
template<>
int
CacheFiles::from_string(const std::string &s_)
CacheFiles::from_string(const std::string_view s_)
{
if(s_ == "libfuse")
_data = CacheFiles::ENUM::LIBFUSE;
+1 -1
View File
@@ -39,7 +39,7 @@ FlushOnClose::to_string() const
template<>
int
FlushOnClose::from_string(const std::string &s_)
FlushOnClose::from_string(const std::string_view s_)
{
if(s_ == "never")
_data = FlushOnClose::ENUM::NEVER;
+1 -1
View File
@@ -41,7 +41,7 @@ FollowSymlinks::to_string(void) const
template<>
int
FollowSymlinks::from_string(const std::string &s_)
FollowSymlinks::from_string(const std::string_view s_)
{
if(s_ == "never")
_data = FollowSymlinks::ENUM::NEVER;
+2 -2
View File
@@ -23,7 +23,7 @@ GIDCacheExpireTimeout::to_string(void) const
}
int
GIDCacheExpireTimeout::from_string(const std::string &s_)
GIDCacheExpireTimeout::from_string(const std::string_view s_)
{
int rv;
@@ -53,7 +53,7 @@ GIDCacheRemoveTimeout::to_string(void) const
}
int
GIDCacheRemoveTimeout::from_string(const std::string &s_)
GIDCacheRemoveTimeout::from_string(const std::string_view s_)
{
int rv;
+2 -2
View File
@@ -10,7 +10,7 @@ public:
public:
std::string to_string(void) const final;
int from_string(const std::string &) final;
int from_string(const std::string_view) final;
};
class GIDCacheRemoveTimeout : public ToFromString
@@ -21,5 +21,5 @@ public:
public:
std::string to_string(void) const final;
int from_string(const std::string &) final;
int from_string(const std::string_view) final;
};
+1 -1
View File
@@ -32,7 +32,7 @@ InodeCalc::to_string(void) const
}
int
InodeCalc::from_string(const std::string &s_)
InodeCalc::from_string(const std::string_view s_)
{
return fs::inode::set_algo(s_);
}
+1 -1
View File
@@ -28,5 +28,5 @@ public:
public:
std::string to_string(void) const final;
int from_string(const std::string &) final;
int from_string(const std::string_view) final;
};
+1 -1
View File
@@ -41,7 +41,7 @@ LinkEXDEV::to_string(void) const
template<>
int
LinkEXDEV::from_string(const std::string &s_)
LinkEXDEV::from_string(const std::string_view s_)
{
if(s_ == "passthrough")
_data = LinkEXDEV::ENUM::PASSTHROUGH;
+1 -1
View File
@@ -38,7 +38,7 @@ LogMetrics::to_string(void) const
}
int
LogMetrics::from_string(const std::string &s_)
LogMetrics::from_string(const std::string_view s_)
{
int rv;
bool val;
+1 -1
View File
@@ -27,5 +27,5 @@ public:
public:
std::string to_string(void) const final;
int from_string(const std::string &) final;
int from_string(const std::string_view) final;
};
+1 -1
View File
@@ -23,7 +23,7 @@
int
MoveOnENOSPC::from_string(const std::string &s_)
MoveOnENOSPC::from_string(const std::string_view s_)
{
int rv;
std::string s;
+1 -1
View File
@@ -35,7 +35,7 @@ public:
}
public:
int from_string(const std::string &s) final;
int from_string(const std::string_view s) final;
std::string to_string() const final;
public:
+1 -1
View File
@@ -23,7 +23,7 @@
template<>
int
NFSOpenHack::from_string(const std::string &s_)
NFSOpenHack::from_string(const std::string_view s_)
{
if(s_ == "off")
_data = NFSOpenHack::ENUM::OFF;
+1 -1
View File
@@ -44,7 +44,7 @@ ConfigPageSize::to_string(void) const
}
int
ConfigPageSize::from_string(const std::string &s_)
ConfigPageSize::from_string(const std::string_view s_)
{
uint64_t v;
uint64_t pagesize;
+1 -1
View File
@@ -33,7 +33,7 @@ public:
public:
std::string to_string(void) const final;
int from_string(const std::string &) final;
int from_string(const std::string_view) final;
public:
ConfigPageSize&
+1 -1
View File
@@ -41,7 +41,7 @@ Passthrough::to_string() const
template<>
int
Passthrough::from_string(const std::string &s_)
Passthrough::from_string(const std::string_view s_)
{
if(s_ == "off")
_data = Passthrough::ENUM::OFF;
+1 -1
View File
@@ -35,7 +35,7 @@ public:
}
int
from_string(const std::string &) final
from_string(const std::string_view) final
{
return -EROFS;
}
+1 -1
View File
@@ -39,7 +39,7 @@ RenameEXDEV::to_string(void) const
template<>
int
RenameEXDEV::from_string(const std::string &s_)
RenameEXDEV::from_string(const std::string_view s_)
{
if(s_ == "passthrough")
_data = RenameEXDEV::ENUM::PASSTHROUGH;
+1 -1
View File
@@ -33,7 +33,7 @@ ConfigSet::to_string(void) const
}
int
ConfigSet::from_string(const std::string &str_)
ConfigSet::from_string(const std::string_view str_)
{
this->clear();
+1 -1
View File
@@ -31,5 +31,5 @@ public:
public:
std::string to_string(void) const final;
int from_string(const std::string &) final;
int from_string(const std::string_view) final;
};
+1 -1
View File
@@ -38,7 +38,7 @@ StatFS::to_string() const
template<>
int
StatFS::from_string(const std::string &s_)
StatFS::from_string(const std::string_view s_)
{
if(s_ == "base")
_data = StatFS::ENUM::BASE;

Some files were not shown because too many files have changed in this diff Show More