create: back up automounted directories, fixes #6652 (1.4-maint)

src/borg/archive.py

@@ -34,7 +34,7 @@
 from .helpers import hardlinkable
 from .helpers import ChunkIteratorFileWrapper, normalize_chunker_params, open_item
 from .helpers import Error, IntegrityError, set_ec
-from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns
+from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns, on_different_mounts
 from .helpers import parse_timestamp, to_localtime
 from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize
 from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
@@ -255,6 +255,27 @@ def stat_update_check(st_old, st_curr):
     return st_curr
 
 
+def is_automount(st_old, st_curr, parent_fd, fd):
+    """
+    Check whether opening the directory triggered an automount, see #6652.
+
+    stat() does not trigger an automount (e.g. autofs, systemd automount units or the ZFS
+    snapshot directories in .zfs/snapshot), but opening the directory does. st_old is then the
+    stat of the automount point and st_curr (the fstat() of fd) is the stat of the root directory
+    of the filesystem that got mounted there, so they look like a different inode.
+
+    We only accept this if both are directories and if fd (the opened directory) is on a
+    different mount than parent_fd (its parent directory).
+    """
+    if parent_fd is None or fd is None:
+        return False
+    if not (stat.S_ISDIR(st_old.st_mode) and stat.S_ISDIR(st_curr.st_mode)):
+        return False
+    if st_old.st_ino == st_curr.st_ino and st_old.st_dev == st_curr.st_dev:
+        return False
+    return on_different_mounts(parent_fd, fd) is True
+
+
 @contextmanager
 def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op='open'):
     with backup_io(op):

src/borg/archiver.py

@@ -46,6 +46,7 @@
     from .algorithms.checksums import crc32
     from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special
     from .archive import BackupError, BackupRaceConditionError, BackupOSError, backup_io, OsOpen, stat_update_check
+    from .archive import is_automount
     from .archive import FilesystemObjectProcessors, TarfileObjectProcessors, MetadataCollector, ChunksProcessor
     from .archive import has_link
     from .cache import Cache, assert_secure, SecurityManager
@@ -240,6 +241,24 @@ def get_func(args):
     raise Exception('expected func attributes not found')
 
 
+def stat_dir_mounted(path, st):
+    """
+    If st (the stat of path) is a directory, open it and return its fstat, otherwise return st.
+
+    stat() does not trigger an automount, but opening the directory does, so we get the stat
+    of the root directory of the filesystem mounted there (not of the automount point), see #6652.
+    """
+    if not stat.S_ISDIR(st.st_mode):
+        return st
+    try:
+        with OsOpen(path=path, flags=flags_dir, noatime=True, op='dir_open') as fd:
+            # fd is None for directories on windows.
+            return st if fd is None else os.fstat(fd)
+    except (BackupOSError, OSError):
+        # the caller opens the directory again later and handles the error there.
+        return st
+
+
 class Highlander(argparse.Action):
     """make sure some option is only given once"""
     def __call__(self, parser, namespace, values, option_string=None):
@@ -674,6 +693,7 @@ def create_inner(archive, cache, fso):
                     try:
                         with backup_io('stat'):
                             st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
+                        st = stat_dir_mounted(path, st)
                         restrict_dev = st.st_dev if args.one_file_system else None
                         self._rec_walk(path=path, parent_fd=None, name=None,
                                        fso=fso, cache=cache, matcher=matcher,
@@ -858,7 +878,17 @@ def _rec_walk(self, *, path, parent_fd, name, fso, cache, matcher,
                     # child_fd is None for directories on windows, in that case a race condition check is not possible.
                     if child_fd is not None:
                         with backup_io('fstat'):
-                            st = stat_update_check(st, os.fstat(child_fd))
+                            st_curr = os.fstat(child_fd)
+                            automounted = is_automount(st, st_curr, parent_fd, child_fd)
+                        if automounted:
+                            # opening the directory mounted a filesystem there, see #6652.
+                            # redo the decisions we made based on the stat of the automount point.
+                            st = st_curr
+                            if (st.st_ino, st.st_dev) in skip_inodes:
+                                return
+                            recurse = restrict_dev is None or st.st_dev == restrict_dev
+                        else:
+                            st = stat_update_check(st, st_curr)
                     if recurse:
                         tag_names = dir_is_tagged(path, exclude_caches, exclude_if_present)
                         if tag_names:
@@ -3821,6 +3851,9 @@ def define_borg_mount(parser):
         macOS examples are the APFS mounts of a typical macOS installation.
         Therefore, when using ``--one-file-system``, you should double-check that the backup works as intended.
 
+        Automount points (e.g. autofs, systemd automount units, ZFS ``.zfs/snapshot/*``) get mounted when
+        borg opens them. For these, borg uses the device number of the filesystem that got mounted there.
+
 
         Error handling
         ++++++++++++++

src/borg/platform/__init__.py

@@ -9,6 +9,7 @@
 from .base import listxattr, getxattr, setxattr, ENOATTR
 from .base import acl_get, acl_set
 from .base import set_flags, get_flags
+from .base import on_different_mounts
 from .base import SaveFile, SyncFile, sync_dir, fdatasync, safe_fadvise
 from .base import swidth, API_VERSION
 from .base import process_alive, get_process_id, local_pid_alive, get_hostname, get_fqdn, get_hostid
@@ -31,6 +32,7 @@
     from .linux import listxattr, getxattr, setxattr
     from .linux import acl_get, acl_set
     from .linux import set_flags, get_flags
+    from .linux import on_different_mounts
     from .linux import SyncFile
 elif is_freebsd:  # pragma: freebsd only
     from .freebsd import API_VERSION as OS_API_VERSION

src/borg/platform/base.py

@@ -95,6 +95,18 @@ def get_flags(path, st, fd=None):
     return getattr(st, 'st_flags', 0)
 
 
+def on_different_mounts(fd1, fd2):
+    """
+    Return whether the open file descriptors *fd1* and *fd2* are on different mounts.
+
+    Returns True or False, or None if this can not be determined.
+
+    This generic implementation compares st_dev. That is only correct on platforms where every
+    st_dev is a separately mounted filesystem (e.g. no btrfs-like subvolumes with their own st_dev).
+    """
+    return os.fstat(fd1).st_dev != os.fstat(fd2).st_dev
+
+
 def sync_dir(path):
     if is_win32:
         # Opening directories is not supported on Windows.

src/borg/platform/linux.pyx

@@ -196,6 +196,33 @@ def get_flags(path, st, fd=None):
     return bsd_flags
 
 
+def _get_mnt_id(fd):
+    """Return the id of the mount the open file descriptor fd is on (Linux >= 3.15), or None."""
+    try:
+        with open(f"/proc/self/fdinfo/{fd}", "rb") as f:
+            for line in f:
+                if line.startswith(b"mnt_id:"):
+                    return int(line.split()[1])
+    except (OSError, ValueError, IndexError):
+        pass
+    return None
+
+
+def on_different_mounts(fd1, fd2):
+    """
+    Return whether the open file descriptors *fd1* and *fd2* are on different mounts.
+
+    Returns True or False, or None if this can not be determined.
+
+    Comparing st_dev is not good enough on Linux, e.g. btrfs subvolumes have their own st_dev,
+    so we compare the mount ids. While both fds are open, both mounts exist, so their ids are unique.
+    """
+    mnt_id1, mnt_id2 = _get_mnt_id(fd1), _get_mnt_id(fd2)
+    if mnt_id1 is None or mnt_id2 is None:
+        return None
+    return mnt_id1 != mnt_id2
+
+
 def acl_use_local_uid_gid(acl):
     """Replace the user/group field with the local uid/gid if possible."""
     entries = []

src/borg/testsuite/archive.py

@@ -1,4 +1,5 @@
 import json
+import os
 from collections import OrderedDict
 from datetime import datetime, timezone
 from io import StringIO
@@ -9,7 +10,7 @@
 from . import BaseTestCase
 from ..crypto.key import PlaintextKey
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
-from ..archive import BackupOSError, backup_io, backup_io_iter, get_item_uid_gid
+from ..archive import BackupOSError, backup_io, backup_io_iter, get_item_uid_gid, is_automount
 from ..helpers import Manifest
 from ..helpers import msgpack
 from ..item import Item, ArchiveItem
@@ -305,6 +306,27 @@ def __next__(self):
         assert False, 'StopIteration handled incorrectly'
 
 
+@pytest.mark.parametrize("mounted", [False, True, None])
+def test_is_automount(tmp_path, monkeypatch, mounted):
+    import borg.archive as archive_module
+
+    monkeypatch.setattr(archive_module, "on_different_mounts", lambda fd1, fd2: mounted)
+    (tmp_path / "dir1").mkdir()
+    (tmp_path / "dir2").mkdir()
+    (tmp_path / "file").touch()
+    st_dir1, st_dir2, st_file = (os.stat(tmp_path / name) for name in ("dir1", "dir2", "file"))
+    parent_fd, fd = 3, 4  # only passed through to on_different_mounts
+    # a different directory is only accepted if it is on a different mount than its parent:
+    assert is_automount(st_dir1, st_dir2, parent_fd, fd) is (mounted is True)
+    # the same directory is no automount, but the normal case:
+    assert is_automount(st_dir1, st_dir1, parent_fd, fd) is False
+    # a file type change is never accepted:
+    assert is_automount(st_file, st_dir2, parent_fd, fd) is False
+    assert is_automount(st_dir1, st_file, parent_fd, fd) is False
+    # without a parent fd (recursion roots), we can not check:
+    assert is_automount(st_dir1, st_dir2, None, fd) is False
+
+
 def test_get_item_uid_gid():
     # test requires that:
     # - a name for user 0 and group 0 exists, usually root:root or root:wheel.

src/borg/testsuite/archiver.py

@@ -2382,6 +2382,55 @@ def fake_get_flags(path, st, fd=None):
             output = self.cmd('create', '--list', self.repository_location + '::test2', 'input')
         self.assert_in('A input/cloudfile', output)
 
+    @unittest.skipIf(is_win32, 'no fd-based directory checks on windows')
+    def test_create_automount(self):
+        """Opening an automount point mounts a filesystem there, so stat and fstat differ, see #6652."""
+        import borg.archiver as archiver_module
+
+        self.create_regular_file('automnt/file', contents=b'hello')
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+
+        class AutomountPointStat:
+            # stat() of an automount point: a different inode (here even on a different st_dev, like
+            # for an autofs direct mount) than the root directory of the filesystem mounted there.
+            def __init__(self, st):
+                self._st = st
+                self.st_ino = st.st_ino + 1000000
+                self.st_dev = st.st_dev + 1
+
+            def __getattr__(self, name):
+                return getattr(self._st, name)
+
+        real_os_stat = archiver_module.os_stat
+
+        def fake_os_stat(*, path=None, parent_fd=None, name=None, follow_symlinks=False):
+            st = real_os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=follow_symlinks)
+            return AutomountPointStat(st) if name == 'automnt' else st
+
+        for mounted in False, True:
+            for one_file_system in False, True:
+                archive = f'test-{mounted}-{one_file_system}'
+                args = ['create', '--list', self.repository_location + '::' + archive, 'input']
+                args += ['--one-file-system'] if one_file_system else []
+                with patch.object(archiver_module, 'os_stat', fake_os_stat), \
+                        patch('borg.archive.on_different_mounts', lambda fd1, fd2: mounted):
+                    if mounted:
+                        # we archive the mounted filesystem and recurse into it (because it is on the same st_dev
+                        # as input, even with --one-file-system - the automount point would have been on another
+                        # st_dev).
+                        output = self.cmd(*args)
+                        assert 'race condition' not in output
+                    else:
+                        # no mount there, so this is a real race condition and we skip the directory.
+                        output = self.cmd(*args, exit_code=EXIT_WARNING)
+                        assert 'input/automnt: file type or inode changed' in output
+                        assert 'E input/automnt\n' in output
+                listing = self.cmd('list', '--short', self.repository_location + '::' + archive)
+                if mounted:
+                    assert 'input/automnt/file' in listing
+                else:
+                    assert 'input/automnt' not in listing
+
     def test_create_json(self):
         self.create_regular_file('file1', size=1024 * 80)
         self.cmd('init', '--encryption=repokey', self.repository_location)
@@ -4600,6 +4649,10 @@ def test_extract_xattrs_errors(self):
     def test_create_exclude_dataless(self):
         pass
 
+    @unittest.skip('patches objects')
+    def test_create_automount(self):
+        pass
+
     @unittest.skip('test_basic_functionality seems incompatible with fakeroot and/or the binary.')
     def test_basic_functionality(self):
         pass

src/borg/testsuite/platform.py

@@ -6,10 +6,12 @@
 import tempfile
 import unittest
 
+import pytest
+
 from ..platformflags import is_win32, is_linux, is_freebsd, is_netbsd, is_darwin, is_haiku
 from ..platform import acl_get, acl_set, swidth
 from ..platform import get_process_id, process_alive
-from ..platform import fdatasync, sync_dir
+from ..platform import fdatasync, sync_dir, on_different_mounts
 from . import BaseTestCase, unopened_tempfile
 from .locking import free_pid
 
@@ -314,3 +316,24 @@ def test_process_id():
     assert len(hostname) > 0
     assert pid > 0
     assert get_process_id() == (hostname, pid, tid)
+
+
+@pytest.mark.skipif(is_win32, reason="can not open directories on windows")
+def test_on_different_mounts(tmp_path):
+    (tmp_path / "subdir").mkdir()
+    # a directory that is usually a separately mounted filesystem (devtmpfs, devfs, procfs, tmpfs, ...):
+    candidates = [p for p in ("/dev", "/proc", "/tmp") if os.path.isdir(p)]
+    other_mount = next((p for p in candidates if os.stat(p).st_dev != os.stat("/").st_dev), None)
+    fds = {}
+    try:
+        for path in "/", other_mount, tmp_path, tmp_path / "subdir":
+            if path is not None:
+                fds[path] = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
+        assert on_different_mounts(fds[tmp_path], fds[tmp_path / "subdir"]) is False
+        assert on_different_mounts(fds[tmp_path], fds[tmp_path]) is False
+        if other_mount is None:
+            pytest.skip("found no separately mounted filesystem")
+        assert on_different_mounts(fds["/"], fds[other_mount]) is True
+    finally:
+        for fd in fds.values():
+            os.close(fd)