From 82b3ba76fa4ea2fba82c7343e831af3139e9cb9b Mon Sep 17 00:00:00 2001 From: Timofey Titovets Date: Mon, 30 Jan 2017 05:29:28 +0300 Subject: [PATCH 1/4] Makefile: make service install compatible with debian systems Signed-off-by: Timofey Titovets --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ff399bd..4cc6576 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ install: lib src test install_scripts: ## Install scipts install -Dm755 scripts/beesd $(PREFIX)/usr/bin/beesd install -Dm644 scripts/beesd.conf.sample $(PREFIX)/etc/bees/beesd.conf.sample - install -Dm644 scripts/beesd@.service $(PREFIX)/usr/lib/systemd/system/beesd@.service + install -Dm644 scripts/beesd@.service $(PREFIX)/lib/systemd/system/beesd@.service help: ## Show help @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##/\t/' From dc00dce842ad5e6b4681b6502e2611e421713b87 Mon Sep 17 00:00:00 2001 From: Zygo Blaxell Date: Wed, 8 Feb 2017 22:01:00 -0500 Subject: [PATCH 2/4] context: purge FD cache every COMMIT_INTERVAL Holding file FDs open for long periods of time delays inode destruction. For very large files this can lead to excessive delays while bees dedups data that will cease to be reachable. Use the same workaround for file FDs (in the root_ino cache) that is used for subvols (in the root cache): forcibly close all cached FDs at regular intervals. The FD cache will reacquire FDs from files that still have existing paths, and will abandon FDs from files that no longer have existing paths. The non-existing-path case is not new (bees has always been able to discover deleted inodes) so it is already handled by existing code. Fixes: https://github.com/Zygo/bees/issues/18 Signed-off-by: Zygo Blaxell --- src/bees-context.cc | 6 ++++++ src/bees.h | 1 + 2 files changed, 7 insertions(+) diff --git a/src/bees-context.cc b/src/bees-context.cc index 1c82bd7..ada5a1c 100644 --- a/src/bees-context.cc +++ b/src/bees-context.cc @@ -55,6 +55,12 @@ BeesFdCache::open_root(shared_ptr ctx, uint64_t root) Fd BeesFdCache::open_root_ino(shared_ptr ctx, uint64_t root, uint64_t ino) { + if (m_file_cache_timer.age() > BEES_COMMIT_INTERVAL) { + BEESINFO("Clearing open FD cache to enable file delete"); + m_file_cache.clear(); + m_file_cache_timer.reset(); + BEESCOUNT(open_clear); + } return m_file_cache(ctx, root, ino); } diff --git a/src/bees.h b/src/bees.h index 8a17960..8c74ca2 100644 --- a/src/bees.h +++ b/src/bees.h @@ -659,6 +659,7 @@ class BeesFdCache { LRUCache, uint64_t> m_root_cache; LRUCache, uint64_t, uint64_t> m_file_cache; Timer m_root_cache_timer; + Timer m_file_cache_timer; public: BeesFdCache(); From 4b592ec2a30d5cf79ee6ce6a47b4228cb4be2f18 Mon Sep 17 00:00:00 2001 From: Timofey Titovets Date: Thu, 9 Feb 2017 11:56:31 +0300 Subject: [PATCH 3/4] Check: if disk with UUID are btrfs by blkid Old check can't find btrfs fs, if fs not mounted Signed-off-by: Timofey Titovets --- scripts/beesd | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/beesd b/scripts/beesd index 6abffc4..cfe256c 100755 --- a/scripts/beesd +++ b/scripts/beesd @@ -52,16 +52,18 @@ DB_SIZE="${DB_SIZE:-$((64*AL16M))}" LOG_SHORT_PATH="${LOG_SHORT_PATH:-N}" LOG_FILTER_TIME="${LOG_FILTER_TIME:-N}" -INFO "Check: BTRFS UUID exists" -if [ ! -d "/sys/fs/btrfs/$UUID" ]; then - ERRO "Can't find BTRFS UUID: $UUID" -fi - INFO "Check: Disk exists" if [ ! -b "/dev/disk/by-uuid/$UUID" ]; then ERRO "Missing disk: /dev/disk/by-uuid/$UUID" fi +it_btrfs(){ [ "$(blkid -s TYPE -o value "$1")" == "btrfs" ]; } + +INFO "Check: Disk with btrfs" +if ! it_btrfs "/dev/disk/by-uuid/$UUID"; then + ERRO "Disk not contain btrfs: /dev/disk/by-uuid/$UUID" +fi + INFO "WORK DIR: $WORK_DIR" mkdir -p "$WORK_DIR" || exit 1 From 5350b0f11347e593fee1d14d1d06ddb7aa11f133 Mon Sep 17 00:00:00 2001 From: Timofey Titovets Date: Tue, 13 Jun 2017 18:05:38 +0300 Subject: [PATCH 4/4] Bees: fix [-Werror=implicit-fallthrough=] In gcc 7+ warning: implicit-fallthrough has been added In some places fallthrough is expectable, disable warning Signed-off-by: Timofey Titovets --- lib/extentwalker.cc | 17 +++++++++-------- src/bees-roots.cc | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/extentwalker.cc b/lib/extentwalker.cc index 626255a..3f6001b 100644 --- a/lib/extentwalker.cc +++ b/lib/extentwalker.cc @@ -520,25 +520,26 @@ namespace crucible { auto type = call_btrfs_get(btrfs_stack_file_extent_type, i.m_data); off_t len = -1; - switch (type) { - default: + switch (type) { + default: cerr << "Unhandled file extent type " << type << " in root " << m_tree_id << " ino " << m_stat.st_ino << endl; break; - case BTRFS_FILE_EXTENT_INLINE: + case BTRFS_FILE_EXTENT_INLINE: len = ranged_cast(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data)); e.m_flags |= FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; // Inline extents are never obscured, so don't bother filling in m_physical_len, etc. - break; - case BTRFS_FILE_EXTENT_PREALLOC: + break; + case BTRFS_FILE_EXTENT_PREALLOC: e.m_flags |= Extent::PREALLOC; - case BTRFS_FILE_EXTENT_REG: { + // fallthrough + case BTRFS_FILE_EXTENT_REG: { e.m_physical = call_btrfs_get(btrfs_stack_file_extent_disk_bytenr, i.m_data); // This is the length of the full extent (decompressed) - off_t ram = ranged_cast(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data)); + off_t ram = ranged_cast(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data)); // This is the length of the part of the extent appearing in the file (decompressed) - len = ranged_cast(call_btrfs_get(btrfs_stack_file_extent_num_bytes, i.m_data)); + len = ranged_cast(call_btrfs_get(btrfs_stack_file_extent_num_bytes, i.m_data)); // This is the offset from start of on-disk extent to the part we see in the file (decompressed) // May be negative due to the kind of bug we're stuck with forever, so no cast range check diff --git a/src/bees-roots.cc b/src/bees-roots.cc index 4fba68d..921132c 100644 --- a/src/bees-roots.cc +++ b/src/bees-roots.cc @@ -768,6 +768,7 @@ BeesCrawl::fetch_extents() break; case BTRFS_FILE_EXTENT_PREALLOC: BEESCOUNT(crawl_prealloc); + // fallthrough case BTRFS_FILE_EXTENT_REG: { auto physical = call_btrfs_get(btrfs_stack_file_extent_disk_bytenr, i.m_data); auto ram = call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data);