build: include localconf everywhere

Overriding makeflags did not work from localconf in the src, lib, or
test directories.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>

Makefile

@@ -9,6 +9,8 @@ SYSTEMD_SYSTEM_UNIT_DIR ?= $(shell pkg-config systemd --variable=systemdsystemun
 
 MARKDOWN := $(firstword $(shell type -P markdown markdown2 markdown_py 2>/dev/null || echo markdown))
 
+BEES_VERSION ?= $(shell git describe --always --dirty || echo UNKNOWN)
+
 # allow local configuration to override above variables
 -include localconf
 
@@ -32,11 +34,11 @@ clean: ## Cleanup
 .PHONY: lib src test
 
 lib: ## Build libs
-	$(MAKE) -C lib
+	$(MAKE) TAG="$(BEES_VERSION)" -C lib
 
 src: ## Build bins
 src: lib
-	$(MAKE) -C src
+	$(MAKE) BEES_VERSION="$(BEES_VERSION)" -C src
 
 test: ## Run tests
 test: lib src

README.md

@@ -336,6 +336,12 @@ Unfixed kernel bugs (as of 4.14.34) with workarounds in Bees:
   or prealloc.  Bees avoids feedback loops this can generate while
   attempting to replace extents over 16MB in length.
 
+* **Systems with many CPU cores** may [lock up when bees runs with one
+  worker thread for every core](https://github.com/Zygo/bees/issues/91).
+  bees limits the number of threads it will try to create based on
+  detected CPU core count.  Users may override this limit with the
+  [`--thread-count` option](options.md).
+
 Not really bugs, but gotchas nonetheless:
 
 * If a process holds a directory FD open, the subvol containing the
@@ -536,12 +542,18 @@ Command Line Options
 
 * --thread-count (-c) COUNT
   * Specify maximum number of worker threads for scanning.  Overrides
-    --thread-factor (-C) and default/autodetected values.
+    --thread-factor (-C) and default/autodetected values,
+    and the hardcoded thread limit.
 * --thread-factor (-C) FACTOR
   * Specify ratio of worker threads to CPU cores.  Overridden by --thread-count (-c).
     Default is 1.0, i.e. 1 worker thread per detected CPU.  Use values
     below 1.0 to leave some cores idle, or above 1.0 if there are more
     disks than CPUs in the filesystem.
+    If the computed thread count is higher than `BEES_DEFAULT_THREAD_LIMIT`
+    (currently 8), then only that number of threads will be created.
+    This limit can be overridden by the `--thread-count` option; however,
+    be aware that there are kernel issues with systems that have many CPU
+    cores when users try to run bees on all of them.
 * --loadavg-target (-g) LOADAVG
   * Specify load average target for dynamic worker threads.
     Threads will be started or stopped subject to the upper limit imposed

include/crucible/btrfs.h

@@ -23,6 +23,7 @@
 #undef min
 #undef max
 #undef mutex
+#undef swap
 
 #ifndef BTRFS_FIRST_FREE_OBJECTID
 

include/crucible/lockset.h

@@ -117,7 +117,7 @@ namespace crucible {
 		while (full() || locked(name)) {
 			m_condvar.wait(lock);
 		}
-		auto rv = m_set.insert(make_pair(name, gettid()));
+		auto rv = m_set.insert(make_pair(name, crucible::gettid()));
 		THROW_CHECK0(runtime_error, rv.second);
 	}
 
@@ -129,7 +129,7 @@ namespace crucible {
 		if (full() || locked(name)) {
 			return false;
 		}
-		auto rv = m_set.insert(make_pair(name, gettid()));
+		auto rv = m_set.insert(make_pair(name, crucible::gettid()));
 		THROW_CHECK1(runtime_error, name, rv.second);
 		return true;
 	}

include/crucible/progress.h

@@ -13,7 +13,7 @@ namespace crucible {
 
 	template <class T>
 	class ProgressTracker {
-		class ProgressTrackerState;
+		struct ProgressTrackerState;
 		class ProgressHolderState;
 	public:
 		using value_type = T;

lib/Makefile

@@ -1,4 +1,4 @@
-TAG := $(shell git describe --always --dirty || echo UNKNOWN)
+TAG ?= $(shell git describe --always --dirty || echo UNKNOWN)
 
 default: libcrucible.so
 %.so: Makefile
@@ -20,6 +20,7 @@ CRUCIBLE_OBJS = \
 	uuid.o \
 
 include ../makeflags
+-include ../localconf
 include ../Defines.mk
 
 configure.h: configure.h.in

lib/chatter.cc

@@ -69,14 +69,14 @@ namespace crucible {
 			DIE_IF_ZERO(strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &ltm));
 
 			header_stream << buf;
-			header_stream << " " << getpid() << "." << gettid() << "<" << m_loglevel << ">";
+			header_stream << " " << getpid() << "." << crucible::gettid() << "<" << m_loglevel << ">";
 			if (!m_name.empty()) {
 				header_stream << " " << m_name;
 			}
 		} else {
 			header_stream << "<" << m_loglevel << ">";
 			header_stream << (m_name.empty() ? "thread" : m_name);
-			header_stream << "[" << gettid() << "]";
+			header_stream << "[" << crucible::gettid() << "]";
 		}
 
 		header_stream << ": ";
@@ -124,6 +124,7 @@ namespace crucible {
 		} else if (!chatter_names->empty()) {
 			cerr << "CRUCIBLE_CHATTER does not list '" << m_file << "' or '" << m_pretty_function << "'" << endl;
 		}
+		(void)m_line; // not implemented yet
 		// cerr << "ChatterBox " << reinterpret_cast<void*>(this) << " constructed" << endl;
 	}
 

lib/error.cc

@@ -32,7 +32,7 @@ namespace crucible {
 
 	// FIXME:  could probably avoid some of these levels of indirection
 	static
-	function<void(string s)> current_catch_explainer = [&](string s) {
+	function<void(string s)> current_catch_explainer = [](string s) {
 		cerr << s << endl;
 	};
 

lib/extentwalker.cc

@@ -14,7 +14,6 @@ namespace crucible {
 	// fm_start, fm_length, fm_flags, m_extents
 	// fe_logical, fe_physical, fe_length, fe_flags
 
-	static const off_t MAX_OFFSET = numeric_limits<off_t>::max();
 	static const off_t FIEMAP_BLOCK_SIZE = 4096;
 
 	static bool __ew_do_log = getenv("EXTENTWALKER_DEBUG");

lib/process.cc

@@ -110,9 +110,6 @@ namespace crucible {
 		}
 	}
 
-	template<>
-	struct ResourceHandle<Process::id, Process>;
-
 	pid_t
 	gettid()
 	{

scripts/beesd@.service.in

@@ -6,20 +6,19 @@ After=sysinit.target
 [Service]
 Type=simple
 ExecStart=@PREFIX@/sbin/beesd --no-timestamps %i
-Nice=19
-KillMode=control-group
-KillSignal=SIGTERM
-CPUShares=128
-StartupCPUShares=256
-BlockIOWeight=100
-StartupBlockIOWeight=250
+CPUAccounting=true
+CPUSchedulingPolicy=batch
+CPUWeight=12
 IOSchedulingClass=idle
 IOSchedulingPriority=7
-CPUSchedulingPolicy=batch
+IOWeight=10
+KillMode=control-group
+KillSignal=SIGTERM
+MemoryAccounting=true
 Nice=19
 Restart=on-abnormal
-CPUAccounting=true
-MemoryAccounting=true
+StartupCPUWeight=25
+StartupIOWeight=25
 
 [Install]
 WantedBy=basic.target

src/.gitignore

@@ -1 +1,2 @@
 bees-version.[ch]
+bees-version.new.c

src/Makefile

@@ -6,6 +6,7 @@ PROGRAMS = \
 all: $(PROGRAMS)
 
 include ../makeflags
+-include ../localconf
 
 LIBS = -lcrucible -lpthread
 LDFLAGS = -L../lib
@@ -20,7 +21,7 @@ BEES_OBJS = \
 	bees-types.o \
 
 bees-version.c: bees.h $(BEES_OBJS:.o=.cc) Makefile
-	echo "const char *BEES_VERSION = \"$(shell git describe --always --dirty || echo UNKNOWN)\";" > bees-version.new.c
+	echo "const char *BEES_VERSION = \"$(BEES_VERSION)\";" > bees-version.new.c
 	mv -f bees-version.new.c bees-version.c
 
 .depends/%.dep: %.cc Makefile

src/bees-context.cc

@@ -11,17 +11,6 @@
 using namespace crucible;
 using namespace std;
 
-static inline
-const char *
-getenv_or_die(const char *name)
-{
-	const char *rv = getenv(name);
-	if (!rv) {
-		THROW_ERROR(runtime_error, "Environment variable " << name << " not defined");
-	}
-	return rv;
-}
-
 BeesFdCache::BeesFdCache()
 {
 	m_root_cache.func([&](shared_ptr<BeesContext> ctx, uint64_t root) -> Fd {
@@ -150,6 +139,10 @@ BeesContext::show_progress()
 Fd
 BeesContext::home_fd()
 {
+	if (!!m_home_fd) {
+		return m_home_fd;
+	}
+
 	const char *base_dir = getenv("BEESHOME");
 	if (!base_dir) {
 		base_dir = ".beeshome";
@@ -169,15 +162,28 @@ BeesContext::BeesContext(shared_ptr<BeesContext> parent) :
 	}
 }
 
+bool
+BeesContext::is_root_ro(uint64_t root)
+{
+	return roots()->is_root_ro(root);
+}
+
 bool
 BeesContext::dedup(const BeesRangePair &brp)
 {
 	// TOOLONG and NOTE can retroactively fill in the filename details, but LOG can't
 	BEESNOTE("dedup " << brp);
 
-	brp.first.fd(shared_from_this());
 	brp.second.fd(shared_from_this());
 
+	if (is_root_ro(brp.second.fid().root())) {
+		// BEESLOGDEBUG("WORKAROUND: dst subvol is read-only in " << name_fd(brp.second.fd()));
+		BEESCOUNT(dedup_workaround_btrfs_send);
+		return false;
+	}
+
+	brp.first.fd(shared_from_this());
+
 	BEESTOOLONG("dedup " << brp);
 
 	BeesAddress first_addr(brp.first.fd(), brp.first.begin());
@@ -756,11 +762,42 @@ BeesResolveAddrResult::BeesResolveAddrResult()
 {
 }
 
+void
+BeesContext::wait_for_balance()
+{
+	Timer balance_timer;
+	BEESNOTE("WORKAROUND: waiting for balance to stop");
+	while (true) {
+		btrfs_ioctl_balance_args args;
+		memset_zero<btrfs_ioctl_balance_args>(&args);
+		const int ret = ioctl(root_fd(), BTRFS_IOC_BALANCE_PROGRESS, &args);
+		if (ret < 0) {
+			// Either can't get balance status or not running, exit either way
+			break;
+		}
+
+		if (!(args.state & BTRFS_BALANCE_STATE_RUNNING)) {
+			// Balance not running, doesn't matter if paused or cancelled
+			break;
+		}
+
+		BEESLOGDEBUG("WORKAROUND: Waiting " << balance_timer << "s for balance to stop");
+		sleep(BEES_BALANCE_POLL_INTERVAL);
+	}
+}
+
 BeesResolveAddrResult
 BeesContext::resolve_addr_uncached(BeesAddress addr)
 {
 	THROW_CHECK1(invalid_argument, addr, !addr.is_magic());
 	THROW_CHECK0(invalid_argument, !!root_fd());
+
+	// Is there a bug where resolve and balance cause a crash (BUG_ON at fs/btrfs/ctree.c:1227)?
+	// Apparently yes, and more than one.
+	// Wait for the balance to finish before we run LOGICAL_INO
+	wait_for_balance();
+
+	// Time how long this takes
 	Timer resolve_timer;
 
 	// There is no performance benefit if we restrict the buffer size.

src/bees-roots.cc

@@ -11,8 +11,6 @@
 using namespace crucible;
 using namespace std;
 
-BeesRoots::ScanMode BeesRoots::s_scan_mode = BeesRoots::SCAN_MODE_ZERO;
-
 string
 format_time(time_t t)
 {
@@ -46,8 +44,8 @@ BeesCrawlState::BeesCrawlState() :
 bool
 BeesCrawlState::operator<(const BeesCrawlState &that) const
 {
-	return tie(m_min_transid, m_objectid, m_offset, m_root, m_max_transid)
-		< tie(that.m_min_transid, that.m_objectid, that.m_offset, that.m_root, that.m_max_transid);
+	return tie(m_min_transid, m_max_transid, m_objectid, m_offset, m_root)
+		< tie(that.m_min_transid, that.m_max_transid, that.m_objectid, that.m_offset, that.m_root);
 }
 
 string
@@ -67,10 +65,21 @@ void
 BeesRoots::set_scan_mode(ScanMode mode)
 {
 	THROW_CHECK1(invalid_argument, mode, mode < SCAN_MODE_COUNT);
-	s_scan_mode = mode;
+	m_scan_mode = mode;
 	BEESLOGINFO("Scan mode set to " << mode << " (" << scan_mode_ntoa(mode) << ")");
 }
 
+void
+BeesRoots::set_workaround_btrfs_send(bool do_avoid)
+{
+	m_workaround_btrfs_send = do_avoid;
+	if (m_workaround_btrfs_send) {
+		BEESLOGINFO("WORKAROUND: btrfs send workaround enabled");
+	} else {
+		BEESLOGINFO("btrfs send workaround disabled");
+	}
+}
+
 string
 BeesRoots::crawl_state_filename() const
 {
@@ -185,9 +194,12 @@ BeesRoots::transid_min()
 		return 0;
 	}
 	uint64_t rv = numeric_limits<uint64_t>::max();
+	const uint64_t max_rv = rv;
 	for (auto i : m_root_crawl_map) {
 		rv = min(rv, i.second->get_state_end().m_min_transid);
 	}
+	// If we get through this loop without setting rv, we'll create broken crawlers due to integer overflow.
+	THROW_CHECK2(runtime_error, rv, max_rv, max_rv > rv);
 	return rv;
 }
 
@@ -195,19 +207,15 @@ uint64_t
 BeesRoots::transid_max_nocache()
 {
 	uint64_t rv = 0;
-	uint64_t root = BTRFS_FS_TREE_OBJECTID;
-	BEESNOTE("Calculating transid_max (" << rv << " as of root " << root << ")");
-	BEESTRACE("Calculating transid_max...");
-
-	rv = btrfs_get_root_transid(root);
-
-	// XXX:  Do we need any of this?  Or is
-	// m_transid_re.update(btrfs_get_root_transid(BTRFS_FS_TREE_OBJECTID)) good enough?
+	BEESNOTE("Calculating transid_max");
+	BEESTRACE("Calculating transid_max");
 
+	// We look for the root of the extent tree and read its transid.
+	// Should run in O(1) time and be fairly reliable.
 	BtrfsIoctlSearchKey sk;
 	sk.tree_id = BTRFS_ROOT_TREE_OBJECTID;
-	sk.min_type = sk.max_type = BTRFS_ROOT_BACKREF_KEY;
-	sk.min_objectid = root;
+	sk.min_type = sk.max_type = BTRFS_ROOT_ITEM_KEY;
+	sk.min_objectid = sk.max_objectid = BTRFS_EXTENT_TREE_OBJECTID;
 
 	while (true) {
 		sk.nr_items = 1024;
@@ -217,21 +225,18 @@ BeesRoots::transid_max_nocache()
 			break;
 		}
 
+		// We are just looking for the highest transid on the filesystem.
+		// We don't care which object it comes from.
 		for (auto i : sk.m_result) {
 			sk.next_min(i);
-			if (i.type == BTRFS_ROOT_BACKREF_KEY) {
-				if (i.transid > rv) {
-					BEESLOGDEBUG("transid_max root " << i.objectid << " parent " << i.offset << " transid " << i.transid);
-					BEESCOUNT(transid_max_miss);
-				}
-				root = i.objectid;
-			}
 			if (i.transid > rv) {
 				rv = i.transid;
 			}
 		}
 	}
-	m_transid_re.update(rv);
+
+	// transid must be greater than zero, or we did something very wrong
+	THROW_CHECK1(runtime_error, rv, rv > 0);
 	return rv;
 }
 
@@ -285,7 +290,7 @@ BeesRoots::crawl_roots()
 		BEESLOGINFO("idle: crawl map is empty!");
 	}
 
-	switch (s_scan_mode) {
+	switch (m_scan_mode) {
 
 		case SCAN_MODE_ZERO: {
 			// Scan the same inode/offset tuple in each subvol (good for snapshots)
@@ -370,6 +375,13 @@ BeesRoots::crawl_roots()
 	return false;
 }
 
+void
+BeesRoots::clear_caches()
+{
+	m_ctx->fd_cache()->clear();
+	m_root_ro_cache.clear();
+}
+
 void
 BeesRoots::crawl_thread()
 {
@@ -380,15 +392,22 @@ BeesRoots::crawl_thread()
 	m_crawl_task = Task("crawl_master", [shared_this]() {
 		auto tqs = TaskMaster::get_queue_count();
 		BEESNOTE("queueing extents to scan, " << tqs << " of " << BEES_MAX_QUEUE_SIZE);
+#if 0
 		bool run_again = true;
 		while (tqs < BEES_MAX_QUEUE_SIZE && run_again) {
 			run_again = shared_this->crawl_roots();
 			tqs = TaskMaster::get_queue_count();
 		}
+#else
+		bool run_again = false;
+		while (tqs < BEES_MAX_QUEUE_SIZE) {
+			run_again = shared_this->crawl_roots();
+			tqs = TaskMaster::get_queue_count();
+			if (!run_again) break;
+		}
+#endif
 		if (run_again) {
 			shared_this->m_crawl_task.run();
-		} else {
-			shared_this->m_task_running = false;
 		}
 	});
 
@@ -412,17 +431,12 @@ BeesRoots::crawl_thread()
 		// Even open files are a problem if they're big enough.
 		auto new_count = m_transid_re.count();
 		if (new_count != last_count) {
-			m_ctx->fd_cache()->clear();
+			clear_caches();
 		}
 		last_count = new_count;
 
 		// If no crawl task is running, start a new one
-		bool already_running = m_task_running.exchange(true);
-		if (!already_running) {
-			auto resumed_after_time = m_crawl_timer.lap();
-			BEESLOGINFO("Crawl master resumed after " << resumed_after_time << "s at transid " << new_count);
-			m_crawl_task.run();
-		}
+		m_crawl_task.run();
 
 		auto poll_time = m_transid_re.seconds_for(m_transid_factor);
 		BEESLOGDEBUG("Polling " << poll_time << "s for next " << m_transid_factor << " transid " << m_transid_re);
@@ -524,6 +538,16 @@ BeesRoots::state_load()
 			loaded_state.m_started = d.at("started");
 		}
 		BEESLOGDEBUG("loaded_state " << loaded_state);
+		if (loaded_state.m_min_transid == numeric_limits<uint64_t>::max()) {
+			BEESLOGWARN("WARNING: root " << loaded_state.m_root << ": bad min_transid " << loaded_state.m_min_transid << ", resetting to 0");
+			loaded_state.m_min_transid = 0;
+			BEESCOUNT(bug_bad_min_transid);
+		}
+		if (loaded_state.m_max_transid == numeric_limits<uint64_t>::max()) {
+			BEESLOGWARN("WARNING: root " << loaded_state.m_root << ": bad max_transid " << loaded_state.m_max_transid << ", resetting to " << loaded_state.m_min_transid);
+			loaded_state.m_max_transid = loaded_state.m_min_transid;
+			BEESCOUNT(bug_bad_max_transid);
+		}
 		insert_root(loaded_state);
 	}
 }
@@ -532,9 +556,14 @@ BeesRoots::BeesRoots(shared_ptr<BeesContext> ctx) :
 	m_ctx(ctx),
 	m_crawl_state_file(ctx->home_fd(), crawl_state_filename()),
 	m_crawl_thread("crawl_transid"),
-	m_writeback_thread("crawl_writeback"),
-	m_task_running(false)
+	m_writeback_thread("crawl_writeback")
 {
+
+	m_root_ro_cache.func([&](uint64_t root) -> bool {
+		return is_root_ro_nocache(root);
+	});
+	m_root_ro_cache.max_size(BEES_ROOT_FD_CACHE_SIZE);
+
 	m_crawl_thread.exec([&]() {
 		// Measure current transid before creating any crawlers
 		catch_all([&]() {
@@ -640,6 +669,7 @@ BeesRoots::open_root_nocache(uint64_t rootid)
 				Stat st(rv);
 				THROW_CHECK1(runtime_error, st.st_ino, st.st_ino == BTRFS_FIRST_FREE_OBJECTID);
 				// BEESLOGDEBUG("open_root_nocache " << rootid << ": " << name_fd(rv));
+
 				BEESCOUNT(root_ok);
 				return rv;
 			}
@@ -661,6 +691,32 @@ BeesRoots::open_root(uint64_t rootid)
 	return m_ctx->fd_cache()->open_root(m_ctx, rootid);
 }
 
+bool
+BeesRoots::is_root_ro_nocache(uint64_t root)
+{
+	Fd root_fd = open_root(root);
+	BEESTRACE("checking subvol flags on root " << root << " path " << name_fd(root_fd));
+
+	uint64_t flags = 0;
+	DIE_IF_NON_ZERO(ioctl(root_fd, BTRFS_IOC_SUBVOL_GETFLAGS, &flags));
+	if (flags & BTRFS_SUBVOL_RDONLY) {
+		BEESLOGDEBUG("WORKAROUND: Avoiding RO subvol " << root);
+		BEESCOUNT(root_workaround_btrfs_send);
+		return true;
+	}
+	return false;
+}
+
+bool
+BeesRoots::is_root_ro(uint64_t root)
+{
+	// If we are not implementing the workaround there is no need for cache
+	if (!m_workaround_btrfs_send) {
+		return false;
+	}
+
+	return m_root_ro_cache(root);
+}
 
 uint64_t
 BeesRoots::next_root(uint64_t root)
@@ -883,6 +939,20 @@ BeesCrawl::fetch_extents()
 		return next_transid();
 	}
 
+	// Check for btrfs send workaround: don't scan RO roots at all, pretend
+	// they are just empty.  We can't free any space there, and we
+	// don't have the necessary analysis logic to be able to use
+	// them as dedup src extents (yet).
+	//
+	// This will keep the max_transid up to date so if the root
+	// is ever switched back to read-write, it won't trigger big
+	// expensive in-kernel searches for ancient transids.
+	if (m_ctx->is_root_ro(old_state.m_root)) {
+		BEESLOGDEBUG("WORKAROUND: RO root " << old_state.m_root);
+		BEESCOUNT(root_workaround_btrfs_send);
+		return next_transid();
+	}
+
 	BEESNOTE("crawling " << get_state_end());
 
 	Timer crawl_timer;
@@ -903,7 +973,7 @@ BeesCrawl::fetch_extents()
 	// Lock in the old state
 	set_state(old_state);
 
-	BEESTRACE("Searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));
+	BEESTRACE("Searching crawl sk " << sk);
 	bool ioctl_ok = false;
 	{
 		BEESNOTE("searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));

src/bees.cc

@@ -32,9 +32,10 @@ using namespace std;
 
 int bees_log_level = 8;
 
-int
+void
 do_cmd_help(char *argv[])
 {
+	// 80col 01234567890123456789012345678901234567890123456789012345678901234567890123456789
 	cerr << "Usage: " << argv[0] << " [options] fs-root-path [fs-root-path-2...]\n"
 		"Performs best-effort extent-same deduplication on btrfs.\n"
 		"\n"
@@ -42,27 +43,36 @@ do_cmd_help(char *argv[])
 		"Other directories will be rejected.\n"
 		"\n"
 		"Options:\n"
-		"\t-h, --help\t\tShow this help\n"
-		"\t-c, --thread-count\tWorker thread count (default CPU count * factor)\n"
-		"\t-C, --thread-factor\tWorker thread factor (default " << BEES_DEFAULT_THREAD_FACTOR << ")\n"
-		"\t-G, --thread-min\t\tMinimum worker thread count with load average target (default 0)\n"
-		"\t-g, --loadavg-target\t\tTarget load average for worker threads (default is no target)\n"
-		"\t-m, --scan-mode\t\tScanning mode (0..2, default 0)\n"
-		"\t-t, --timestamps\tShow timestamps in log output (default)\n"
-		"\t-T, --no-timestamps\tOmit timestamps in log output\n"
-		"\t-p, --absolute-paths\tShow absolute paths (default)\n"
-		"\t-P, --strip-paths\tStrip $CWD from beginning of all paths in the log\n"
-		"\t-v, --verbose\tSet maximum log level (0..8, default 8)\n"
+		"    -h, --help            Show this help\n"
+		"\n"
+		"Load management options:\n"
+		"    -c, --thread-count    Worker thread count (default CPU count * factor)\n"
+		"    -C, --thread-factor   Worker thread factor (default " << BEES_DEFAULT_THREAD_FACTOR << ")\n"
+		"    -G, --thread-min      Minimum worker thread count (default 0)\n"
+		"    -g, --loadavg-target  Target load average for worker threads (default none)\n"
+		"\n"
+		"Filesystem tree traversal options:\n"
+		"    -m, --scan-mode       Scanning mode (0..2, default 0)\n"
+		"\n"
+		"Workarounds:\n"
+		"    -a, --workaround-btrfs-send    Workaround for btrfs send\n"
+		"\n"
+		"Logging options:\n"
+		"    -t, --timestamps      Show timestamps in log output (default)\n"
+		"    -T, --no-timestamps   Omit timestamps in log output\n"
+		"    -p, --absolute-paths  Show absolute paths (default)\n"
+		"    -P, --strip-paths     Strip $CWD from beginning of all paths in the log\n"
+		"    -v, --verbose         Set maximum log level (0..8, default 8)\n"
 		"\n"
 		"Optional environment variables:\n"
-		"\tBEESHOME\tPath to hash table and configuration files\n"
-		"\t\t\t(default is .beeshome/ in the root of each filesystem).\n"
+		"    BEESHOME    Path to hash table and configuration files\n"
+		"                (default is .beeshome/ in the root of each filesystem).\n"
 		"\n"
-		"\tBEESSTATUS\tFile to write status to (tmpfs recommended, e.g. /run).\n"
-		"\t\t\tNo status is written if this variable is unset.\n"
+		"    BEESSTATUS  File to write status to (tmpfs recommended, e.g. /run).\n"
+		"                No status is written if this variable is unset.\n"
 		"\n"
+	// 80col 01234567890123456789012345678901234567890123456789012345678901234567890123456789
 	<< endl;
-	return 0;
 }
 
 // tracing ----------------------------------------
@@ -115,9 +125,9 @@ BeesNote::~BeesNote()
 	tl_next = m_prev;
 	unique_lock<mutex> lock(s_mutex);
 	if (tl_next) {
-		s_status[gettid()] = tl_next;
+		s_status[crucible::gettid()] = tl_next;
 	} else {
-		s_status.erase(gettid());
+		s_status.erase(crucible::gettid());
 	}
 }
 
@@ -128,7 +138,7 @@ BeesNote::BeesNote(function<void(ostream &os)> f) :
 	m_prev = tl_next;
 	tl_next = this;
 	unique_lock<mutex> lock(s_mutex);
-	s_status[gettid()] = tl_next;
+	s_status[crucible::gettid()] = tl_next;
 }
 
 void
@@ -194,20 +204,6 @@ BeesNote::get_status()
 
 // static inline helpers ----------------------------------------
 
-static inline
-bool
-bees_addr_check(uint64_t v)
-{
-	return !(v & (1ULL << 63));
-}
-
-static inline
-bool
-bees_addr_check(int64_t v)
-{
-	return !(v & (1ULL << 63));
-}
-
 string
 pretty(double d)
 {
@@ -616,7 +612,7 @@ BeesTempFile::make_copy(const BeesFileRange &src)
 
 	// We seem to get lockups without this!
 	if (did_block_write) {
-#if 1
+#if 0
 		// Is this fixed by "Btrfs: fix deadlock between dedup on same file and starting writeback"?
 		// No.
 		// Is this fixed in kernel 4.14.34?
@@ -643,11 +639,11 @@ bees_main(int argc, char *argv[])
 	BeesNote::set_name("bees");
 	BEESNOTE("main");
 
-	list<shared_ptr<BeesContext>> all_contexts;
-	shared_ptr<BeesContext> bc;
-
 	THROW_CHECK1(invalid_argument, argc, argc >= 0);
 
+	// Create a context so we can apply configuration to it
+	shared_ptr<BeesContext> bc = make_shared<BeesContext>();
+
 	string cwd(readlink_or_die("/proc/self/cwd"));
 
 	// Defaults
@@ -656,26 +652,48 @@ bees_main(int argc, char *argv[])
 	unsigned thread_count = 0;
 	unsigned thread_min = 0;
 	double load_target = 0;
+	bool workaround_btrfs_send = false;
+	BeesRoots::ScanMode root_scan_mode = BeesRoots::SCAN_MODE_ZERO;
+
+	// Configure getopt_long
+	static const struct option long_options[] = {
+		{ "thread-factor",         required_argument, NULL, 'C' },
+		{ "thread-min",            required_argument, NULL, 'G' },
+		{ "strip-paths",           no_argument,       NULL, 'P' },
+		{ "no-timestamps",         no_argument,       NULL, 'T' },
+		{ "workaround-btrfs-send", no_argument,       NULL, 'a' },
+		{ "thread-count",          required_argument, NULL, 'c' },
+		{ "loadavg-target",        required_argument, NULL, 'g' },
+		{ "help",                  no_argument,       NULL, 'h' },
+		{ "scan-mode",             required_argument, NULL, 'm' },
+		{ "absolute-paths",        no_argument,       NULL, 'p' },
+		{ "timestamps",            no_argument,       NULL, 't' },
+		{ "verbose",               required_argument, NULL, 'v' },
+		{ 0, 0, 0, 0 },
+	};
+
+	// Build getopt_long's short option list from the long_options table.
+	// While we're at it, make sure we didn't duplicate any options.
+	string getopt_list;
+	set<decltype(option::val)> option_vals;
+	for (const struct option *op = long_options; op->val; ++op) {
+		THROW_CHECK1(runtime_error, op->val, !option_vals.count(op->val));
+		option_vals.insert(op->val);
+		if ((op->val & 0xff) != op->val) {
+			continue;
+		}
+		getopt_list += op->val;
+		if (op->has_arg == required_argument) {
+			getopt_list += ':';
+		}
+	}
 
 	// Parse options
 	int c;
-	while (1) {
+	while (true) {
 		int option_index = 0;
-		static const struct option long_options[] = {
-			{ "thread-factor",  required_argument, NULL, 'C' },
-			{ "thread-min",     required_argument, NULL, 'G' },
-			{ "strip-paths",    no_argument,       NULL, 'P' },
-			{ "no-timestamps",  no_argument,       NULL, 'T' },
-			{ "thread-count",   required_argument, NULL, 'c' },
-			{ "loadavg-target", required_argument, NULL, 'g' },
-			{ "help",           no_argument,       NULL, 'h' },
-			{ "scan-mode", 	    required_argument, NULL, 'm' },
-			{ "absolute-paths", no_argument,       NULL, 'p' },
-			{ "timestamps",     no_argument,       NULL, 't' },
-			{ "verbose",        required_argument, NULL, 'v' },
-		};
 
-		c = getopt_long(argc, argv, "C:G:PTc:hg:m:ptv:", long_options, &option_index);
+		c = getopt_long(argc, argv, getopt_list.c_str(), long_options, &option_index);
 		if (-1 == c) {
 			break;
 		}
@@ -694,6 +712,9 @@ bees_main(int argc, char *argv[])
 			case 'T':
 				chatter_prefix_timestamp = false;
 				break;
+			case 'a':
+				workaround_btrfs_send = true;
+				break;
 			case 'c':
 				thread_count = stoul(optarg);
 				break;
@@ -701,7 +722,7 @@ bees_main(int argc, char *argv[])
 				load_target = stod(optarg);
 				break;
 			case 'm':
-				BeesRoots::set_scan_mode(static_cast<BeesRoots::ScanMode>(stoul(optarg)));
+				root_scan_mode = static_cast<BeesRoots::ScanMode>(stoul(optarg));
 				break;
 			case 'p':
 				crucible::set_relative_path("");
@@ -720,12 +741,17 @@ bees_main(int argc, char *argv[])
 				break;
 
 			case 'h':
-				do_cmd_help(argv); // fallthrough
 			default:
-				return 2;
+				do_cmd_help(argv);
+				return EXIT_FAILURE;
 		}
 	}
 
+	if (optind + 1 != argc) {
+		BEESLOGERR("Only one filesystem path per bees process");
+		return EXIT_FAILURE;
+	}
+
 	Chatter::enable_timestamp(chatter_prefix_timestamp);
 
 	if (!relative_path().empty()) {
@@ -750,6 +776,11 @@ bees_main(int argc, char *argv[])
 			thread_factor = BEES_DEFAULT_THREAD_FACTOR;
 		}
 		thread_count = max(1U, static_cast<unsigned>(ceil(thread::hardware_concurrency() * thread_factor)));
+		if (thread_count > BEES_DEFAULT_THREAD_LIMIT) {
+			BEESLOGNOTICE("Limiting computed thread count to " << BEES_DEFAULT_THREAD_LIMIT);
+			BEESLOGNOTICE("Use --thread-count to override this limit");
+			thread_count = BEES_DEFAULT_THREAD_LIMIT;
+		}
 	}
 
 	if (load_target != 0) {
@@ -762,19 +793,16 @@ bees_main(int argc, char *argv[])
 	BEESLOGNOTICE("setting worker thread pool maximum size to " << thread_count);
 	TaskMaster::set_thread_count(thread_count);
 
-	// Create a context and start crawlers
-	bool did_subscription = false;
-	while (optind < argc) {
-		catch_all([&]() {
-			bc = make_shared<BeesContext>(bc);
-			bc->set_root_path(argv[optind++]);
-			did_subscription = true;
-		});
-	}
+	// Set root path
+	string root_path = argv[optind++];
+	BEESLOGNOTICE("setting root path to '" << root_path << "'");
+	bc->set_root_path(root_path);
 
-	if (!did_subscription) {
-		BEESLOGWARN("WARNING: no filesystems added");
-	}
+	// Workaround for btrfs send
+	bc->roots()->set_workaround_btrfs_send(workaround_btrfs_send);
+
+	// Set root scan mode
+	bc->roots()->set_scan_mode(root_scan_mode);
 
 	BeesThread status_thread("status", [&]() {
 		bc->dump_status();
@@ -784,7 +812,7 @@ bees_main(int argc, char *argv[])
 	bc->show_progress();
 
 	// That is all.
-	return 0;
+	return EXIT_SUCCESS;
 }
 
 int
@@ -794,7 +822,7 @@ main(int argc, char *argv[])
 
 	if (argc < 2) {
 		do_cmd_help(argv);
-		return 2;
+		return EXIT_FAILURE;
 	}
 
 	int rv = 1;

src/bees.h

@@ -85,6 +85,9 @@ const size_t BEES_OPEN_FILE_LIMIT = (BEES_FILE_FD_CACHE_SIZE + BEES_ROOT_FD_CACH
 // Worker thread factor (multiplied by detected number of CPU cores)
 const double BEES_DEFAULT_THREAD_FACTOR = 1.0;
 
+// Don't use more than this number of threads unless explicitly configured
+const size_t BEES_DEFAULT_THREAD_LIMIT = 8;
+
 // Log warnings when an operation takes too long
 const double BEES_TOO_LONG = 5.0;
 
@@ -114,6 +117,9 @@ const size_t BEES_TRANSID_FACTOR = 10;
 // The actual limit in LOGICAL_INO seems to be 2730, but let's leave a little headroom
 const size_t BEES_MAX_EXTENT_REF_COUNT = 2560;
 
+// Wait this long for a balance to stop
+const double BEES_BALANCE_POLL_INTERVAL = 60.0;
+
 // Flags
 const int FLAGS_OPEN_COMMON   = O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC | O_NOATIME | O_LARGEFILE | O_NOCTTY;
 const int FLAGS_OPEN_DIR      = FLAGS_OPEN_COMMON | O_RDONLY | O_DIRECTORY;
@@ -167,7 +173,7 @@ public:
 	T at(string idx) const;
 
 friend ostream& operator<< <>(ostream &os, const BeesStatTmpl<T> &bs);
-friend class BeesStats;
+friend struct BeesStats;
 };
 
 using BeesRates = BeesStatTmpl<double>;
@@ -532,13 +538,15 @@ class BeesRoots : public enable_shared_from_this<BeesRoots> {
 	BeesThread				m_writeback_thread;
 	RateEstimator				m_transid_re;
 	size_t					m_transid_factor = BEES_TRANSID_FACTOR;
-	atomic<bool>				m_task_running;
 	Task					m_crawl_task;
+	bool					m_workaround_btrfs_send = false;
+	LRUCache<bool, uint64_t>		m_root_ro_cache;
 
 	void insert_new_crawl();
 	void insert_root(const BeesCrawlState &bcs);
 	Fd open_root_nocache(uint64_t root);
 	Fd open_root_ino_nocache(uint64_t root, uint64_t ino);
+	bool is_root_ro_nocache(uint64_t root);
 	uint64_t transid_min();
 	uint64_t transid_max();
 	uint64_t transid_max_nocache();
@@ -555,6 +563,7 @@ class BeesRoots : public enable_shared_from_this<BeesRoots> {
 	void current_state_set(const BeesCrawlState &bcs);
 	RateEstimator& transid_re();
 	size_t crawl_batch(shared_ptr<BeesCrawl> crawl);
+	void clear_caches();
 
 friend class BeesFdCache;
 friend class BeesCrawl;
@@ -564,6 +573,7 @@ public:
 	Fd open_root(uint64_t root);
 	Fd open_root_ino(uint64_t root, uint64_t ino);
 	Fd open_root_ino(const BeesFileId &bfi) { return open_root_ino(bfi.root(), bfi.ino()); }
+	bool is_root_ro(uint64_t root);
 
 	// TODO:  think of better names for these.
 	// or TODO:  do extent-tree scans instead
@@ -574,10 +584,11 @@ public:
 		SCAN_MODE_COUNT, // must be last
 	};
 
-	static void set_scan_mode(ScanMode new_mode);
+	void set_scan_mode(ScanMode new_mode);
+	void set_workaround_btrfs_send(bool do_avoid);
 
 private:
-	static ScanMode s_scan_mode;
+	ScanMode m_scan_mode = SCAN_MODE_ZERO;
 	static string scan_mode_ntoa(ScanMode new_mode);
 
 };
@@ -708,6 +719,7 @@ class BeesContext : public enable_shared_from_this<BeesContext> {
 	void set_root_fd(Fd fd);
 
 	BeesResolveAddrResult resolve_addr_uncached(BeesAddress addr);
+	void wait_for_balance();
 
 	BeesFileRange scan_one_extent(const BeesFileRange &bfr, const Extent &e);
 	void rewrite_file_range(const BeesFileRange &bfr);
@@ -724,6 +736,7 @@ public:
 
 	BeesFileRange scan_forward(const BeesFileRange &bfr);
 
+	bool is_root_ro(uint64_t root);
 	BeesRangePair dup_extent(const BeesFileRange &src);
 	bool dedup(const BeesRangePair &brp);
 

test/Makefile

@@ -14,6 +14,7 @@ test: $(PROGRAMS:%=%.txt) Makefile
 FORCE:
 
 include ../makeflags
+-include ../localconf
 
 LIBS = -lcrucible -lpthread
 LDFLAGS = -L../lib -Wl,-rpath=$(shell realpath ../lib)

test/task.cc

@@ -99,7 +99,7 @@ test_barrier(size_t count)
 		oss << "task #" << c;
 		Task t(
 			oss.str(),
-			[c, &task_done, &mtx, &cv, bl]() mutable {
+			[c, &task_done, &mtx, bl]() mutable {
 				// cerr << "Task #" << c << endl;
 				unique_lock<mutex> lock(mtx);
 				task_done.at(c) = true;
@@ -166,8 +166,9 @@ test_exclusion(size_t count)
 		oss << "task #" << c;
 		Task t(
 			oss.str(),
-			[c, &only_one, &mtx, &excl, bl]() mutable {
+			[c, &only_one, &excl, bl]() mutable {
 				// cerr << "Task #" << c << endl;
+				(void)c;
 				auto lock = excl.try_lock();
 				if (!lock) {
 					excl.insert_task(Task::current_task());