context: raise limit on the number of concurrent ioctls to cpu_cores/2

This might improve performance on systems with more than 3 CPU cores...or
it might bring such a machine to its knees.

TODO:  find out which of those two things happens.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>

src/bees-context.cc

@@ -1,6 +1,7 @@
 #include "bees.h"
 
 #include "crucible/limits.h"
+#include "crucible/process.h"
 #include "crucible/string.h"
 
 #include <fstream>
@@ -280,7 +281,7 @@ BeesContext::dedup(const BeesRangePair &brp)
 	// To avoid hammering all the cores with long-running ioctls,
 	// only do one dedup at any given time.
 	BEESNOTE("Waiting to dedup " << brp);
-	unique_lock<mutex> lock(bees_ioctl_mutex);
+	auto dedup_lock = bees_ioctl_lock_set.make_lock(gettid());
 #endif
 
 	BEESNOTE("dedup " << brp);
@@ -850,7 +851,7 @@ BeesContext::scan_forward(const BeesFileRange &bfr)
 			catch_all([&]() {
 				uint64_t extent_bytenr = e.bytenr();
 				BEESNOTE("waiting for extent bytenr " << to_hex(extent_bytenr));
-				decltype(m_extent_lock_set)::Lock extent_lock(m_extent_lock_set, extent_bytenr);
+				auto extent_lock = m_extent_lock_set.make_lock(extent_bytenr);
 				Timer one_extent_timer;
 				return_bfr = scan_one_extent(bfr, e);
 				BEESCOUNTADD(scanf_extent_ms, one_extent_timer.age() * 1000);
@@ -886,7 +887,7 @@ BeesContext::resolve_addr_uncached(BeesAddress addr)
 	// To avoid hammering all the cores with long-running ioctls,
 	// only do one resolve at any given time.
 	BEESNOTE("waiting to resolve addr " << addr);
-	unique_lock<mutex> lock(bees_ioctl_mutex);
+	auto lock = bees_ioctl_lock_set.make_lock(gettid());
 
 	Timer resolve_timer;
 

src/bees-hash.cc

@@ -311,7 +311,7 @@ BeesHashTable::fetch_missing_extent(HashType hash)
 	BEESNOTE("waiting to fetch hash extent #" << extent_number << ", " << missing_buckets << " left to fetch");
 
 	// Acquire blocking lock on this extent only
-	decltype(m_extent_lock_set)::Lock extent_lock(m_extent_lock_set, extent_number);
+	auto extent_lock = m_extent_lock_set.make_lock(extent_number);
 
 	// Check missing again because someone else might have fetched this
 	// extent for us while we didn't hold any locks

src/bees-roots.cc

@@ -1,6 +1,7 @@
 #include "bees.h"
 
 #include "crucible/cache.h"
+#include "crucible/process.h"
 #include "crucible/string.h"
 
 #include <fstream>
@@ -554,7 +555,7 @@ void
 BeesCrawl::crawl_thread()
 {
 	Timer crawl_timer;
-	LockSet<uint64_t>::Lock crawl_lock(m_ctx->roots()->lock_set(), m_state.m_root, false);
+	auto crawl_lock = m_ctx->roots()->lock_set().make_lock(m_state.m_root, false);
 	while (!m_stopped) {
 		BEESNOTE("waiting for crawl thread limit " << m_state);
 		crawl_lock.lock();
@@ -646,7 +647,7 @@ BeesCrawl::fetch_extents()
 	bool ioctl_ok = false;
 	{
 		BEESNOTE("waiting to search crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));
-		unique_lock<mutex> lock(bees_ioctl_mutex);
+		auto lock = bees_ioctl_lock_set.make_lock(gettid());
 
 		BEESNOTE("searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));
 		BEESTOOLONG("Searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));

src/bees.cc

@@ -203,7 +203,7 @@ operator<<(ostream &os, const BeesStatTmpl<T> &bs)
  * Some of them consume egregious amounts of kernel CPU time and are
  * not interruptible, so if we have more threads than cores we will
  * effectively crash the kernel. */
-mutex bees_ioctl_mutex;
+LockSet<pid_t> bees_ioctl_lock_set;
 
 template <class T>
 T&
@@ -583,6 +583,9 @@ bees_main(int argc, const char **argv)
 	THROW_CHECK1(invalid_argument, argc, argc >= 0);
 	vector<string> args(argv + 1, argv + argc);
 
+	// Set global concurrency limits - use only half the cores for ioctls
+	bees_ioctl_lock_set.max_size(max(1U, bees_worker_thread_count() / 2));
+
 	// Create a context and start crawlers
 	bool did_subscription = false;
 	for (string arg : args) {

src/bees.h

@@ -830,7 +830,7 @@ string pretty(double d);
 extern RateLimiter bees_info_rate_limit;
 void bees_sync(int fd);
 string format_time(time_t t);
-extern mutex bees_ioctl_mutex;
+extern LockSet<pid_t> bees_ioctl_lock_set;
 extern unsigned bees_worker_thread_count();
 
 #endif