subvol-threads: increase resource and thread limits

With kernel 4.14 there is no sign of the previous LOGICAL_INO performance
problems, so there seems to be no need to throttle threads using this
ioctl.

Increase the FD cache size limits and scan thread count.  Let the kernel
figure out scheduling.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>

src/bees-context.cc

@@ -30,13 +30,14 @@ BeesFdCache::BeesFdCache()
 		BEESCOUNTADD(open_root_ms, open_timer.age() * 1000);
 		return rv;
 	});
+	m_root_cache.max_size(BEES_ROOT_FD_CACHE_SIZE);
 	m_file_cache.func([&](shared_ptr<BeesContext> ctx, uint64_t root, uint64_t ino) -> Fd {
 		Timer open_timer;
 		auto rv = ctx->roots()->open_root_ino_nocache(root, ino);
 		BEESCOUNTADD(open_ino_ms, open_timer.age() * 1000);
 		return rv;
 	});
-	m_file_cache.max_size(BEES_FD_CACHE_SIZE);
+	m_file_cache.max_size(BEES_FILE_FD_CACHE_SIZE);
 }
 
 Fd

src/bees-roots.cc

@@ -302,7 +302,8 @@ BeesRoots::BeesRoots(shared_ptr<BeesContext> ctx) :
 	m_crawl_state_file(ctx->home_fd(), crawl_state_filename()),
 	m_writeback_thread("crawl_writeback")
 {
-	m_lock_set.max_size(bees_worker_thread_count());
+	// This is a sanity check to prevent us from running out of FDs
+	m_lock_set.max_size(BEES_WORKER_THREAD_LIMIT);
 
 	catch_all([&]() {
 		state_load();

src/bees.cc

@@ -19,6 +19,10 @@
 #include <linux/fs.h>
 #include <sys/ioctl.h>
 
+// setrlimit
+#include <sys/time.h>
+#include <sys/resource.h>
+
 #include <getopt.h>
 
 using namespace crucible;
@@ -662,7 +666,19 @@ bees_main(int argc, char *argv[])
 	}
 
 	// There can be only one because we measure running time with it
-	bees_ioctl_lock_set.max_size(1);
+	// EXPERIMENT:  don't try this on kernels before v4.14
+	// bees_ioctl_lock_set.max_size(1);
+
+	BEESLOG("setting rlimit NOFILE to " << BEES_OPEN_FILE_LIMIT);
+
+	struct rlimit lim = {
+		.rlim_cur = BEES_OPEN_FILE_LIMIT,
+		.rlim_max = BEES_OPEN_FILE_LIMIT,
+	};
+	int rv = setrlimit(RLIMIT_NOFILE, &lim);
+	if (rv) {
+		BEESLOG("setrlimit(RLIMIT_NOFILE, { " << lim.rlim_cur << " }): " << strerror(errno));
+	};
 
 	// Create a context and start crawlers
 	bool did_subscription = false;

src/bees.h

@@ -75,14 +75,25 @@ const int BEES_PROGRESS_INTERVAL = BEES_STATS_INTERVAL;
 // Status is output every freakin second.  Use a ramdisk.
 const int BEES_STATUS_INTERVAL = 1;
 
-// Number of FDs to open (not counting 100 roots)
-const size_t BEES_FD_CACHE_SIZE = 384;
+// Number of file FDs to cache when not in active use
+const size_t BEES_FILE_FD_CACHE_SIZE = 4096;
+
+// Number of root FDs to cache when not in active use
+const size_t BEES_ROOT_FD_CACHE_SIZE = 1024;
+
+// Number of FDs to open (rlimit)
+const size_t BEES_OPEN_FILE_LIMIT = (BEES_FILE_FD_CACHE_SIZE + BEES_ROOT_FD_CACHE_SIZE) * 2 + 100;
+
+// Worker thread limit (more threads may be created, but only this number will be active concurrently)
+const size_t BEES_WORKER_THREAD_LIMIT = 128;
 
 // Log warnings when an operation takes too long
 const double BEES_TOO_LONG = 2.5;
 
 // Avoid any extent where LOGICAL_INO takes this long
-const double BEES_TOXIC_DURATION = 9.9;
+// const double BEES_TOXIC_DURATION = 9.9;
+// EXPERIMENT:  Kernel v4.14+ may let us ignore toxicity
+const double BEES_TOXIC_DURATION = BEES_COMMIT_INTERVAL;
 
 // How long between hash table histograms
 const double BEES_HASH_TABLE_ANALYZE_INTERVAL = BEES_STATS_INTERVAL;