GGLinnk/bees
1
0
Fork 0
mirror of https://github.com/Zygo/bees.git synced 2025-05-17 21:35:45 +02:00
Code Packages

subvol-threads: increase resource and thread limits

Browse Source 
With kernel 4.14 there is no sign of the previous LOGICAL_INO performance
problems, so there seems to be no need to throttle threads using this
ioctl.

Increase the FD cache size limits and scan thread count.  Let the kernel
figure out scheduling.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
This commit is contained in:
Zygo Blaxell 2018-01-15 23:07:12 -05:00
parent 42a6053229
commit 8d3a27bf85
3 changed files with 31 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/bees-context.cc
View File

@ -29,13 +29,14 @@ BeesFdCache::BeesFdCache()
BEESCOUNTADD(open_root_ms, open_timer.age() * 1000);
return rv;
});
m_root_cache.max_size(BEES_ROOT_FD_CACHE_SIZE);
m_file_cache.func([&](shared_ptr<BeesContext> ctx, uint64_t root, uint64_t ino) -> Fd {
Timer open_timer;
auto rv = ctx->roots()->open_root_ino_nocache(root, ino);
BEESCOUNTADD(open_ino_ms, open_timer.age() * 1000);
return rv;
});
m_file_cache.max_size(BEES_FD_CACHE_SIZE);
m_file_cache.max_size(BEES_FILE_FD_CACHE_SIZE);
}
Fd

15
src/bees.cc
View File

@ -19,6 +19,10 @@
#include <linux/fs.h>
#include <sys/ioctl.h>
// setrlimit
#include <sys/time.h>
#include <sys/resource.h>
#include <getopt.h>
using namespace crucible;
@ -647,6 +651,17 @@ bees_main(int argc, char *argv[])
BEESLOG("using relative path " << relative_path() << "\n");
}
BEESLOG("setting rlimit NOFILE to " << BEES_OPEN_FILE_LIMIT);
struct rlimit lim = {
.rlim_cur = BEES_OPEN_FILE_LIMIT,
.rlim_max = BEES_OPEN_FILE_LIMIT,
};
int rv = setrlimit(RLIMIT_NOFILE, &lim);
if (rv) {
BEESLOG("setrlimit(RLIMIT_NOFILE, { " << lim.rlim_cur << " }): " << strerror(errno));
};
// Create a context and start crawlers
bool did_subscription = false;
while (optind < argc) {

17
src/bees.h
View File

@ -75,14 +75,25 @@ const int BEES_PROGRESS_INTERVAL = BEES_STATS_INTERVAL;
// Status is output every freakin second. Use a ramdisk.
const int BEES_STATUS_INTERVAL = 1;
// Number of FDs to open (not counting 100 roots)
const size_t BEES_FD_CACHE_SIZE = 384;
// Number of file FDs to cache when not in active use
const size_t BEES_FILE_FD_CACHE_SIZE = 4096;
// Number of root FDs to cache when not in active use
const size_t BEES_ROOT_FD_CACHE_SIZE = 1024;
// Number of FDs to open (rlimit)
const size_t BEES_OPEN_FILE_LIMIT = (BEES_FILE_FD_CACHE_SIZE + BEES_ROOT_FD_CACHE_SIZE) * 2 + 100;
// Worker thread limit (more threads may be created, but only this number will be active concurrently)
const size_t BEES_WORKER_THREAD_LIMIT = 128;
// Log warnings when an operation takes too long
const double BEES_TOO_LONG = 2.5;
// Avoid any extent where LOGICAL_INO takes this long
const double BEES_TOXIC_DURATION = 9.9;
// const double BEES_TOXIC_DURATION = 9.9;
// EXPERIMENT: Kernel v4.14+ may let us ignore toxicity
const double BEES_TOXIC_DURATION = BEES_COMMIT_INTERVAL;
// How long between hash table histograms
const double BEES_HASH_TABLE_ANALYZE_INTERVAL = BEES_STATS_INTERVAL;