GGLinnk/bees
1
0
Fork 0
mirror of https://github.com/Zygo/bees.git synced 2025-06-17 01:56:16 +02:00
Code Packages Activity

bees: dynamic thread pool size based on system load average

Browse Source 
Add -g / --loadavg-target parameter to track system load and add or
remove bees worker threads dynamically to keep system load close to the
loadavg target.  Thread count may vary from zero to the maximum
specified by -c or -C, and is adjusted every 5 seconds.

This is better than implementing a similar load average scheme from
outside of the process (though that is still possible) because the
in-process load tracker does not disrupt the performance timing feedback
mechanisms as a freezer cgroup or SIGSTOP would when controlling bees
from outside.  The internal load average tracker can also adjust the
number of active threads while an external tracker can only choose from
the maximum or zero.

Also fix a bug where a Task could deadlock waiting for itself to exit
if it tries to insert a new Task after the number of worker threads has
been set to zero.

Also correct usage message for --scan-mode (values are 0..2) since
we are touching adjacent lines anyway.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
This commit is contained in:
Zygo Blaxell
2018-07-08 21:20:15 -04:00
parent 96eb100ded
commit e66086516f
7 changed files with 203 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/bees.cc
View File

@ -45,7 +45,8 @@ do_cmd_help(char *argv[])
"\t-h, --help\t\tShow this help\n"
"\t-c, --thread-count\tWorker thread count (default CPU count * factor)\n"
"\t-C, --thread-factor\tWorker thread factor (default " << BEES_DEFAULT_THREAD_FACTOR << ")\n"
"\t-m, --scan-mode\t\tScanning mode (0..1, default 0)\n"
"\t-g, --loadavg-target\t\tTarget load average for worker threads (default is no target)\n"
"\t-m, --scan-mode\t\tScanning mode (0..2, default 0)\n"
"\t-t, --timestamps\tShow timestamps in log output (default)\n"
"\t-T, --no-timestamps\tOmit timestamps in log output\n"
"\t-p, --absolute-paths\tShow absolute paths (default)\n"
@ -652,24 +653,26 @@ bees_main(int argc, char *argv[])
bool chatter_prefix_timestamp = true;
double thread_factor = 0;
unsigned thread_count = 0;
double load_target = 0;
// Parse options
int c;
while (1) {
int option_index = 0;
static struct option long_options[] = {
static const struct option long_options[] = {
{ "thread-factor", required_argument, NULL, 'C' },
{ "strip-paths", no_argument, NULL, 'P' },
{ "no-timestamps", no_argument, NULL, 'T' },
{ "thread-count", required_argument, NULL, 'c' },
{ "help", no_argument, NULL, 'h' },
{ "loadavg-target", required_argument, NULL, 'g' },
{ "scan-mode", required_argument, NULL, 'm' },
{ "absolute-paths", no_argument, NULL, 'p' },
{ "timestamps", no_argument, NULL, 't' },
{ "verbose", required_argument, NULL, 'v' },
};
c = getopt_long(argc, argv, "C:PTc:hm:ptv:", long_options, &option_index);
c = getopt_long(argc, argv, "C:PTc:hg:m:ptv:", long_options, &option_index);
if (-1 == c) {
break;
}
@ -688,6 +691,9 @@ bees_main(int argc, char *argv[])
case 'c':
thread_count = stoul(optarg);
break;
case 'g':
load_target = stod(optarg);
break;
case 'm':
BeesRoots::set_scan_mode(static_cast<BeesRoots::ScanMode>(stoul(optarg)));
break;
@ -700,7 +706,8 @@ bees_main(int argc, char *argv[])
case 'v':
{
int new_log_level = stoul(optarg);
THROW_CHECK1(out_of_range, new_log_level, new_log_level >= 0 || new_log_level <= 8);
THROW_CHECK1(out_of_range, new_log_level, new_log_level <= 8);
THROW_CHECK1(out_of_range, new_log_level, new_log_level >= 0);
bees_log_level = new_log_level;
BEESLOGNOTICE("log level set to " << bees_log_level);
}
@ -739,6 +746,12 @@ bees_main(int argc, char *argv[])
thread_count = max(1U, static_cast<unsigned>(ceil(thread::hardware_concurrency() * thread_factor)));
}
if (load_target != 0) {
BEESLOGNOTICE("setting load average target to " << load_target);
}
TaskMaster::set_loadavg_target(load_target);
BEESLOGNOTICE("setting worker thread pool maximum size to " << load_target);
TaskMaster::set_thread_count(thread_count);
// Create a context and start crawlers