From 99fe4521014241b6ba58b986ebfe59b86e80b9f9 Mon Sep 17 00:00:00 2001 From: Zygo Blaxell Date: Fri, 13 Jan 2017 00:50:39 -0500 Subject: [PATCH] context: raise limit on the number of concurrent ioctls to cpu_cores/2 This might improve performance on systems with more than 3 CPU cores...or it might bring such a machine to its knees. TODO: find out which of those two things happens. Signed-off-by: Zygo Blaxell --- src/bees-context.cc | 7 ++++--- src/bees-hash.cc | 2 +- src/bees-roots.cc | 5 +++-- src/bees.cc | 5 ++++- src/bees.h | 2 +- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/bees-context.cc b/src/bees-context.cc index dab36aa..ffa95d1 100644 --- a/src/bees-context.cc +++ b/src/bees-context.cc @@ -1,6 +1,7 @@ #include "bees.h" #include "crucible/limits.h" +#include "crucible/process.h" #include "crucible/string.h" #include @@ -280,7 +281,7 @@ BeesContext::dedup(const BeesRangePair &brp) // To avoid hammering all the cores with long-running ioctls, // only do one dedup at any given time. BEESNOTE("Waiting to dedup " << brp); - unique_lock lock(bees_ioctl_mutex); + auto dedup_lock = bees_ioctl_lock_set.make_lock(gettid()); #endif BEESNOTE("dedup " << brp); @@ -850,7 +851,7 @@ BeesContext::scan_forward(const BeesFileRange &bfr) catch_all([&]() { uint64_t extent_bytenr = e.bytenr(); BEESNOTE("waiting for extent bytenr " << to_hex(extent_bytenr)); - decltype(m_extent_lock_set)::Lock extent_lock(m_extent_lock_set, extent_bytenr); + auto extent_lock = m_extent_lock_set.make_lock(extent_bytenr); Timer one_extent_timer; return_bfr = scan_one_extent(bfr, e); BEESCOUNTADD(scanf_extent_ms, one_extent_timer.age() * 1000); @@ -886,7 +887,7 @@ BeesContext::resolve_addr_uncached(BeesAddress addr) // To avoid hammering all the cores with long-running ioctls, // only do one resolve at any given time. BEESNOTE("waiting to resolve addr " << addr); - unique_lock lock(bees_ioctl_mutex); + auto lock = bees_ioctl_lock_set.make_lock(gettid()); Timer resolve_timer; diff --git a/src/bees-hash.cc b/src/bees-hash.cc index e2ff889..0a478d4 100644 --- a/src/bees-hash.cc +++ b/src/bees-hash.cc @@ -311,7 +311,7 @@ BeesHashTable::fetch_missing_extent(HashType hash) BEESNOTE("waiting to fetch hash extent #" << extent_number << ", " << missing_buckets << " left to fetch"); // Acquire blocking lock on this extent only - decltype(m_extent_lock_set)::Lock extent_lock(m_extent_lock_set, extent_number); + auto extent_lock = m_extent_lock_set.make_lock(extent_number); // Check missing again because someone else might have fetched this // extent for us while we didn't hold any locks diff --git a/src/bees-roots.cc b/src/bees-roots.cc index 92ad9a3..c87d7ff 100644 --- a/src/bees-roots.cc +++ b/src/bees-roots.cc @@ -1,6 +1,7 @@ #include "bees.h" #include "crucible/cache.h" +#include "crucible/process.h" #include "crucible/string.h" #include @@ -554,7 +555,7 @@ void BeesCrawl::crawl_thread() { Timer crawl_timer; - LockSet::Lock crawl_lock(m_ctx->roots()->lock_set(), m_state.m_root, false); + auto crawl_lock = m_ctx->roots()->lock_set().make_lock(m_state.m_root, false); while (!m_stopped) { BEESNOTE("waiting for crawl thread limit " << m_state); crawl_lock.lock(); @@ -646,7 +647,7 @@ BeesCrawl::fetch_extents() bool ioctl_ok = false; { BEESNOTE("waiting to search crawl sk " << static_cast(sk)); - unique_lock lock(bees_ioctl_mutex); + auto lock = bees_ioctl_lock_set.make_lock(gettid()); BEESNOTE("searching crawl sk " << static_cast(sk)); BEESTOOLONG("Searching crawl sk " << static_cast(sk)); diff --git a/src/bees.cc b/src/bees.cc index a23683c..6343e1d 100644 --- a/src/bees.cc +++ b/src/bees.cc @@ -203,7 +203,7 @@ operator<<(ostream &os, const BeesStatTmpl &bs) * Some of them consume egregious amounts of kernel CPU time and are * not interruptible, so if we have more threads than cores we will * effectively crash the kernel. */ -mutex bees_ioctl_mutex; +LockSet bees_ioctl_lock_set; template T& @@ -583,6 +583,9 @@ bees_main(int argc, const char **argv) THROW_CHECK1(invalid_argument, argc, argc >= 0); vector args(argv + 1, argv + argc); + // Set global concurrency limits - use only half the cores for ioctls + bees_ioctl_lock_set.max_size(max(1U, bees_worker_thread_count() / 2)); + // Create a context and start crawlers bool did_subscription = false; for (string arg : args) { diff --git a/src/bees.h b/src/bees.h index cb4c448..ddaeca4 100644 --- a/src/bees.h +++ b/src/bees.h @@ -830,7 +830,7 @@ string pretty(double d); extern RateLimiter bees_info_rate_limit; void bees_sync(int fd); string format_time(time_t t); -extern mutex bees_ioctl_mutex; +extern LockSet bees_ioctl_lock_set; extern unsigned bees_worker_thread_count(); #endif