From bbcfd9daa68e0ba447328964fff4c9f4b0b5e5dd Mon Sep 17 00:00:00 2001 From: Zygo Blaxell Date: Tue, 29 Nov 2022 22:44:42 -0500 Subject: [PATCH] roots: replace BEES_TRANSID_FACTOR with BEES_TRANSID_POLL_INTERVAL Restart crawl_more (and update crawl roots and flush FD caches) every time the transid changes, and only when the transid changes, but not more often than a reasonable minimum poll interval. Clean up the log message: use the proper thread name and remove the wildly inaccurate estimate of when crawl will resume. Signed-off-by: Zygo Blaxell --- src/bees-roots.cc | 14 ++++++-------- src/bees.h | 5 ++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/bees-roots.cc b/src/bees-roots.cc index 4335ecc..9ed8732 100644 --- a/src/bees-roots.cc +++ b/src/bees-roots.cc @@ -707,12 +707,10 @@ BeesRoots::crawl_roots() return true; } - BEESNOTE("Crawl done"); BEESCOUNT(crawl_done); - auto want_transid = m_transid_re.count() + m_transid_factor; - auto ran_out_time = m_crawl_timer.lap(); - BEESLOGINFO("Crawl more ran out of data after " << ran_out_time << "s, waiting about " << m_transid_re.seconds_until(want_transid) << "s for transid " << want_transid << "..."); + const auto ran_out_time = m_crawl_timer.lap(); + BEESLOGINFO("crawl_more ran out of data after " << ran_out_time << "s"); // Do not run again return false; @@ -748,7 +746,7 @@ BeesRoots::crawl_thread() // Monitor transid_max and wake up roots when it changes BEESNOTE("tracking transid"); - auto last_transid = m_transid_re.count(); + uint64_t last_transid = 0; while (!m_stop_requested) { BEESTRACE("Measure current transid"); catch_all([&]() { @@ -771,9 +769,9 @@ BeesRoots::crawl_thread() } last_transid = new_transid; - auto poll_time = m_transid_re.seconds_for(m_transid_factor); - BEESLOGDEBUG("Polling " << poll_time << "s for next " << m_transid_factor << " transid " << m_transid_re); - BEESNOTE("waiting " << poll_time << "s for next " << m_transid_factor << " transid " << m_transid_re); + const auto poll_time = max(BEES_TRANSID_POLL_INTERVAL, m_transid_re.seconds_for(1)); + BEESLOGDEBUG("Polling " << poll_time << "s for next transid " << m_transid_re); + BEESNOTE("waiting " << poll_time << "s for next transid " << m_transid_re); unique_lock lock(m_stop_mutex); if (m_stop_requested) { BEESLOGDEBUG("Stop requested in crawl thread"); diff --git a/src/bees.h b/src/bees.h index 4417517..55b2925 100644 --- a/src/bees.h +++ b/src/bees.h @@ -100,8 +100,8 @@ const size_t BEES_MAX_EXTENT_REF_COUNT = (16 * 1024 * 1024 / 24) - 1; // How long between hash table histograms const double BEES_HASH_TABLE_ANALYZE_INTERVAL = BEES_STATS_INTERVAL; -// Wait this many transids between crawls -const size_t BEES_TRANSID_FACTOR = 10; +// Wait at least this long for a new transid +const double BEES_TRANSID_POLL_INTERVAL = 30.0; // Workaround for silly dedupe / ineffective readahead behavior const size_t BEES_READAHEAD_SIZE = 1024 * 1024; @@ -543,7 +543,6 @@ class BeesRoots : public enable_shared_from_this { BeesThread m_crawl_thread; BeesThread m_writeback_thread; RateEstimator m_transid_re; - size_t m_transid_factor = BEES_TRANSID_FACTOR; bool m_workaround_btrfs_send = false; shared_ptr m_scanner;