1
0
mirror of https://github.com/Zygo/bees.git synced 2025-08-02 13:53:28 +02:00

9 Commits

Author SHA1 Message Date
Zygo Blaxell
a466ccf2f1 build: include localconf everywhere
Overriding makeflags did not work from localconf in the src, lib, or
test directories.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:52:45 -05:00
Zygo Blaxell
ba04fe1349 roots: make it build with clang
Remove an unnecessary cast that was breaking namespace lookup for clang.

Closes: #159

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:48 -05:00
Zygo Blaxell
830df63d4c chatter: make it build with clang
Silence the unused variable warning.  The compiler is correct, but we
may implement line-level debug at some point in the future, so we
want to keep the member and parameters.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:42 -05:00
Zygo Blaxell
20c9d2ff6a clang: fix struct/class declaration/definition mismatches
clang does not like a defined class to be declared as a struct.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:40 -05:00
Zygo Blaxell
7bbb4d14cb bees context: make it build with clang
Remove unused function getenv_or_die.  All of our environment variable
parameters are optional or have default values.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:38 -05:00
Zygo Blaxell
363c45b8cd bees: make it build with clang
Remove unused "addr check" functions.  We have ranged_cast for detecting
overflow bits.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:35 -05:00
Zygo Blaxell
4ec2b8ac16 task: make it build with clang
Remove unused closure captures.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:30 -05:00
Zygo Blaxell
26d31225fa extentwalker: make it build with clang
Remove unused MAX_OFFSET.

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2021-02-08 12:49:27 -05:00
Zygo Blaxell
21ae937201 roots: reimplement transid_max_nocache using extent tree root
Commit 9a97699dd9 upstream.

This commit accidentally fixes a bug where we call btrfs_get_root_transid
with BTRFS_FS_TREE_OBJECTID instead of m_ctx->root_fd().  This leads
to storms of messages like this:

	crawl_transid[5334]: exception type std::system_error: BTRFS_IOC_INO_LOOKUP: rv = readlink(path.c_str(), buf, size + 1): No such file or directory at fs.cc:430: No such file or directory

The code was working before because BTRFS_FS_TREE_OBJECTID == 5.
bees is constantly opening files, and the Linux kernel fills in unused
fd numbers starting from 0, so it's quite likely that the process has fd
5 open to some existing file somewhere on the target btrfs filesystem
most of the time.  If fd 5 is closed, or if it is open to an orphan
file (one without an existing name), the ioctl in btrfs_get_root_id
(called by btrfs_get_root_transid) will fail and throw and exception.
The exception breaks out of the crawl_transid task before it can do any
scanning work, so bees will stop deduping until FD 5 is open again with
an existing file.  This can only happen if other threads are opening
files, so if bees is idle at the instant when this failure occurs,
it will never dedupe again until the process is terminated and restarted.

The remainder is the original commit message:

ROOT_TREE contains the ROOT_ITEM for EXTENT_TREE.  Every modification
(that we care about) to a btrfs must go through EXTENT_TREE, and must
modify the page in ROOT_TREE pointing to the root of EXTENT_TREE...
which makes that a very good source for the filesystem transid.

Remove the loop and the root lookups, and just look at one item for
max_transid.

Also note that every caller of transid_max_nocache() immediately
feeds the return value to m_transid_re.update(), so don't do that
inside transid_max_nocache().

Signed-off-by: Zygo Blaxell <bees@furryterror.org>
2020-12-23 17:27:44 -05:00
12 changed files with 12 additions and 44 deletions

View File

@@ -13,7 +13,7 @@ namespace crucible {
template <class T>
class ProgressTracker {
class ProgressTrackerState;
struct ProgressTrackerState;
class ProgressHolderState;
public:
using value_type = T;

View File

@@ -20,6 +20,7 @@ CRUCIBLE_OBJS = \
uuid.o \
include ../makeflags
-include ../localconf
include ../Defines.mk
configure.h: configure.h.in

View File

@@ -124,6 +124,7 @@ namespace crucible {
} else if (!chatter_names->empty()) {
cerr << "CRUCIBLE_CHATTER does not list '" << m_file << "' or '" << m_pretty_function << "'" << endl;
}
(void)m_line; // not implemented yet
// cerr << "ChatterBox " << reinterpret_cast<void*>(this) << " constructed" << endl;
}

View File

@@ -14,7 +14,6 @@ namespace crucible {
// fm_start, fm_length, fm_flags, m_extents
// fe_logical, fe_physical, fe_length, fe_flags
static const off_t MAX_OFFSET = numeric_limits<off_t>::max();
static const off_t FIEMAP_BLOCK_SIZE = 4096;
static bool __ew_do_log = getenv("EXTENTWALKER_DEBUG");

View File

@@ -110,9 +110,6 @@ namespace crucible {
}
}
template<>
struct ResourceHandle<Process::id, Process>;
pid_t
gettid()
{

View File

@@ -6,6 +6,7 @@ PROGRAMS = \
all: $(PROGRAMS)
include ../makeflags
-include ../localconf
LIBS = -lcrucible -lpthread
LDFLAGS = -L../lib

View File

@@ -11,17 +11,6 @@
using namespace crucible;
using namespace std;
static inline
const char *
getenv_or_die(const char *name)
{
const char *rv = getenv(name);
if (!rv) {
THROW_ERROR(runtime_error, "Environment variable " << name << " not defined");
}
return rv;
}
BeesFdCache::BeesFdCache()
{
m_root_cache.func([&](shared_ptr<BeesContext> ctx, uint64_t root) -> Fd {

View File

@@ -219,7 +219,6 @@ BeesRoots::transid_max_nocache()
while (true) {
sk.nr_items = 1024;
BEESTRACE("transid_max search sk " << sk);
sk.do_ioctl(m_ctx->root_fd());
if (sk.m_result.empty()) {
@@ -416,15 +415,13 @@ BeesRoots::crawl_thread()
BEESNOTE("tracking transid");
auto last_count = m_transid_re.count();
while (true) {
BEESTRACE("Measure current transid");
// Measure current transid
catch_all([&]() {
BEESTRACE("calling transid_max_nocache");
m_transid_re.update(transid_max_nocache());
});
BEESTRACE("Make sure we have a full complement of crawlers");
// Make sure we have a full complement of crawlers
catch_all([&]() {
BEESTRACE("calling insert_new_crawl");
insert_new_crawl();
});
@@ -492,24 +489,19 @@ BeesRoots::insert_new_crawl()
unique_lock<mutex> lock(m_mutex);
set<uint64_t> excess_roots;
for (auto i : m_root_crawl_map) {
BEESTRACE("excess_roots.insert(" << i.first << ")");
excess_roots.insert(i.first);
}
lock.unlock();
while (new_bcs.m_root) {
BEESTRACE("excess_roots.erase(" << new_bcs.m_root << ")");
excess_roots.erase(new_bcs.m_root);
BEESTRACE("insert_root(" << new_bcs << ")");
insert_root(new_bcs);
BEESCOUNT(crawl_create);
BEESTRACE("next_root(" << new_bcs.m_root << ")");
new_bcs.m_root = next_root(new_bcs.m_root);
}
for (auto i : excess_roots) {
new_bcs.m_root = i;
BEESTRACE("crawl_state_erase(" << new_bcs << ")");
crawl_state_erase(new_bcs);
}
}
@@ -981,7 +973,7 @@ BeesCrawl::fetch_extents()
// Lock in the old state
set_state(old_state);
BEESTRACE("Searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));
BEESTRACE("Searching crawl sk " << sk);
bool ioctl_ok = false;
{
BEESNOTE("searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));

View File

@@ -204,20 +204,6 @@ BeesNote::get_status()
// static inline helpers ----------------------------------------
static inline
bool
bees_addr_check(uint64_t v)
{
return !(v & (1ULL << 63));
}
static inline
bool
bees_addr_check(int64_t v)
{
return !(v & (1ULL << 63));
}
string
pretty(double d)
{

View File

@@ -173,7 +173,7 @@ public:
T at(string idx) const;
friend ostream& operator<< <>(ostream &os, const BeesStatTmpl<T> &bs);
friend class BeesStats;
friend struct BeesStats;
};
using BeesRates = BeesStatTmpl<double>;

View File

@@ -14,6 +14,7 @@ test: $(PROGRAMS:%=%.txt) Makefile
FORCE:
include ../makeflags
-include ../localconf
LIBS = -lcrucible -lpthread
LDFLAGS = -L../lib -Wl,-rpath=$(shell realpath ../lib)

View File

@@ -99,7 +99,7 @@ test_barrier(size_t count)
oss << "task #" << c;
Task t(
oss.str(),
[c, &task_done, &mtx, &cv, bl]() mutable {
[c, &task_done, &mtx, bl]() mutable {
// cerr << "Task #" << c << endl;
unique_lock<mutex> lock(mtx);
task_done.at(c) = true;
@@ -166,8 +166,9 @@ test_exclusion(size_t count)
oss << "task #" << c;
Task t(
oss.str(),
[c, &only_one, &mtx, &excl, bl]() mutable {
[c, &only_one, &excl, bl]() mutable {
// cerr << "Task #" << c << endl;
(void)c;
auto lock = excl.try_lock();
if (!lock) {
excl.insert_task(Task::current_task());