fanotify-watch: Not really part of Bees, but a useful tool nonetheless

Makefile

@@ -1,36 +1,19 @@
-PREFIX ?= /
+default install all: lib src test README.html
 
-default all: lib src test README.html
+clean:
-
-clean: ## Cleanup
 	git clean -dfx
 
 .PHONY: lib src
 
-lib: ## Build libs
+lib:
 	$(MAKE) -C lib
 
-src: ## Build bins
 src: lib
 	$(MAKE) -C src
 
-test: ## Run tests
 test: lib src
 	$(MAKE) -C test
 
 README.html: README.md
 	markdown README.md > README.html.new
 	mv -f README.html.new README.html
-
-install: ## Install bees + libs
-install: lib src test
-	install -Dm644 lib/libcrucible.so $(PREFIX)/usr/lib/libcrucible.so
-	install -Dm755 bin/bees $(PREFIX)/usr/bin/bees
-
-install_scripts: ## Install scipts
-	install -Dm755 scripts/beesd $(PREFIX)/usr/bin/beesd
-	install -Dm644 scripts/beesd.conf.sample $(PREFIX)/etc/bees/beesd.conf.sample
-	install -Dm644 scripts/beesd@.service $(PREFIX)/lib/systemd/system/beesd@.service
-
-help: ## Show help
-	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##/\t/'

README.md

@@ -1,52 +1,30 @@
 BEES
 ====
 
-Best-Effort Extent-Same, a btrfs dedup agent.
+Best-Effort Extent-Same, a btrfs deduplication daemon.
 
 About Bees
 ----------
 
-Bees is a block-oriented userspace dedup agent designed to avoid
+Bees is a daemon designed to run continuously on live file servers.
-scalability problems on large filesystems.
+Bees scans and deduplicates whole filesystems in a single pass instead
+of separate scan and dedup phases.  RAM usage does _not_ depend on
+unique data size or the number of input files.  Hash tables and scan
+progress are stored persistently so the daemon can resume after a reboot.
+Bees uses the Linux kernel's `dedupe_file_range` feature to ensure data
+is handled safely even if other applications concurrently modify it.
 
-Bees is designed to degrade gracefully when underprovisioned with RAM.
+Bees is intentionally btrfs-specific for performance and capability.
-Bees does not use more RAM or storage as filesystem data size increases.
+Bees uses the btrfs `SEARCH_V2` ioctl to scan for new data without the
-The dedup hash table size is fixed at creation time and does not change.
+overhead of repeatedly walking filesystem trees with the POSIX API.
-The effective dedup block size is dynamic and adjusts automatically to
+Bees uses `LOGICAL_INO` and `INO_PATHS` to leverage btrfs's existing
-fit the hash table into the configured RAM limit.  Hash table overflow
+metadata instead of building its own redundant data structures.
-is not implemented to eliminate the IO overhead of hash table overflow.
+Bees can cope with Btrfs filesystem compression.  Bees can reassemble
-Hash table entries are only 16 bytes per dedup block to keep the average
+Btrfs extents to deduplicate extents that contain a mix of duplicate
-dedup block size small.
+and unique data blocks.
 
-Bees does not require alignment between dedup blocks or extent boundaries
+Bees includes a number of workarounds for Btrfs kernel bugs to (try to)
-(i.e. it can handle any multiple-of-4K offset between dup block pairs).
+avoid ruining your day.  You're welcome.
-Bees rearranges blocks into shared and unique extents if required to
-work within current btrfs kernel dedup limitations.
-
-Bees can dedup any combination of compressed and uncompressed extents.
-
-Bees operates in a single pass which removes duplicate extents immediately
-during scan.  There are no separate scanning and dedup phases.
-
-Bees uses only data-safe btrfs kernel operations, so it can dedup live
-data (e.g. build servers, sqlite databases, VM disk images).  It does
-not modify file attributes or timestamps.
-
-Bees does not store any information about filesystem structure, so it is
-not affected by the number or size of files (except to the extent that
-these cause performance problems for btrfs in general).  It retrieves such
-information on demand through btrfs SEARCH_V2 and LOGICAL_INO ioctls.
-This eliminates the storage required to maintain the equivalents of
-these functions in userspace.  It's also why bees has no XFS support.
-
-Bees is a daemon designed to run continuously and maintain its state
-across crahes and reboots.  Bees uses checkpoints for persistence to
-eliminate the IO overhead of a transactional data store.  On restart,
-bees will dedup any data that was added to the filesystem since the
-last checkpoint.
-
-Bees is used to dedup filesystems ranging in size from 16GB to 35TB, with
-hash tables ranging in size from 128MB to 11GB.
 
 How Bees Works
 --------------
@@ -100,9 +78,11 @@ and some metadata bits).  Each entry represents a minimum of 4K on disk.
         1TB                16MB               1024K
        64TB                 1GB               1024K
 
-To change the size of the hash table, use 'truncate' to change the hash
+It is possible to resize the hash table by changing the size of
-table size, delete `beescrawl.dat` so that bees will start over with a
+`beeshash.dat` (e.g. with `truncate`) and restarting `bees`.  This
-fresh full-filesystem rescan, and restart `bees'.
+does not preserve all the existing hash table entries, but it does
+preserve more than zero of them--especially if the old and new sizes
+are a power-of-two multiple of each other.
 
 Things You Might Expect That Bees Doesn't Have
 ----------------------------------------------
@@ -149,9 +129,6 @@ blocks, but has no defragmentation capability yet.  When possible, Bees
 will attempt to work with existing extent boundaries, but it will not
 aggregate blocks together from multiple extents to create larger ones.
 
-* It is possible to resize the hash table without starting over with
-a new full-filesystem scan; however, this has not been implemented yet.
-
 Good Btrfs Feature Interactions
 -------------------------------
 
@@ -167,17 +144,19 @@ Bees has been tested in combination with the following:
 * IO errors during dedup (read errors will throw exceptions, Bees will catch them and skip over the affected extent)
 * Filesystems mounted *with* the flushoncommit option
 * 4K filesystem data block size / clone alignment
-* 64-bit and 32-bit host CPUs (amd64, x86, arm)
+* 64-bit CPUs (amd64)
 * Large (>16M) extents
 * Huge files (>1TB--although Btrfs performance on such files isn't great in general)
 * filesystems up to 25T bytes, 100M+ files
 
+
 Bad Btrfs Feature Interactions
 ------------------------------
 
 Bees has not been tested with the following, and undesirable interactions may occur:
 
 * Non-4K filesystem data block size (should work if recompiled)
+* 32-bit CPUs (x86, arm)
 * Non-equal hash (SUM) and filesystem data block (CLONE) sizes (probably never will work)
 * btrfs read-only snapshots (never tested, probably wouldn't work well)
 * btrfs send/receive (receive is probably OK, but send requires RO snapshots.  See above)
@@ -221,32 +200,25 @@ Other Caveats
 A Brief List Of Btrfs Kernel Bugs
 ---------------------------------
 
-Missing features (usually not available in older LTS kernels):
+Fixed bugs:
 
 * 3.13: `FILE_EXTENT_SAME` ioctl added.  No way to reliably dedup with
   concurrent modifications before this.
 * 3.16: `SEARCH_V2` ioctl added.  Bees could use `SEARCH` instead.
 * 4.2: `FILE_EXTENT_SAME` no longer updates mtime, can be used at EOF.
-
+  Kernel deadlock bugs fixed.
-Bug fixes (sometimes included in older LTS kernels):
-
-* 4.5: hang in the `INO_PATHS` ioctl used by Bees.
-* 4.5: use-after-free in the `FILE_EXTENT_SAME` ioctl used by Bees.
 * 4.7: *slow backref* bug no longer triggers a softlockup panic.  It still
   too long to resolve a block address to a root/inode/offset triple.
-* 4.10-rc1: reduced CPU time cost of the LOGICAL_INO ioctl and dedup
-  backref processing in general.
 
 Unfixed kernel bugs (as of 4.5.7) with workarounds in Bees:
 
-* *slow backrefs* (aka toxic extents): If the number of references to a
+* *slow backref*: If the number of references to a single shared extent
-  single shared extent within a single file grows above a few thousand,
+  within a single file grows above a few thousand, the kernel consumes CPU
-  the kernel consumes CPU for minutes at a time while holding various
+  for up to 40 uninterruptible minutes while holding various locks that
-  locks that block access to the filesystem.  Bees avoids this bug by
+  block access to the filesystem.  Bees avoids this bug by measuring the
-  measuring the time the kernel spends performing certain operations
+  time the kernel spends performing certain operations and permanently
-  and permanently blacklisting any extent or hash where the kernel
+  blacklisting any extent or hash where the kernel starts to get slow.
-  starts to get slow.  Inside Bees, such blocks are marked as 'toxic'
+  Inside Bees, such blocks are marked as 'toxic' hash/block addresses.
-  hash/block addresses.
 
 * `LOGICAL_INO` output is arbitrarily limited to 2730 references
   even if more buffer space is provided for results.  Once this number
@@ -271,7 +243,7 @@ Unfixed kernel bugs (as of 4.5.7) with workarounds in Bees:
   precisely the specified range of offending fragmented blocks.
 
 * When writing BeesStringFile, a crash can cause the directory entry
-  `beescrawl.dat.tmp` to exist without a corresponding inode.
+  `beescrawl.UUID.dat.tmp` to exist without a corresponding inode.
   This directory entry cannot be renamed or removed; however, it does
   not prevent the creation of a second directory entry with the same
   name that functions normally, so it doesn't prevent Bees operation.
@@ -279,13 +251,10 @@ Unfixed kernel bugs (as of 4.5.7) with workarounds in Bees:
   The orphan directory entry can be removed by deleting its subvol,
   so place BEESHOME on a separate subvol so you can delete these orphan
   directory entries when they occur (or use btrfs zero-log before mounting
-  the filesystem after a crash).  Alternatively, place BEESHOME on a
+  the filesystem after a crash).
-  non-btrfs filesystem.
 
-* If the `fsync()` in `BeesTempFile::make_copy` is removed, the filesystem
+* If the fsync() BeesTempFile::make_copy is removed, the filesystem
-  hangs within a few hours, requiring a reboot to recover.  On the other
+  hangs within a few hours, requiring a reboot to recover.
-  hand, there may be net performance benefits to calling `fsync()` before
-  or after each dedup.  This needs further investigation.
 
 Not really a bug, but a gotcha nonetheless:
 
@@ -301,10 +270,9 @@ Not really a bug, but a gotcha nonetheless:
 Requirements
 ------------
 
-* C++11 compiler (tested with GCC 4.9 and 6.2.0)
+* C++11 compiler (tested with GCC 4.9)
 
-  Sorry.  I really like closures and shared_ptr, so support
+  Sorry.  I really like closures.
-  for earlier compiler versions is unlikely.
 
 * btrfs-progs (tested with 4.1..4.7)
 
@@ -313,14 +281,22 @@ Requirements
 
 * libuuid-dev
 
-  This library is only required for a feature that was removed after v0.1.
+  TODO: remove the one function used from this library.
-  The lingering support code can be removed.
+  It supports a feature Bees no longer implements.
 
-* Linux kernel 4.4.3 or later
+* Linux kernel 4.2 or later
 
   Don't bother trying to make Bees work with older kernels.
   It won't end well.
 
+* 64-bit host and target CPU
+
+  This code has never been tested on a 32-bit target CPU.
+
+  A 64-bit host CPU may be required for the self-tests.
+  Some of the ioctls don't work properly with a 64-bit
+  kernel and 32-bit userspace.
+
 Build
 -----
 
@@ -344,49 +320,17 @@ of 16M).  This example creates a 1GB hash table:
         truncate -s 1g "$BEESHOME/beeshash.dat"
         chmod 700 "$BEESHOME/beeshash.dat"
 
-bees can only process the root subvol of a btrfs (seriously--if the
-argument is not the root subvol directory, Bees will just throw an
-exception and stop).
-
-Use a bind mount, and let only bees access it:
-
-	UUID=3399e413-695a-4b0b-9384-1b0ef8f6c4cd
-	mkdir -p /var/lib/bees/$UUID
-	mount /dev/disk/by-uuid/$UUID /var/lib/bees/$UUID -osubvol=/
-
-If you don't set BEESHOME, the path ".beeshome" will be used relative
-to the root subvol of the filesystem.  For example:
-
-	btrfs sub create /var/lib/bees/$UUID/.beeshome
-	truncate -s 1g /var/lib/bees/$UUID/.beeshome/beeshash.dat
-	chmod 700 /var/lib/bees/$UUID/.beeshome/beeshash.dat
-
-You can use any relative path in BEESHOME.  The path will be taken
-relative to the root of the deduped filesystem (in other words it can
-be the name of a subvol):
-
-	export BEESHOME=@my-beeshome
-	btrfs sub create /var/lib/bees/$UUID/$BEESHOME
-	truncate -s 1g /var/lib/bees/$UUID/$BEESHOME/beeshash.dat
-	chmod 700 /var/lib/bees/$UUID/$BEESHOME/beeshash.dat
-
 Configuration
 -------------
 
 The only runtime configurable options are environment variables:
 
 * BEESHOME: Directory containing Bees state files:
- * beeshash.dat  | persistent hash table.  Must be a multiple of 16M.
+ * beeshash.dat         | persistent hash table (must be a multiple of 16M)
-                   This contains 16-byte records:  8 bytes for CRC64,
+ * beescrawl.`UUID`.dat | state of SEARCH_V2 crawlers
-                   8 bytes for physical address and some metadata bits.
+ * beesstats.txt        | statistics and performance counters
- * beescrawl.dat | state of SEARCH_V2 crawlers.  ASCII text.
+* BEESSTATS: File containing a snapshot of current Bees state (performance
- * beesstats.txt | statistics and performance counters.  ASCII text.
+  counters and current status of each thread).
-* BEESSTATUS: File containing a snapshot of current Bees state:  performance
-  counters and current status of each thread.  The file is meant to be
-  human readable, but understanding it probably requires reading the source.
-  You can watch bees run in realtime with a command like:
-
-	watch -n1 cat $BEESSTATUS
 
 Other options (e.g. interval between filesystem crawls) can be configured
 in src/bees.h.
@@ -394,27 +338,39 @@ in src/bees.h.
 Running
 -------
 
-Reduce CPU and IO priority to be kinder to other applications sharing
+We created this directory in the previous section:
-this host (or raise them for more aggressive disk space recovery).  If you
+
-use cgroups, put `bees` in its own cgroup, then reduce the `blkio.weight`
+        export BEESHOME=/some/path
-and `cpu.shares` parameters.  You can also use `schedtool` and `ionice`
+
-in the shell script that launches `bees`:
+Use a tmpfs for BEESSTATUS, it updates once per second:
+
+        export BEESSTATUS=/run/bees.status
+
+bees can only process the root subvol of a btrfs (seriously--if the
+argument is not the root subvol directory, Bees will just throw an
+exception and stop).
+
+Use a bind mount, and let only bees access it:
+
+        mount -osubvol=/ /dev/<your-filesystem> /var/lib/bees/root
+
+Reduce CPU and IO priority to be kinder to other applications
+sharing this host (or raise them for more aggressive disk space
+recovery).  If you use cgroups, put `bees` in its own cgroup, then reduce
+the `blkio.weight` and `cpu.shares` parameters.  You can also use
+`schedtool` and `ionice` in the shell script that launches `bees`:
 
         schedtool -D -n20 $$
         ionice -c3 -p $$
 
 Let the bees fly:
 
-	for fs in /var/lib/bees/*-*-*-*-*/; do
+        bees /var/lib/bees/root >> /var/log/bees.log 2>&1
-		bees "$fs" >> "$fs/.beeshome/bees.log" 2>&1 &
-	done
 
 You'll probably want to arrange for /var/log/bees.log to be rotated
 periodically.  You may also want to set umask to 077 to prevent disclosure
 of information about the contents of the filesystem through the log file.
 
-There are also some shell wrappers in the `scripts/` directory.
-
 
 Bug Reports and Contributions
 -----------------------------
@@ -430,6 +386,6 @@ You can also use Github:
 Copyright & License
 ===================
 
-Copyright 2015-2017 Zygo Blaxell <bees@furryterror.org>.
+Copyright 2015-2016 Zygo Blaxell <bees@furryterror.org>.
 
 GPL (version 3 or later).

include/crucible/bool.h

@@ -0,0 +1,13 @@
+#ifndef CRUCIBLE_BOOL_H
+#define CRUCIBLE_BOOL_H
+
+namespace crucible {
+	struct DefaultBool {
+		bool m_b;
+		DefaultBool(bool init = false) : m_b(init) {}
+		operator bool() const { return m_b; }
+		bool &operator=(const bool &that) { return m_b = that; }
+	};
+}
+
+#endif // CRUCIBLE_BOOL_H

include/crucible/btrfs.h

@@ -130,7 +130,7 @@
 	};
 
 #endif
-
+	 
 #ifndef BTRFS_IOC_CLONE_RANGE
 
 	struct btrfs_ioctl_clone_range_args {

include/crucible/cache.h

@@ -8,7 +8,6 @@
 #include <map>
 #include <mutex>
 #include <tuple>
-#include <vector>
 
 namespace crucible {
 	using namespace std;
@@ -18,7 +17,7 @@ namespace crucible {
 	public:
 		using Key = tuple<Arguments...>;
 		using Func = function<Return(Arguments...)>;
-		using Time = size_t;
+		using Time = unsigned;
 		using Value = pair<Time, Return>;
 	private:
 		Func		m_fn;
@@ -28,7 +27,7 @@ namespace crucible {
 		size_t		m_max_size;
 		mutex		m_mutex;
 
-		bool check_overflow();
+		void check_overflow();
 	public:
 		LRUCache(Func f = Func(), size_t max_size = 100);
 
@@ -52,24 +51,21 @@ namespace crucible {
 	}
 
 	template <class Return, class... Arguments>
-	bool
+	void
 	LRUCache<Return, Arguments...>::check_overflow()
 	{
-		if (m_map.size() <= m_max_size) {
+		if (m_map.size() <= m_max_size) return;
-			return false;
+		vector<pair<Key, Time>> map_contents;
-		}
+		map_contents.reserve(m_map.size());
-		vector<pair<Key, Time>> key_times;
-		key_times.reserve(m_map.size());
 		for (auto i : m_map) {
-			key_times.push_back(make_pair(i.first, i.second.first));
+			map_contents.push_back(make_pair(i.first, i.second.first));
 		}
-		sort(key_times.begin(), key_times.end(), [](const pair<Key, Time> &a, const pair<Key, Time> &b) {
+		sort(map_contents.begin(), map_contents.end(), [](const pair<Key, Time> &a, const pair<Key, Time> &b) {
 			return a.second < b.second;
 		});
-		for (size_t i = 0; i < key_times.size() / 2; ++i) {
+		for (size_t i = 0; i < map_contents.size() / 2; ++i) {
-			m_map.erase(key_times[i].first);
+			m_map.erase(map_contents[i].first);
 		}
-		return true;
 	}
 
 	template <class Return, class... Arguments>
@@ -144,14 +140,9 @@ namespace crucible {
 				// We hold a lock on this key so we are the ones to insert it
 				THROW_CHECK0(runtime_error, inserted);
 
-				// Release key lock, keep the cache lock
+				// Release key lock and clean out overflow
 				key_lock.unlock();
-
+				check_overflow();
-				// Check to see if we have too many items and reduce if so.
-				if (check_overflow()) {
-					// Reset iterator
-					found = m_map.find(k);
-				}
 			}
 		}
 
@@ -162,9 +153,7 @@ namespace crucible {
 		if (!inserted) {
 			found->second.first = m_ctr++;
 		}
-		// Make copy before releasing lock
+		return found->second.second;
-		auto rv = found->second.second;
-		return rv;
 	}
 
 	template<class Return, class... Arguments>
@@ -215,12 +204,7 @@ namespace crucible {
 
 				// Release key lock and clean out overflow
 				key_lock.unlock();
-
+				check_overflow();
-				// Check to see if we have too many items and reduce if so.
-				if (check_overflow()) {
-					// Reset iterator
-					found = m_map.find(k);
-				}
 			}
 		}
 

include/crucible/chatter.h

@@ -86,6 +86,16 @@ namespace crucible {
 		}
 	};
 
+	template <>
+	struct ChatterTraits<ostream &> {
+		Chatter &
+		operator()(Chatter &c, ostream & arg)
+		{
+			c.get_os() << arg;
+			return c;
+		}
+	};
+
 	class ChatterBox {
 		string m_file;
 		int m_line;

include/crucible/crc64.h

@@ -3,11 +3,11 @@
 
 #include <cstdint>
 #include <cstdlib>
-#include <cstring>
 
 namespace crucible {
 	namespace Digest {
 		namespace CRC {
+			uint64_t crc64(const char *s);
 			uint64_t crc64(const void *p, size_t len);
 		};
 	};

include/crucible/execpipe.h

@@ -0,0 +1,28 @@
+#ifndef CRUCIBLE_EXECPIPE_H
+#define CRUCIBLE_EXECPIPE_H
+
+#include "crucible/fd.h"
+
+#include <functional>
+#include <limits>
+#include <string>
+
+namespace crucible {
+	using namespace std;
+
+	void redirect_stdin(const Fd &child_fd);
+	void redirect_stdin_stdout(const Fd &child_fd);
+	void redirect_stdin_stdout_stderr(const Fd &child_fd);
+	void redirect_stdout(const Fd &child_fd);
+	void redirect_stdout_stderr(const Fd &child_fd);
+
+	// Open a pipe (actually socketpair) to child process, then execute code in that process.
+	// e.g. popen([] () { system("echo Hello, World!"); });
+	// Forked process will exit when function returns.
+	Fd popen(function<int()> f, function<void(const Fd &child_fd)> import_fd_fn = redirect_stdin_stdout);
+
+	// Read all the data from fd into a string
+        string read_all(Fd fd, size_t max_bytes = numeric_limits<size_t>::max(), size_t chunk_bytes = 4096);
+};
+
+#endif // CRUCIBLE_EXECPIPE_H

include/crucible/extentwalker.h

@@ -8,15 +8,15 @@ namespace crucible {
 
 	// FIXME:  ExtentCursor is probably a better name
 	struct Extent {
-		off_t		m_begin = 0;
+		off_t		m_begin;
-		off_t		m_end = 0;
+		off_t		m_end;
-		uint64_t	m_physical = 0;
+		uint64_t	m_physical;
-		uint64_t	m_flags = 0;
+		uint64_t	m_flags;
 
 		// Btrfs extent reference details
-		off_t		m_physical_len = 0;
+		off_t		m_physical_len;
-		off_t		m_logical_len = 0;
+		off_t		m_logical_len;
-		off_t		m_offset = 0;
+		off_t		m_offset;
 
 		// fiemap flags are uint32_t, so bits 32..63 are OK for us
 
@@ -38,12 +38,10 @@ namespace crucible {
 		off_t physical_len() const { return m_physical_len; }
 		off_t logical_len() const { return m_logical_len; }
 		off_t offset() const { return m_offset; }
-		bool compressed() const;
-		uint64_t bytenr() const;
 		bool operator==(const Extent &that) const;
 		bool operator!=(const Extent &that) const { return !(*this == that); }
 
-		Extent() = default;
+		Extent();
 		Extent(const Extent &e) = default;
 	};
 

include/crucible/fd.h

@@ -70,11 +70,10 @@ namespace crucible {
 	string mmap_flags_ntoa(int flags);
 
 	// Unlink, rename
+	void unlink_or_die(const string &file);
 	void rename_or_die(const string &from, const string &to);
 	void renameat_or_die(int fromfd, const string &frompath, int tofd, const string &topath);
 
-	void ftruncate_or_die(int fd, off_t size);
-
 	// Read or write structs:
 	// There is a template specialization to read or write strings
 	// Three-arg version of read_or_die/write_or_die throws an error on incomplete read/writes
@@ -121,9 +120,6 @@ namespace crucible {
 	template<> void pread_or_die<string>(int fd, string& str, off_t offset);
 	template<> void pread_or_die<vector<char>>(int fd, vector<char>& str, off_t offset);
 	template<> void pread_or_die<vector<uint8_t>>(int fd, vector<uint8_t>& str, off_t offset);
-	template<> void pwrite_or_die<string>(int fd, const string& str, off_t offset);
-	template<> void pwrite_or_die<vector<char>>(int fd, const vector<char>& str, off_t offset);
-	template<> void pwrite_or_die<vector<uint8_t>>(int fd, const vector<uint8_t>& str, off_t offset);
 
 	// A different approach to reading a simple string
 	string read_string(int fd, size_t size);

include/crucible/fs.h

@@ -13,7 +13,6 @@
 
 #include <cstdint>
 #include <iosfwd>
-#include <set>
 #include <vector>
 
 #include <fcntl.h>
@@ -151,14 +150,13 @@ namespace crucible {
 		BtrfsIoctlSearchHeader();
 		vector<char> m_data;
 		size_t set_data(const vector<char> &v, size_t offset);
-		bool operator<(const BtrfsIoctlSearchHeader &that) const;
 	};
 
 	ostream & operator<<(ostream &os, const btrfs_ioctl_search_header &hdr);
 	ostream & operator<<(ostream &os, const BtrfsIoctlSearchHeader &hdr);
 
 	struct BtrfsIoctlSearchKey : public btrfs_ioctl_search_key {
-		BtrfsIoctlSearchKey(size_t buf_size = 4096);
+		BtrfsIoctlSearchKey(size_t buf_size = 1024 * 1024);
 		virtual bool do_ioctl_nothrow(int fd);
 		virtual void do_ioctl(int fd);
 
@@ -166,15 +164,14 @@ namespace crucible {
 		void next_min(const BtrfsIoctlSearchHeader& ref);
 
 		size_t m_buf_size;
-		set<BtrfsIoctlSearchHeader> m_result;
+		vector<BtrfsIoctlSearchHeader> m_result;
-
 	};
 
 	ostream & operator<<(ostream &os, const btrfs_ioctl_search_key &key);
 	ostream & operator<<(ostream &os, const BtrfsIoctlSearchKey &key);
 
 	string btrfs_search_type_ntoa(unsigned type);
-	string btrfs_search_objectid_ntoa(uint64_t objectid);
+	string btrfs_search_objectid_ntoa(unsigned objectid);
 
 	uint64_t btrfs_get_root_id(int fd);
 	uint64_t btrfs_get_root_transid(int fd);

include/crucible/interp.h

@@ -0,0 +1,106 @@
+#ifndef CRUCIBLE_INTERP_H
+#define CRUCIBLE_INTERP_H
+
+#include "crucible/error.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace crucible {
+	using namespace std;
+
+	struct ArgList : public vector<string> {
+		ArgList(const char **argv);
+		// using vector<string>::vector ... doesn't work:
+		// error: ‘std::vector<std::basic_string<char> >::vector’ names constructor
+		// Still doesn't work in 4.9 because it can't manage a conversion
+		ArgList(const vector<string> &&that);
+	};
+
+	struct ArgActor {
+		struct ArgActorBase {
+			virtual void predicate(void *obj, string arg);
+		};
+
+		template <class T>
+		struct ArgActorDerived {
+			function<void(T, string)> m_func;
+
+			ArgActorDerived(decltype(m_func) func) :
+				m_func(func)
+			{
+			}
+
+			void predicate(void *obj, string arg) override
+			{
+				T &op = *(reinterpret_cast<T*>(obj));
+				m_func(op, obj);
+			}
+		};
+
+		template <class T>
+		ArgActor(T, function<void(T, string)> func) :
+			m_actor(make_shared(ArgActorDerived<T>(func)))
+		{
+		}
+
+		ArgActor() = default;
+
+		void predicate(void *t, string arg)
+		{
+			if (m_actor) {
+				m_actor->predicate(t, arg);
+			} else {
+				THROW_ERROR(invalid_argument, "null m_actor for predicate arg '" << arg << "'");
+			}
+		}
+
+	private:
+		shared_ptr<ArgActorBase> m_actor;
+	};
+
+	struct ArgParser {
+		~ArgParser();
+		ArgParser();
+
+		void add_opt(string opt, ArgActor actor);
+
+		template <class T>
+		void
+		parse(T t, const ArgList &args)
+		{
+			void *vt = &t;
+			parse_backend(vt, args);
+		}
+		
+	private:
+		void parse_backend(void *t, const ArgList &args);
+		map<string, ArgActor>	m_string_opts;
+	};
+
+	struct Command {
+		virtual ~Command();
+		virtual int exec(const ArgList &args) = 0;
+	};
+
+	struct Proc : public Command {
+		int exec(const ArgList &args) override;
+		Proc(const function<int(const ArgList &)> &f);
+	private:
+		function<int(const ArgList &)> m_cmd;
+	};
+
+	struct Interp {
+		virtual ~Interp();
+		Interp(const map<string, shared_ptr<Command> > &cmdlist);
+		void add_command(const string &name, const shared_ptr<Command> &command);
+		int exec(const ArgList &args);
+	private:
+		Interp(const Interp &) = delete;
+		map<string, shared_ptr<Command> > m_commands;
+	};
+
+};
+#endif // CRUCIBLE_INTERP_H

include/crucible/lockset.h

@@ -7,7 +7,6 @@
 
 #include <condition_variable>
 #include <iostream>
-#include <limits>
 #include <mutex>
 #include <set>
 
@@ -18,18 +17,14 @@ namespace crucible {
 	class LockSet {
 
 	public:
+		using key_type = T;
 		using set_type = set<T>;
-		using key_type = typename set_type::key_type;
 
 	private:
 
 		set_type			m_set;
 		mutex				m_mutex;
 		condition_variable		m_condvar;
-		size_t				m_max_size = numeric_limits<size_t>::max();
-
-		bool full();
-		bool locked(const key_type &name);
 
 	public:
 		~LockSet();
@@ -43,8 +38,6 @@ namespace crucible {
 		set_type copy();
 		void wait_unlock(double interval);
 
-		void max_size(size_t max);
-
 		class Lock {
 			LockSet		&m_lockset;
 			key_type	m_name;
@@ -75,33 +68,12 @@ namespace crucible {
 		assert(m_set.empty());
 	}
 
-	template <class T>
-	bool
-	LockSet<T>::full()
-	{
-		return m_set.size() >= m_max_size;
-	}
-
-	template <class T>
-	bool
-	LockSet<T>::locked(const key_type &name)
-	{
-		return m_set.count(name);
-	}
-
-	template <class T>
-	void
-	LockSet<T>::max_size(size_t s)
-	{
-		m_max_size = s;
-	}
-
 	template <class T>
 	void
 	LockSet<T>::lock(const key_type &name)
 	{
 		unique_lock<mutex> lock(m_mutex);
-		while (full() || locked(name)) {
+		while (m_set.count(name)) {
 			m_condvar.wait(lock);
 		}
 		auto rv = m_set.insert(name);
@@ -113,7 +85,7 @@ namespace crucible {
 	LockSet<T>::try_lock(const key_type &name)
 	{
 		unique_lock<mutex> lock(m_mutex);
-		if (full() || locked(name)) {
+		if (m_set.count(name)) {
 			return false;
 		}
 		auto rv = m_set.insert(name);
@@ -126,8 +98,8 @@ namespace crucible {
 	LockSet<T>::unlock(const key_type &name)
 	{
 		unique_lock<mutex> lock(m_mutex);
-		auto erase_count = m_set.erase(name);
 		m_condvar.notify_all();
+		auto erase_count = m_set.erase(name);
 		THROW_CHECK1(invalid_argument, erase_count, erase_count == 1);
 	}
 
@@ -161,10 +133,7 @@ namespace crucible {
 	LockSet<T>::copy()
 	{
 		unique_lock<mutex> lock(m_mutex);
-		// Make temporary copy of set while protected by mutex
+		return m_set;
-		auto rv = m_set;
-		// Return temporary copy after releasing lock
-		return rv;
 	}
 
 	template <class T>

include/crucible/ntoa.h

@@ -7,12 +7,12 @@ namespace crucible {
 	using namespace std;
 
 	struct bits_ntoa_table {
-		unsigned long long n;
+		unsigned long n;
-		unsigned long long mask;
+		unsigned long mask;
 		const char *a;
 	};
 
-	string bits_ntoa(unsigned long long n, const bits_ntoa_table *a);
+	string bits_ntoa(unsigned long n, const bits_ntoa_table *a);
 
 };
 

include/crucible/resource.h

@@ -3,11 +3,11 @@
 
 #include "crucible/error.h"
 
+#include <cassert>
 #include <map>
 #include <memory>
 #include <mutex>
 #include <iostream>
-#include <stdexcept>
 
 namespace crucible {
 	using namespace std;
@@ -44,30 +44,36 @@ namespace crucible {
 
 	private:
 		using traits_type = ResourceTraits<Key, Resource>;
-		using weak_ptr_type = weak_ptr<Resource>;
+
-		using map_type = map<key_type, weak_ptr_type>;
+		class ResourceHolder {
+			resource_ptr_type m_ptr;
+		public:
+			~ResourceHolder();
+			ResourceHolder(resource_ptr_type that);
+			ResourceHolder(const ResourceHolder &that) = default;
+			ResourceHolder(ResourceHolder &&that) = default;
+			ResourceHolder& operator=(ResourceHolder &&that) = default;
+			ResourceHolder& operator=(const ResourceHolder &that) = default;
+			resource_ptr_type get_resource_ptr() const;
+		};
+
+		using holder_ptr_type = shared_ptr<ResourceHolder>;
+		using weak_holder_ptr_type = weak_ptr<ResourceHolder>;
+		using map_type = map<key_type, weak_holder_ptr_type>;
 
 		// The only instance variable
-		resource_ptr_type m_ptr;
+		holder_ptr_type m_ptr;
 
 		// A bunch of static variables and functions
-		static mutex s_map_mutex;
+		static mutex &s_mutex();
-		static mutex s_ptr_mutex;
+		static shared_ptr<map_type> s_map();
-		static map_type s_map;
+		static holder_ptr_type insert(const key_type &key);
-		static resource_ptr_type insert(const key_type &key);
+		static holder_ptr_type insert(const resource_ptr_type &res);
-		static resource_ptr_type insert(const resource_ptr_type &res);
+		static void erase(const key_type &key);
-		static void clean_locked();
 		static ResourceTraits<Key, Resource> s_traits;
 
 	public:
 
-		// Exceptions
-		struct duplicate_resource : public invalid_argument {
-			key_type m_key;
-			key_type get_key() const;
-			duplicate_resource(const key_type &key);
-		};
-
 		// test for resource.  A separate operator because key_type could be confused with bool.
 		bool operator!() const;
 
@@ -83,16 +89,9 @@ namespace crucible {
 		ResourceHandle(const resource_ptr_type &res);
 		ResourceHandle& operator=(const resource_ptr_type &res);
 
-		// default constructor is public and mostly harmless
+		// default constructor is public
 		ResourceHandle() = default;
 
-		// copy/assign/move/move-assign - with a mutex to help shared_ptr be atomic
-		ResourceHandle(const ResourceHandle &that);
-		ResourceHandle(ResourceHandle &&that);
-		ResourceHandle& operator=(const ResourceHandle &that);
-		ResourceHandle& operator=(ResourceHandle &&that);
-		~ResourceHandle();
-
 		// forward anything else to the Resource constructor
 		// if we can do so unambiguously
 		template<class A1, class A2, class... Args>
@@ -110,7 +109,7 @@ namespace crucible {
 
 		// get pointer to Resource object (nothrow, result may be null)
 		resource_ptr_type get_resource_ptr() const;
-		// this version throws
+		// this version throws and is probably not thread safe
 		resource_ptr_type operator->() const;
 
 		// dynamic casting of the resource (throws if cast fails)
@@ -146,100 +145,144 @@ namespace crucible {
 	}
 
 	template <class Key, class Resource>
-	ResourceHandle<Key, Resource>::duplicate_resource::duplicate_resource(const key_type &key) :
+	ResourceHandle<Key, Resource>::ResourceHolder::ResourceHolder(resource_ptr_type that) :
-		invalid_argument("duplicate resource"),
+		m_ptr(that)
-		m_key(key)
 	{
+		// Cannot insert ourselves here since our shared_ptr does not exist yet.
 	}
 
 	template <class Key, class Resource>
-	auto
+	mutex &
-	ResourceHandle<Key, Resource>::duplicate_resource::get_key() const -> key_type
+	ResourceHandle<Key, Resource>::s_mutex()
 	{
-		return m_key;
+		static mutex gcc_won_t_instantiate_this_either;
+		return gcc_won_t_instantiate_this_either;
+	}
+
+	template <class Key, class Resource>
+	shared_ptr<typename ResourceHandle<Key, Resource>::map_type>
+	ResourceHandle<Key, Resource>::s_map()
+	{
+		static shared_ptr<map_type> gcc_won_t_instantiate_the_damn_static_vars;
+		if (!gcc_won_t_instantiate_the_damn_static_vars) {
+			gcc_won_t_instantiate_the_damn_static_vars = make_shared<map_type>();
+		}
+		return gcc_won_t_instantiate_the_damn_static_vars;
 	}
 
 	template <class Key, class Resource>
 	void
-	ResourceHandle<Key, Resource>::clean_locked()
+	ResourceHandle<Key, Resource>::erase(const key_type &key)
 	{
-		// Must be called with lock held
+		unique_lock<mutex> lock(s_mutex());
-		for (auto i = s_map.begin(); i != s_map.end(); ) {
+		// Resources are allowed to set their Keys to null.
-			auto this_i = i;
+		if (s_traits.is_null_key(key)) {
-			++i;
+			// Clean out any dead weak_ptr objects.
-			if (this_i->second.expired()) {
+			for (auto i = s_map()->begin(); i != s_map()->end(); ) {
-				s_map.erase(this_i);
+				if (! (*i).second.lock()) {
+					i = s_map()->erase(i);
+				} else {
+					++i;
+				}
 			}
+			return;
+		}
+		auto erased = s_map()->erase(key);
+		if (erased != 1) {
+			cerr << __PRETTY_FUNCTION__ << ": WARNING: s_map()->erase(" << key << ") returned " << erased << " != 1" << endl;
 		}
 	}
 
 	template <class Key, class Resource>
-	typename ResourceHandle<Key, Resource>::resource_ptr_type
+	ResourceHandle<Key, Resource>::ResourceHolder::~ResourceHolder()
+	{
+		if (!m_ptr) {
+			// Probably something harmless like a failed constructor.
+			cerr << __PRETTY_FUNCTION__ << ": WARNING: destroying null m_ptr" << endl;
+			return;
+		}
+		Key key = s_traits.get_key(*m_ptr);
+		ResourceHandle::erase(key);
+	}
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::holder_ptr_type
 	ResourceHandle<Key, Resource>::insert(const key_type &key)
 	{
 		// no Resources for null keys
 		if (s_traits.is_null_key(key)) {
-			return resource_ptr_type();
+			return holder_ptr_type();
 		}
-		unique_lock<mutex> lock(s_map_mutex);
+		unique_lock<mutex> lock(s_mutex());
-		auto found = s_map.find(key);
+		// find ResourceHolder for non-null key
-		if (found != s_map.end()) {
+		auto found = s_map()->find(key);
-			resource_ptr_type rv = found->second.lock();
+		if (found != s_map()->end()) {
+			holder_ptr_type rv = (*found).second.lock();
+			// a weak_ptr may have expired
 			if (rv) {
-				// Use existing Resource
 				return rv;
-			} else {
-				// It's OK for the map to temporarily contain an expired weak_ptr to some dead Resource
-				clean_locked();
 			}
 		}
 		// not found or expired, throw any existing ref away and make a new one
 		resource_ptr_type rpt = s_traits.make_resource(key);
+		holder_ptr_type hpt = make_shared<ResourceHolder>(rpt);
 		// store weak_ptr in map
-		s_map[key] = rpt;
+		(*s_map())[key] = hpt;
 		// return shared_ptr
-		return rpt;
+		return hpt;
 	};
 
 	template <class Key, class Resource>
-	typename ResourceHandle<Key, Resource>::resource_ptr_type
+	typename ResourceHandle<Key, Resource>::holder_ptr_type
 	ResourceHandle<Key, Resource>::insert(const resource_ptr_type &res)
 	{
-		// no Resources for null keys
+		// no Resource, no ResourceHolder.
 		if (!res) {
-			return resource_ptr_type();
+			return holder_ptr_type();
 		}
+		// no ResourceHolders for null keys either.
 		key_type key = s_traits.get_key(*res);
 		if (s_traits.is_null_key(key)) {
-			return resource_ptr_type();
+			return holder_ptr_type();
 		}
-		unique_lock<mutex> lock(s_map_mutex);
+		unique_lock<mutex> lock(s_mutex());
-		// find Resource for non-null key
+		// find ResourceHolder for non-null key
-		auto found = s_map.find(key);
+		auto found = s_map()->find(key);
-		if (found != s_map.end()) {
+		if (found != s_map()->end()) {
-			resource_ptr_type rv = found->second.lock();
+			holder_ptr_type rv = (*found).second.lock();
-			// It's OK for the map to temporarily contain an expired weak_ptr to some dead Resource...
+			// The map doesn't own the ResourceHolders, the ResourceHandles do.
+			// It's OK for the map to contain an expired weak_ptr to some dead ResourceHolder...
 			if (rv) {
-				// ...but not a duplicate Resource.
+				// found ResourceHolder, look at pointer
-				if (rv.owner_before(res) || res.owner_before(rv)) {
+				resource_ptr_type rp = rv->get_resource_ptr();
-					throw duplicate_resource(key);
+				// We do not store references to null Resources.
+				assert(rp);
+				// Key retrieved for an existing object must match key searched or be null.
+				key_type found_key = s_traits.get_key(*rp);
+				bool found_key_is_null = s_traits.is_null_key(found_key);
+				assert(found_key_is_null || found_key == key);
+				if (!found_key_is_null) {
+					// We do not store references to duplicate resources.
+					if (rp.owner_before(res) || res.owner_before(rp)) {
+						cerr << "inserting new Resource with existing Key " << key << " not allowed at " << __PRETTY_FUNCTION__ << endl;;
+						abort();
+						// THROW_ERROR(out_of_range, "inserting new Resource with existing Key " << key << " not allowed at " << __PRETTY_FUNCTION__);
+					}
+					// rv is good, return it
+					return rv;
 				}
-				// Use the existing Resource (discard the caller's).
-				return rv;
-			} else {
-				// Clean out expired weak_ptrs
-				clean_locked();
 			}
 		}
-		// not found or expired, make a new one or replace old one
+		// not found or expired, make a new one
-		s_map[key] = res;
+		holder_ptr_type rv = make_shared<ResourceHolder>(res);
-		return res;
+		s_map()->insert(make_pair(key, weak_holder_ptr_type(rv)));
+		// no need to check s_map result, we are either replacing a dead weak_ptr or adding a new one
+		return rv;
 	};
 
 	template <class Key, class Resource>
 	ResourceHandle<Key, Resource>::ResourceHandle(const key_type &key)
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
 		m_ptr = insert(key);
 	}
 
@@ -247,7 +290,6 @@ namespace crucible {
 	ResourceHandle<Key, Resource>&
 	ResourceHandle<Key, Resource>::operator=(const key_type &key)
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
 		m_ptr = insert(key);
 		return *this;
 	}
@@ -255,7 +297,6 @@ namespace crucible {
 	template <class Key, class Resource>
 	ResourceHandle<Key, Resource>::ResourceHandle(const resource_ptr_type &res)
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
 		m_ptr = insert(res);
 	}
 
@@ -263,91 +304,36 @@ namespace crucible {
 	ResourceHandle<Key, Resource>&
 	ResourceHandle<Key, Resource>::operator=(const resource_ptr_type &res)
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
 		m_ptr = insert(res);
 		return *this;
 	}
 
 	template <class Key, class Resource>
-	ResourceHandle<Key, Resource>::ResourceHandle(const ResourceHandle &that)
+	typename ResourceHandle<Key, Resource>::resource_ptr_type
+	ResourceHandle<Key, Resource>::ResourceHolder::get_resource_ptr() const
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
+		return m_ptr;
-		m_ptr = that.m_ptr;
-	}
-
-	template <class Key, class Resource>
-	ResourceHandle<Key, Resource>::ResourceHandle(ResourceHandle &&that)
-	{
-		unique_lock<mutex> lock(s_ptr_mutex);
-		swap(m_ptr, that.m_ptr);
-	}
-
-	template <class Key, class Resource>
-	ResourceHandle<Key, Resource> &
-	ResourceHandle<Key, Resource>::operator=(ResourceHandle &&that)
-	{
-		unique_lock<mutex> lock(s_ptr_mutex);
-		m_ptr = that.m_ptr;
-		that.m_ptr.reset();
-		return *this;
-	}
-
-	template <class Key, class Resource>
-	ResourceHandle<Key, Resource> &
-	ResourceHandle<Key, Resource>::operator=(const ResourceHandle &that)
-	{
-		unique_lock<mutex> lock(s_ptr_mutex);
-		m_ptr = that.m_ptr;
-		return *this;
-	}
-
-	template <class Key, class Resource>
-	ResourceHandle<Key, Resource>::~ResourceHandle()
-	{
-		unique_lock<mutex> lock_ptr(s_ptr_mutex);
-		// No pointer, nothing to do
-		if (!m_ptr) {
-			return;
-		}
-		// Save key so we can clean the map
-		auto key = s_traits.get_key(*m_ptr);
-		// Save pointer so we can release lock before deleting
-		auto ptr_copy = m_ptr;
-		m_ptr.reset();
-		// Release lock
-		lock_ptr.unlock();
-		// Delete our (possibly last) reference to pointer
-		ptr_copy.reset();
-		// Remove weak_ptr from map if it has expired
-		// (and not been replaced in the meantime)
-		unique_lock<mutex> lock_map(s_map_mutex);
-		auto found = s_map.find(key);
-		if (found != s_map.end() && found->second.expired()) {
-			s_map.erase(key);
-		}
 	}
 
 	template <class Key, class Resource>
 	typename ResourceHandle<Key, Resource>::resource_ptr_type
 	ResourceHandle<Key, Resource>::get_resource_ptr() const
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
+		if (!m_ptr) {
-		// Make isolated copy of pointer with lock held, and return the copy
+			return resource_ptr_type();
-		auto rv = m_ptr;
+		}
-		return rv;
+		return m_ptr->get_resource_ptr();
 	}
 
 	template <class Key, class Resource>
 	typename ResourceHandle<Key, Resource>::resource_ptr_type
 	ResourceHandle<Key, Resource>::operator->() const
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
+		resource_ptr_type rp = get_resource_ptr();
-		if (!m_ptr) {
+		if (!rp) {
 			THROW_ERROR(out_of_range, __PRETTY_FUNCTION__ << " called on null Resource");
 		}
-		// Make isolated copy of pointer with lock held, and return the copy
+		return rp;
-		auto rv = m_ptr;
-		return rv;
 	}
 
 	template <class Key, class Resource>
@@ -355,12 +341,12 @@ namespace crucible {
 	shared_ptr<T>
 	ResourceHandle<Key, Resource>::cast() const
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
 		shared_ptr<T> dp;
-		if (!m_ptr) {
+		resource_ptr_type rp = get_resource_ptr();
+		if (!rp) {
 			return dp;
 		}
-		dp = dynamic_pointer_cast<T>(m_ptr);
+		dp = dynamic_pointer_cast<T>(rp);
 		if (!dp) {
 			throw bad_cast();
 		}
@@ -371,11 +357,11 @@ namespace crucible {
 	typename ResourceHandle<Key, Resource>::key_type
 	ResourceHandle<Key, Resource>::get_key() const
 	{
-		unique_lock<mutex> lock(s_ptr_mutex);
+		resource_ptr_type rp = get_resource_ptr();
-		if (!m_ptr) {
+		if (!rp) {
 			return s_traits.get_null_key();
 		} else {
-			return s_traits.get_key(*m_ptr);
+			return s_traits.get_key(*rp);
 		}
 	}
 
@@ -392,19 +378,9 @@ namespace crucible {
 		return s_traits.is_null_key(operator key_type());
 	}
 
-	// Apparently GCC wants these to be used before they are defined.
 	template <class Key, class Resource>
 	ResourceTraits<Key, Resource> ResourceHandle<Key, Resource>::s_traits;
 
-	template <class Key, class Resource>
-	mutex ResourceHandle<Key, Resource>::s_map_mutex;
-
-	template <class Key, class Resource>
-	mutex ResourceHandle<Key, Resource>::s_ptr_mutex;
-
-	template <class Key, class Resource>
-	typename ResourceHandle<Key, Resource>::map_type ResourceHandle<Key, Resource>::s_map;
-
 
 }
 

include/crucible/time.h

@@ -32,11 +32,10 @@ namespace crucible {
 		Timer	m_timer;
 		double	m_rate;
 		double	m_burst;
-		double  m_tokens = 0.0;
+		double  m_tokens;
 		mutex	m_mutex;
 
 		void update_tokens();
-		RateLimiter() = delete;
 	public:
 		RateLimiter(double rate, double burst);
 		RateLimiter(double rate);

include/crucible/timequeue.h

@@ -23,7 +23,7 @@ namespace crucible {
 	private:
 		struct Item {
 			Timestamp m_time;
-			unsigned long m_id;
+			unsigned m_id;
 			Task m_task;
 
 			bool operator<(const Item &that) const {
@@ -77,7 +77,7 @@ namespace crucible {
 	void
 	TimeQueue<Task>::push(const Task &task, double delay)
 	{
-		Timestamp time = chrono::high_resolution_clock::now() +
+		Timestamp time = chrono::high_resolution_clock::now() + 
 			chrono::duration_cast<chrono::high_resolution_clock::duration>(chrono::duration<double>(delay));
 		unique_lock<mutex> lock(m_mutex);
 		while (m_set.size() > m_max_queue_depth) {
@@ -91,7 +91,7 @@ namespace crucible {
 	void
 	TimeQueue<Task>::push_nowait(const Task &task, double delay)
 	{
-		Timestamp time = chrono::high_resolution_clock::now() +
+		Timestamp time = chrono::high_resolution_clock::now() + 
 			chrono::duration_cast<chrono::high_resolution_clock::duration>(chrono::duration<double>(delay));
 		unique_lock<mutex> lock(m_mutex);
 		m_set.insert(Item(time, task));

include/crucible/version.h

@@ -1,8 +0,0 @@
-#ifndef CRUCIBLE_VERSION_H
-#define CRUCIBLE_VERSION_H
-
-namespace crucible {
-	extern const char *VERSION;
-}
-
-#endif CRUCIBLE_VERSION_H

include/crucible/workqueue.h

@@ -124,9 +124,7 @@ namespace crucible {
 		if (m_set.empty()) {
 			return key_type();
 		} else {
-			// Make copy with lock held
+			return *m_set.begin();
-			auto rv = *m_set.begin();
-			return rv;
 		}
 	}
 
@@ -151,8 +149,7 @@ namespace crucible {
 	WorkQueue<Task>::copy()
 	{
 		unique_lock<mutex> lock(m_mutex);
-		auto rv = m_set;
+		return m_set;
-		return rv;
 	}
 
 	template <class Task>

lib/.gitignore

@@ -1 +0,0 @@
-.version.*

lib/Makefile

@@ -4,31 +4,34 @@ OBJS = \
 	crc64.o \
 	chatter.o \
 	error.o \
+	execpipe.o \
 	extentwalker.o \
 	fd.o \
 	fs.o \
+	interp.o \
 	ntoa.o \
 	path.o \
 	process.o \
 	string.o \
 	time.o \
 	uuid.o \
-	.version.o \
 
 include ../makeflags
 
-depends.mk: *.cc
+LDFLAGS = -shared -luuid
+
+depends.mk: *.c *.cc
+	for x in *.c; do $(CC) $(CFLAGS) -M "$$x"; done > depends.mk.new
 	for x in *.cc; do $(CXX) $(CXXFLAGS) -M "$$x"; done >> depends.mk.new
 	mv -fv depends.mk.new depends.mk
 
-.version.cc: Makefile ../makeflags *.cc ../include/crucible/*.h
-	echo "namespace crucible { const char *VERSION = \"$(shell git describe --always --dirty || echo UNKNOWN)\"; }" > .version.new.cc
-	mv -f .version.new.cc .version.cc
-
 -include depends.mk
 
+%.o: %.c
+	$(CC) $(CFLAGS) -o $@ -c $<
+
 %.o: %.cc ../include/crucible/%.h
 	$(CXX) $(CXXFLAGS) -o $@ -c $<
 
 libcrucible.so: $(OBJS) Makefile
-	$(CXX) $(LDFLAGS) -o $@ $(OBJS) -shared -luuid
+	$(CXX) $(LDFLAGS) -o $@ $(OBJS)

lib/chatter.cc

@@ -15,7 +15,7 @@
 namespace crucible {
 	using namespace std;
 
-	static shared_ptr<set<string>> chatter_names;
+	static auto_ptr<set<string>> chatter_names;
 	static const char *SPACETAB = " \t";
 
 	static

lib/crc64.cc

@@ -1,31 +1,3 @@
-/* crc64.c -- compute CRC-64
- * Copyright (C) 2013 Mark Adler
- * Version 1.4  16 Dec 2013  Mark Adler
- */
-
-/*
- This software is provided 'as-is', without any express or implied
- warranty.  In no event will the author be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- Mark Adler
- madler@alumni.caltech.edu
- */
-
-/* Substantially modified by Paul Jones for usage in bees */
-
 #include "crucible/crc64.h"
 
 #define POLY64REV 0xd800000000000000ULL
@@ -33,16 +5,13 @@
 namespace crucible {
 
 	static bool init = false;
-	static uint64_t CRCTable[8][256];
+	static uint64_t CRCTable[256];
 
 	static void init_crc64_table()
 	{
 		if (!init) {
-			uint64_t crc;
+			for (int i = 0; i <= 255; i++) {
-
+				uint64_t part = i;
-			// Generate CRCs for all single byte sequences
-			for (int n = 0; n < 256; n++) {
-				uint64_t part = n;
 				for (int j = 0; j < 8; j++) {
 					if (part & 1) {
 						part = (part >> 1) ^ POLY64REV;
@@ -50,53 +19,37 @@ namespace crucible {
 						part >>= 1;
 					}
 				}
-				CRCTable[0][n] = part;
+				CRCTable[i] = part;
-			}
-
-			// Generate nested CRC table for slice-by-8 lookup
-			for (int n = 0; n < 256; n++) {
-				crc = CRCTable[0][n];
-				for (int k = 1; k < 8; k++) {
-					crc = CRCTable[0][crc & 0xff] ^ (crc >> 8);
-					CRCTable[k][n] = crc;
-				}
 			}
 			init = true;
 		}
 	}
 
+	uint64_t
+	Digest::CRC::crc64(const char *s)
+	{
+		init_crc64_table();
+
+		uint64_t crc = 0;
+		for (; *s; s++) {
+			uint64_t temp1 = crc >> 8;
+			uint64_t temp2 = CRCTable[(crc ^ static_cast<uint64_t>(*s)) & 0xff];
+			crc = temp1 ^ temp2;
+		}
+
+		return crc;
+	}
+
 	uint64_t
 	Digest::CRC::crc64(const void *p, size_t len)
 	{
 		init_crc64_table();
-		const unsigned char *next = static_cast<const unsigned char *>(p);
+
 		uint64_t crc = 0;
-
+		for (const unsigned char *s = static_cast<const unsigned char *>(p); len; --len) {
-		// Process individual bytes until we reach an 8-byte aligned pointer
+			uint64_t temp1 = crc >> 8;
-		while (len && (reinterpret_cast<uintptr_t>(next) & 7) != 0) {
+			uint64_t temp2 = CRCTable[(crc ^ *s++) & 0xff];
-			crc = CRCTable[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+			crc = temp1 ^ temp2;
-			len--;
-		}
-
-		// Fast middle processing, 8 bytes (aligned!) per loop
-		while (len >= 8) {
-			crc ^= *(reinterpret_cast< const uint64_t *>(next));
-			crc = CRCTable[7][crc & 0xff] ^
-				  CRCTable[6][(crc >> 8) & 0xff] ^
-				  CRCTable[5][(crc >> 16) & 0xff] ^
-				  CRCTable[4][(crc >> 24) & 0xff] ^
-				  CRCTable[3][(crc >> 32) & 0xff] ^
-				  CRCTable[2][(crc >> 40) & 0xff] ^
-				  CRCTable[1][(crc >> 48) & 0xff] ^
-				  CRCTable[0][crc >> 56];
-			next += 8;
-			len -= 8;
-		}
-
-		// Process remaining bytes (can't be larger than 8)
-		while (len) {
-			crc = CRCTable[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
-			len--;
 		}
 
 		return crc;

lib/execpipe.cc

@@ -0,0 +1,104 @@
+#include "crucible/execpipe.h"
+
+#include "crucible/chatter.h"
+#include "crucible/error.h"
+#include "crucible/process.h"
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+namespace crucible {
+	using namespace std;
+
+	void
+	redirect_stdin(const Fd &child_fd)
+	{
+		dup2_or_die(child_fd, STDIN_FILENO);
+	}
+
+	void
+	redirect_stdin_stdout(const Fd &child_fd)
+	{
+		dup2_or_die(child_fd, STDOUT_FILENO);
+		dup2_or_die(child_fd, STDIN_FILENO);
+	}
+
+	void
+	redirect_stdin_stdout_stderr(const Fd &child_fd)
+	{
+		dup2_or_die(child_fd, STDERR_FILENO);
+		dup2_or_die(child_fd, STDOUT_FILENO);
+		dup2_or_die(child_fd, STDIN_FILENO);
+	}
+
+	void
+	redirect_stdout_stderr(const Fd &child_fd)
+	{
+		dup2_or_die(child_fd, STDERR_FILENO);
+		dup2_or_die(child_fd, STDOUT_FILENO);
+	}
+
+	void
+	redirect_stdout(const Fd &child_fd)
+	{
+		dup2_or_die(child_fd, STDOUT_FILENO);
+	}
+
+	void
+	redirect_stderr(const Fd &child_fd)
+	{
+		dup2_or_die(child_fd, STDERR_FILENO);
+	}
+
+	Fd popen(function<int()> f, function<void(const Fd &child_fd)> import_fd_fn)
+	{
+		Fd parent_fd, child_fd;
+		{
+			pair<Fd, Fd> fd_pair = socketpair_or_die();
+			parent_fd = fd_pair.first;
+			child_fd = fd_pair.second;
+		}
+
+		pid_t fv;
+		DIE_IF_MINUS_ONE(fv = fork());
+
+		if (fv) {
+			child_fd->close();
+			return parent_fd;
+		} else {
+			int rv = EXIT_FAILURE;
+			catch_all([&]() {
+				parent_fd->close();
+				import_fd_fn(child_fd);
+				// system("ls -l /proc/$$/fd/ >&2");
+
+				rv = f();
+			});
+			_exit(rv);
+			cerr << "PID " << getpid() << " TID " << gettid() << "STILL ALIVE" << endl;
+			system("ls -l /proc/$$/task/ >&2");
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	string
+	read_all(Fd fd, size_t max_bytes, size_t chunk_bytes)
+	{
+		char buf[chunk_bytes];
+		string str;
+		size_t rv;
+		while (1) {
+			read_partial_or_die(fd, static_cast<void *>(buf), chunk_bytes, rv);
+			if (rv == 0) {
+				break;
+			}
+			if (max_bytes - str.size() < rv) {
+				THROW_ERROR(out_of_range, "Output size limit " << max_bytes << " exceeded by appending " << rv << " bytes read to " << str.size() << " already in string");
+			}
+			str.append(buf, rv);
+		}
+		return str;
+	}
+}

lib/extentwalker.cc

@@ -79,6 +79,17 @@ namespace crucible {
 			<< "] }";
 	}
 
+	Extent::Extent() :
+		m_begin(0),
+		m_end(0),
+		m_physical(0),
+		m_flags(0),
+		m_physical_len(0),
+		m_logical_len(0),
+		m_offset(0)
+	{
+	}
+
 	Extent::operator bool() const
 	{
 		THROW_CHECK2(invalid_argument, m_begin, m_end, m_end >= m_begin);
@@ -98,18 +109,6 @@ namespace crucible {
 		return m_begin == that.m_begin && m_end == that.m_end && m_physical == that.m_physical && m_flags == that.m_flags;
 	}
 
-	bool
-	Extent::compressed() const
-	{
-		return m_flags & FIEMAP_EXTENT_ENCODED;
-	}
-
-	uint64_t
-	Extent::bytenr() const
-	{
-		return compressed() ? m_physical : m_physical - m_offset;
-	}
-
 	ExtentWalker::ExtentWalker(Fd fd) :
 		m_fd(fd),
 		m_current(m_extents.begin())
@@ -469,7 +468,7 @@ namespace crucible {
 	BtrfsExtentWalker::Vec
 	BtrfsExtentWalker::get_extent_map(off_t pos)
 	{
-		BtrfsIoctlSearchKey sk(sc_extent_fetch_max * (sizeof(btrfs_file_extent_item) + sizeof(btrfs_ioctl_search_header)));
+		BtrfsIoctlSearchKey sk;
 		if (!m_root_fd) {
 			m_root_fd = m_fd;
 		}
@@ -520,26 +519,25 @@ namespace crucible {
 
 			auto type = call_btrfs_get(btrfs_stack_file_extent_type, i.m_data);
 			off_t len = -1;
-			switch (type) {
+                        switch (type) {
-				default:
+                                default:
 					cerr << "Unhandled file extent type " << type << " in root " << m_tree_id << " ino " << m_stat.st_ino << endl;
 					break;
-				case BTRFS_FILE_EXTENT_INLINE:
+                                case BTRFS_FILE_EXTENT_INLINE:
 					len = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data));
 					e.m_flags |= FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
 					// Inline extents are never obscured, so don't bother filling in m_physical_len, etc.
-					break;
+                                        break;
-				case BTRFS_FILE_EXTENT_PREALLOC:
+                                case BTRFS_FILE_EXTENT_PREALLOC:
 					e.m_flags |= Extent::PREALLOC;
-					// fallthrough
+                                case BTRFS_FILE_EXTENT_REG: {
-				case BTRFS_FILE_EXTENT_REG: {
 					e.m_physical = call_btrfs_get(btrfs_stack_file_extent_disk_bytenr, i.m_data);
 
 					// This is the length of the full extent (decompressed)
-					off_t ram = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data));
+                                        off_t ram = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data));
 
 					// This is the length of the part of the extent appearing in the file (decompressed)
-					len = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_num_bytes, i.m_data));
+                                        len = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_num_bytes, i.m_data));
 
 					// This is the offset from start of on-disk extent to the part we see in the file (decompressed)
 					// May be negative due to the kind of bug we're stuck with forever, so no cast range check

lib/fd.cc

@@ -230,14 +230,6 @@ namespace crucible {
 		}
 	}
 
-	void
-	ftruncate_or_die(int fd, off_t size)
-	{
-		if (::ftruncate(fd, size)) {
-			THROW_ERRNO("ftruncate: " << name_fd(fd) << " size " << size);
-		}
-	}
-
 	string
 	socket_domain_ntoa(int domain)
 	{
@@ -434,27 +426,6 @@ namespace crucible {
 		return pread_or_die(fd, text.data(), text.size(), offset);
 	}
 
-	template<>
-	void
-	pwrite_or_die<vector<uint8_t>>(int fd, const vector<uint8_t> &text, off_t offset)
-	{
-		return pwrite_or_die(fd, text.data(), text.size(), offset);
-	}
-
-	template<>
-	void
-	pwrite_or_die<vector<char>>(int fd, const vector<char> &text, off_t offset)
-	{
-		return pwrite_or_die(fd, text.data(), text.size(), offset);
-	}
-
-	template<>
-	void
-	pwrite_or_die<string>(int fd, const string &text, off_t offset)
-	{
-		return pwrite_or_die(fd, text.data(), text.size(), offset);
-	}
-
 	Stat::Stat()
 	{
 		memset_zero<stat>(this);

lib/fs.cc

@@ -707,19 +707,11 @@ namespace crucible {
 		return offset + len;
 	}
 
-	bool
-	BtrfsIoctlSearchHeader::operator<(const BtrfsIoctlSearchHeader &that) const
-	{
-		return tie(objectid, type, offset, len, transid) < tie(that.objectid, that.type, that.offset, that.len, that.transid);
-	}
-
 	bool
 	BtrfsIoctlSearchKey::do_ioctl_nothrow(int fd)
 	{
 		vector<char> ioctl_arg = vector_copy_struct<btrfs_ioctl_search_key>(this);
-		// Normally we like to be paranoid and fill empty bytes with zero,
+		ioctl_arg.resize(sizeof(btrfs_ioctl_search_args_v2) + m_buf_size, 0);
-		// but these buffers can be huge.  80% of a 4GHz CPU huge.
-		ioctl_arg.resize(sizeof(btrfs_ioctl_search_args_v2) + m_buf_size);
 		btrfs_ioctl_search_args_v2 *ioctl_ptr = reinterpret_cast<btrfs_ioctl_search_args_v2 *>(ioctl_arg.data());
 
 		ioctl_ptr->buf_size = m_buf_size;
@@ -733,12 +725,13 @@ namespace crucible {
 		static_cast<btrfs_ioctl_search_key&>(*this) = ioctl_ptr->key;
 
 		m_result.clear();
+		m_result.reserve(nr_items);
 
 		size_t offset = pointer_distance(ioctl_ptr->buf, ioctl_ptr);
 		for (decltype(nr_items) i = 0; i < nr_items; ++i) {
 			BtrfsIoctlSearchHeader item;
 			offset = item.set_data(ioctl_arg, offset);
-			m_result.insert(item);
+			m_result.push_back(item);
 		}
 
 		return true;
@@ -841,7 +834,7 @@ namespace crucible {
 	}
 
 	string
-	btrfs_search_objectid_ntoa(uint64_t objectid)
+	btrfs_search_objectid_ntoa(unsigned objectid)
 	{
 		static const bits_ntoa_table table[] = {
 			NTOA_TABLE_ENTRY_ENUM(BTRFS_ROOT_TREE_OBJECTID),
@@ -913,7 +906,7 @@ namespace crucible {
 	ostream &
 	operator<<(ostream &os, const BtrfsIoctlSearchHeader &hdr)
 	{
-		os << "BtrfsIoctlSearchHeader { "
+		os << "BtrfsIoctlSearchHeader { " 
 			<< static_cast<const btrfs_ioctl_search_header &>(hdr)
 			<< ", data = ";
 		hexdump(os, hdr.m_data);
@@ -923,7 +916,7 @@ namespace crucible {
 	ostream &
 	operator<<(ostream &os, const BtrfsIoctlSearchKey &key)
 	{
-		os << "BtrfsIoctlSearchKey { "
+		os << "BtrfsIoctlSearchKey { " 
 			<< static_cast<const btrfs_ioctl_search_key &>(key)
 			<< ", buf_size = " << key.m_buf_size
 			<< ", buf[" << key.m_result.size() << "] = {";

lib/interp.cc

@@ -0,0 +1,96 @@
+#include "crucible/interp.h"
+
+#include "crucible/chatter.h"
+
+namespace crucible {
+	using namespace std;
+
+	int
+	Proc::exec(const ArgList &args)
+	{
+		return m_cmd(args);
+	}
+
+	Proc::Proc(const function<int(const ArgList &)> &f) :
+		m_cmd(f)
+	{
+	}
+
+	Command::~Command()
+	{
+	}
+
+	ArgList::ArgList(const char **argv)
+	{
+		while (argv && *argv) {
+			push_back(*argv++);
+		}
+	}
+
+	ArgList::ArgList(const vector<string> &&that) :
+		vector<string>(that)
+	{
+	}
+
+	Interp::~Interp()
+	{
+	}
+
+	Interp::Interp(const map<string, shared_ptr<Command> > &cmdlist) :
+		m_commands(cmdlist)
+	{
+	}
+
+	void
+	Interp::add_command(const string &name, const shared_ptr<Command> &command)
+	{
+		m_commands[name] = command;
+	}
+
+	int
+	Interp::exec(const ArgList &args)
+	{
+		auto next_arg = args.begin();
+		++next_arg;
+		return m_commands.at(args[0])->exec(vector<string>(next_arg, args.end()));
+	}
+
+	ArgParser::~ArgParser()
+	{
+	}
+
+	ArgParser::ArgParser()
+	{
+	}
+
+	void
+	ArgParser::add_opt(string opt, ArgActor actor)
+	{
+		m_string_opts[opt] = actor;
+	}
+
+	void
+	ArgParser::parse_backend(void *t, const ArgList &args)
+	{
+		bool quote_args = false;
+		for (string arg : args) {
+			if (quote_args) {
+				cerr << "arg: '" << arg << "'" << endl;
+				continue;
+			}
+			if (arg == "--") {
+				quote_args = true;
+				continue;
+			}
+			if (arg.compare(0, 2, "--") == 0) {
+				auto found = m_string_opts.find(arg.substr(2, string::npos));
+				if (found != m_string_opts.end()) {
+					found->second.predicate(t, "foo");
+				}
+				(void)t;
+			}
+		}
+	}
+
+
+};

lib/ntoa.cc

@@ -7,7 +7,7 @@
 namespace crucible {
 	using namespace std;
 
-	string bits_ntoa(unsigned long long n, const bits_ntoa_table *table)
+	string bits_ntoa(unsigned long n, const bits_ntoa_table *table)
 	{
 		string out;
 		while (n && table->a) {

makeflags

@@ -1,4 +1,4 @@
-CCFLAGS  = -Wall -Wextra -Werror -O3 -march=native -I../include -ggdb -fpic -D_FILE_OFFSET_BITS=64
+CCFLAGS  = -Wall -Wextra -Werror -O3 -I../include -ggdb -fpic
 # CCFLAGS  = -Wall -Wextra -Werror -O0 -I../include -ggdb -fpic
 CFLAGS   = $(CCFLAGS) -std=c99
 CXXFLAGS = $(CCFLAGS) -std=c++11 -Wold-style-cast

scripts/beesd

@@ -1,128 +0,0 @@
-#!/bin/bash
-# /usr/bin/beesd
-
-## Helpful functions
-INFO(){ echo "INFO:" "$@"; }
-ERRO(){ echo "ERROR:" "$@"; exit 1; }
-YN(){ [[ "$1" =~ (1|Y|y) ]]; }
-
-## Global vars
-export BEESHOME BEESSTATUS
-export WORK_DIR CONFIG_DIR
-export CONFIG_FILE
-export UUID AL16M
-
-readonly AL16M="$((16*1024*1024))"
-readonly CONFIG_DIR=/etc/bees/
-
-## Pre checks
-{
-    [ ! -d "$CONFIG_DIR" ] && ERRO "Missing: $CONFIG_DIR"
-    [ "$UID" == "0" ] || ERRO "Must be runned as root"
-}
-
-command -v bees &> /dev/null || ERRO "Missing 'bees' command"
-
-## Parse args
-UUID="$1"
-case "$UUID" in
-    *-*-*-*-*)
-        FILE_CONFIG=""
-        for file in "$CONFIG_DIR"/*.conf; do
-            [ ! -f "$file" ] && continue
-            if grep -q "$UUID" "$file"; then
-                INFO "Find $UUID in $file, use as conf"
-                FILE_CONFIG="$file"
-            fi
-        done
-        [ ! -f "$FILE_CONFIG" ] && ERRO "No config for $UUID"
-        source "$FILE_CONFIG"
-    ;;
-    *)
-        echo "beesd <btrfs_uuid>"
-        exit 1
-    ;;
-esac
-
-WORK_DIR="${WORK_DIR:-/run/bees/}"
-MNT_DIR="${MNT_DIR:-$WORK_DIR/mnt/$UUID}"
-BEESHOME="${BEESHOME:-$MNT_DIR/.beeshome}"
-BEESSTATUS="${BEESSTATUS:-$WORK_DIR/$UUID.status}"
-DB_SIZE="${DB_SIZE:-$((64*AL16M))}"
-LOG_SHORT_PATH="${LOG_SHORT_PATH:-N}"
-LOG_FILTER_TIME="${LOG_FILTER_TIME:-N}"
-
-INFO "Check: Disk exists"
-if [ ! -b "/dev/disk/by-uuid/$UUID" ]; then
-    ERRO "Missing disk: /dev/disk/by-uuid/$UUID"
-fi
-
-it_btrfs(){ [ "$(blkid -s TYPE -o value "$1")" == "btrfs" ]; }
-
-INFO "Check: Disk with btrfs"
-if ! it_btrfs "/dev/disk/by-uuid/$UUID"; then
-    ERRO "Disk not contain btrfs: /dev/disk/by-uuid/$UUID"
-fi
-
-INFO "WORK DIR: $WORK_DIR"
-mkdir -p "$WORK_DIR" || exit 1
-
-INFO "MOUNT DIR: $MNT_DIR"
-mkdir -p "$MNT_DIR" || exit 1
-
-umount_w(){ mountpoint -q "$1" && umount -l "$1"; }
-force_umount(){ umount_w "$MNT_DIR"; }
-trap force_umount SIGINT SIGTERM EXIT
-
-mount -osubvolid=5 /dev/disk/by-uuid/$UUID "$MNT_DIR" || exit 1
-
-if [ ! -d "$BEESHOME" ]; then
-    INFO "Create subvol $BEESHOME for store bees data"
-    btrfs sub cre "$BEESHOME"
-else
-    btrfs sub show "$BEESHOME" &> /dev/null || ERRO "$BEESHOME MUST BE A SUBVOL!"
-fi
-
-# Check DB size
-{
-    DB_PATH="$BEESHOME/beeshash.dat"
-    touch "$DB_PATH"
-    OLD_SIZE="$(du -b "$DB_PATH" | sed 's/\t/ /g' | cut -d' ' -f1)"
-    NEW_SIZE="$DB_SIZE"
-    if (( "$NEW_SIZE"%AL16M > 0 )); then
-        ERRO "DB_SIZE Must be multiple of 16M"
-    fi
-    if (( "$OLD_SIZE" != "$NEW_SIZE" )); then
-        INFO "Resize db: $OLD_SIZE -> $NEW_SIZE"
-        [ -f "$BEESHOME/beescrawl.$UUID.dat" ] && rm "$BEESHOME/beescrawl.$UUID.dat"
-        truncate -s $NEW_SIZE $DB_PATH
-    fi
-    chmod 700 "$DB_PATH"
-}
-
-MNT_DIR="${MNT_DIR//\/\//\/}"
-
-filter_time(){
-    if YN $LOG_FILTER_TIME; then
-        sed -e 's/^.*crawl:/crawl:/g' \
-            -e 's/^.*status:/status:/g' \
-            -e 's/^.*bees:/bees:/g' \
-            -e 's/^.*crawl_writeback:/crawl_writeback:/g' \
-            -e 's/^.*main:/main:/g' \
-            -e 's/^.*hash_prefetch:/hash_prefetch:/g'
-    else
-        cat
-    fi
-}
-
-filter_path(){
-    if YN $LOG_SHORT_PATH; then
-        sed -e "s#$MNT_DIR##g"
-    else
-        cat
-    fi
-}
-
-bees "$MNT_DIR" 3>&1 2>&1 | filter_time | filter_path
-
-exit 0

scripts/beesd.conf.sample

@@ -1,34 +0,0 @@
-## Config for Bees: /etc/bees/beesd.conf.sample
-## https://github.com/Zygo/bees
-## It's a default values, change it, if needed
-
-# Which FS will be used
-UUID=5d3c0ad5-bedf-463d-8235-b4d4f6f99476
-
-## System Vars
-# Change carefully
-# WORK_DIR=/run/bees/
-# MNT_DIR="$WORK_DIR/mnt/$UUID"
-# BEESHOME="$MNT_DIR/.beeshome"
-# BEESSTATUS="$WORK_DIR/$UUID.status"
-
-## Make path shorter in logs
-# LOG_SHORT_PATH=N
-
-## Remove timestamp from bees output
-# LOG_FILTER_TIME=N
-
-## Bees DB size
-# Hash Table Sizing
-# sHash table entries are 16 bytes each
-# (64-bit hash, 52-bit block number, and some metadata bits)
-# Each entry represents a minimum of 4K on disk.
-# unique data size    hash table size    average dedup block size
-#     1TB                 4GB                  4K
-#     1TB                 1GB                 16K
-#     1TB               256MB                 64K
-#     1TB                16MB               1024K
-#    64TB                 1GB               1024K
-#
-# Size MUST be power of 16M
-# DB_SIZE=$((64*$AL16M)) # 1G in bytes

scripts/beesd@.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=Bees - Best-Effort Extent-Same, a btrfs deduplicator daemon: %i
-After=local-fs.target
-
-[Service]
-ExecStart=/usr/bin/beesd %i
-Nice=19
-IOSchedulingClass=idle
-CPUAccounting=true
-MemoryAccounting=true
-# CPUQuota=95%
-
-[Install]
-WantedBy=local-fs.target

src/.gitignore

@@ -1 +0,0 @@
-bees-version.[ch]

src/Makefile

@@ -1,5 +1,6 @@
 PROGRAMS = \
 	../bin/bees \
+	../bin/fanotify-watch \
 	../bin/fiemap \
 	../bin/fiewalk \
 
@@ -14,10 +15,6 @@ depends.mk: Makefile *.cc
 	for x in *.cc; do $(CXX) $(CXXFLAGS) -M "$$x"; done > depends.mk.new
 	mv -fv depends.mk.new depends.mk
 
-bees-version.c: Makefile *.cc *.h
-	echo "const char *BEES_VERSION = \"$(shell git describe --always --dirty || echo UNKNOWN)\";" > bees-version.new.c
-	mv -f bees-version.new.c bees-version.c
-
 -include depends.mk
 
 %.o: %.cc %.h
@@ -35,11 +32,9 @@ BEES_OBJS = \
 	bees-roots.o \
 	bees-thread.o \
 	bees-types.o \
-	bees-version.o \
 
 ../bin/bees: $(BEES_OBJS)
 	$(CXX) $(CXXFLAGS) -o "$@" $(BEES_OBJS) $(LDFLAGS) $(LIBS)
 
 clean:
-	-rm -fv bees-version.h
+	-rm -fv *.o
-	-rm -fv *.o bees-version.c

src/bees-context.cc

@@ -5,7 +5,6 @@
 
 #include <fstream>
 #include <iostream>
-#include <vector>
 
 using namespace crucible;
 using namespace std;
@@ -24,16 +23,10 @@ getenv_or_die(const char *name)
 BeesFdCache::BeesFdCache()
 {
 	m_root_cache.func([&](shared_ptr<BeesContext> ctx, uint64_t root) -> Fd {
-		Timer open_timer;
+		return ctx->roots()->open_root_nocache(root);
-		auto rv = ctx->roots()->open_root_nocache(root);
-		BEESCOUNTADD(open_root_ms, open_timer.age() * 1000);
-		return rv;
 	});
 	m_file_cache.func([&](shared_ptr<BeesContext> ctx, uint64_t root, uint64_t ino) -> Fd {
-		Timer open_timer;
+		return ctx->roots()->open_root_ino_nocache(root, ino);
-		auto rv = ctx->roots()->open_root_ino_nocache(root, ino);
-		BEESCOUNTADD(open_ino_ms, open_timer.age() * 1000);
-		return rv;
 	});
 }
 
@@ -55,12 +48,6 @@ BeesFdCache::open_root(shared_ptr<BeesContext> ctx, uint64_t root)
 Fd
 BeesFdCache::open_root_ino(shared_ptr<BeesContext> ctx, uint64_t root, uint64_t ino)
 {
-	if (m_file_cache_timer.age() > BEES_COMMIT_INTERVAL) {
-		BEESINFO("Clearing open FD cache to enable file delete");
-		m_file_cache.clear();
-		m_file_cache_timer.reset();
-		BEESCOUNT(open_clear);
-	}
 	return m_file_cache(ctx, root, ino);
 }
 
@@ -241,24 +228,15 @@ BeesContext::show_progress()
 	}
 }
 
-Fd
-BeesContext::home_fd()
-{
-	const char *base_dir = getenv("BEESHOME");
-	if (!base_dir) {
-		base_dir = ".beeshome";
-	}
-	m_home_fd = openat(root_fd(), base_dir, FLAGS_OPEN_DIR);
-	if (!m_home_fd) {
-		THROW_ERRNO("openat: " << name_fd(root_fd()) << " / " << base_dir);
-	}
-	return m_home_fd;
-}
-
 BeesContext::BeesContext(shared_ptr<BeesContext> parent) :
 	m_parent_ctx(parent)
 {
+	auto base_dir = getenv_or_die("BEESHOME");
+	BEESLOG("BEESHOME = " << base_dir);
+	m_home_fd = open_or_die(base_dir, FLAGS_OPEN_DIR);
 	if (m_parent_ctx) {
+		m_hash_table = m_parent_ctx->hash_table();
+		m_hash_table->set_shared(true);
 		m_fd_cache = m_parent_ctx->fd_cache();
 	}
 }
@@ -975,8 +953,7 @@ BeesContext::tmpfile()
 	if (!m_tmpfiles[this_thread::get_id()]) {
 		m_tmpfiles[this_thread::get_id()] = make_shared<BeesTempFile>(shared_from_this());
 	}
-	auto rv = m_tmpfiles[this_thread::get_id()];
+	return m_tmpfiles[this_thread::get_id()];
-	return rv;
 }
 
 shared_ptr<BeesFdCache>
@@ -987,8 +964,7 @@ BeesContext::fd_cache()
 	if (!m_fd_cache) {
 		m_fd_cache = make_shared<BeesFdCache>();
 	}
-	auto rv = m_fd_cache;
+	return m_fd_cache;
-	return rv;
 }
 
 shared_ptr<BeesRoots>
@@ -999,8 +975,7 @@ BeesContext::roots()
 	if (!m_roots) {
 		m_roots = make_shared<BeesRoots>(shared_from_this());
 	}
-	auto rv = m_roots;
+	return m_roots;
-	return rv;
 }
 
 shared_ptr<BeesHashTable>
@@ -1011,8 +986,7 @@ BeesContext::hash_table()
 	if (!m_hash_table) {
 		m_hash_table = make_shared<BeesHashTable>(shared_from_this(), "beeshash.dat");
 	}
-	auto rv = m_hash_table;
+	return m_hash_table;
-	return rv;
 }
 
 void

src/bees-hash.cc

@@ -11,6 +11,13 @@
 using namespace crucible;
 using namespace std;
 
+static inline
+bool
+using_any_madvise()
+{
+	return true;
+}
+
 ostream &
 operator<<(ostream &os, const BeesHash &bh)
 {
@@ -94,6 +101,8 @@ BeesHashTable::get_extent_range(HashType hash)
 void
 BeesHashTable::flush_dirty_extents()
 {
+	if (using_shared_map()) return;
+
 	THROW_CHECK1(runtime_error, m_buckets, m_buckets > 0);
 
 	unique_lock<mutex> lock(m_extent_mutex);
@@ -115,12 +124,16 @@ BeesHashTable::flush_dirty_extents()
 			uint8_t *dirty_extent_end = m_extent_ptr[extent_number + 1].p_byte;
 			THROW_CHECK1(out_of_range, dirty_extent,     dirty_extent     >= m_byte_ptr);
 			THROW_CHECK1(out_of_range, dirty_extent_end, dirty_extent_end <= m_byte_ptr_end);
-			THROW_CHECK2(out_of_range, dirty_extent_end, dirty_extent, dirty_extent_end - dirty_extent == BLOCK_SIZE_HASHTAB_EXTENT);
+			if (using_shared_map()) {
-			BEESTOOLONG("pwrite(fd " << m_fd << " '" << name_fd(m_fd)<< "', length " << to_hex(dirty_extent_end - dirty_extent) << ", offset " << to_hex(dirty_extent - m_byte_ptr) << ")");
+				BEESTOOLONG("flush extent " << extent_number);
-			// Page locks slow us down more than copying the data does
+				copy(dirty_extent, dirty_extent_end, dirty_extent);
-			vector<uint8_t> extent_copy(dirty_extent, dirty_extent_end);
+			} else {
-			pwrite_or_die(m_fd, extent_copy, dirty_extent - m_byte_ptr);
+				BEESTOOLONG("pwrite(fd " << m_fd << " '" << name_fd(m_fd)<< "', length " << to_hex(dirty_extent_end - dirty_extent) << ", offset " << to_hex(dirty_extent - m_byte_ptr) << ")");
-			BEESCOUNT(hash_extent_out);
+				// Page locks slow us down more than copying the data does
+				vector<uint8_t> extent_copy(dirty_extent, dirty_extent_end);
+				pwrite_or_die(m_fd, extent_copy, dirty_extent - m_byte_ptr);
+				BEESCOUNT(hash_extent_out);
+			}
 		});
 		BEESNOTE("flush rate limited at extent #" << extent_number << " (" << extent_counter << " of " << dirty_extent_copy.size() << ")");
 		m_flush_rate_limit.sleep_for(BLOCK_SIZE_HASHTAB_EXTENT);
@@ -130,6 +143,7 @@ BeesHashTable::flush_dirty_extents()
 void
 BeesHashTable::set_extent_dirty(HashType hash)
 {
+	if (using_shared_map()) return;
 	THROW_CHECK1(runtime_error, m_buckets, m_buckets > 0);
 	auto pr = get_extent_range(hash);
 	uint64_t extent_number = reinterpret_cast<Extent *>(pr.first) - m_extent_ptr;
@@ -142,8 +156,10 @@ BeesHashTable::set_extent_dirty(HashType hash)
 void
 BeesHashTable::writeback_loop()
 {
-	while (true) {
+	if (!using_shared_map()) {
-		flush_dirty_extents();
+		while (1) {
+			flush_dirty_extents();
+		}
 	}
 }
 
@@ -259,10 +275,9 @@ BeesHashTable::prefetch_loop()
 
 		graph_blob << "Now:     " << format_time(time(NULL)) << "\n";
 		graph_blob << "Uptime:  " << m_ctx->total_timer().age() << " seconds\n";
-		graph_blob << "Version: " << BEES_VERSION << "\n";
 
-		graph_blob
+		graph_blob 
-			<< "\nHash table page occupancy histogram (" << occupied_count << "/" << total_count << " cells occupied, " << (occupied_count * 100 / total_count) << "%)\n"
+			<< "\nHash table page occupancy histogram (" << occupied_count << "/" << total_count << " cells occupied, " << (occupied_count * 100 / total_count) << "%)\n" 
 			<< out.str() << "0%      |      25%      |      50%      |      75%      |   100% page fill\n"
 			<< "compressed " << compressed_count << " (" << percent(compressed_count, occupied_count) << ")"
 			<< " new-style " << compressed_offset_count << " (" << percent(compressed_offset_count, occupied_count) << ")"
@@ -295,6 +310,7 @@ void
 BeesHashTable::fetch_missing_extent(HashType hash)
 {
 	BEESTOOLONG("fetch_missing_extent for hash " << to_hex(hash));
+	if (using_shared_map()) return;
 	THROW_CHECK1(runtime_error, m_buckets, m_buckets > 0);
 	auto pr = get_extent_range(hash);
 	uint64_t extent_number = reinterpret_cast<Extent *>(pr.first) - m_extent_ptr;
@@ -308,7 +324,7 @@ BeesHashTable::fetch_missing_extent(HashType hash)
 	size_t missing_buckets = m_buckets_missing.size();
 	lock.unlock();
 
-	BEESNOTE("waiting to fetch hash extent #" << extent_number << ", " << missing_buckets << " left to fetch");
+	BEESNOTE("fetch waiting for hash extent #" << extent_number << ", " << missing_buckets << " left to fetch");
 
 	// Acquire blocking lock on this extent only
 	LockSet<uint64_t>::Lock extent_lock(m_extent_lock_set, extent_number);
@@ -335,6 +351,9 @@ BeesHashTable::fetch_missing_extent(HashType hash)
 	}
 
 	BEESCOUNT(hash_extent_in);
+	// We don't block when fetching an extent but we do slow down the
+	// prefetch thread.
+	m_prefetch_rate_limit.borrow(BLOCK_SIZE_HASHTAB_EXTENT);
 	lock.lock();
 	m_buckets_missing.erase(extent_number);
 }
@@ -377,6 +396,7 @@ BeesHashTable::find_cell(HashType hash)
 void
 BeesHashTable::erase_hash_addr(HashType hash, AddrType addr)
 {
+	// if (m_shared) return;
 	fetch_missing_extent(hash);
 	BEESTOOLONG("erase hash " << to_hex(hash) << " addr " << addr);
 	unique_lock<mutex> lock(m_bucket_mutex);
@@ -554,36 +574,12 @@ BeesHashTable::try_mmap_flags(int flags)
 }
 
 void
-BeesHashTable::open_file()
+BeesHashTable::set_shared(bool shared)
 {
-	// OK open hash table
+	m_shared = shared;
-	BEESNOTE("opening hash table '" << m_filename << "' target size " << m_size << " (" << pretty(m_size) << ")");
-
-	// Try to open existing hash table
-	Fd new_fd = openat(m_ctx->home_fd(), m_filename.c_str(), FLAGS_OPEN_FILE_RW, 0700);
-
-	// If that doesn't work, try to make a new one
-	if (!new_fd) {
-		string tmp_filename = m_filename + ".tmp";
-		BEESLOGNOTE("creating new hash table '" << tmp_filename << "'");
-		unlinkat(m_ctx->home_fd(), tmp_filename.c_str(), 0);
-		new_fd = openat_or_die(m_ctx->home_fd(), tmp_filename, FLAGS_CREATE_FILE, 0700);
-		BEESLOGNOTE("truncating new hash table '" << tmp_filename << "' size " << m_size << " (" << pretty(m_size) << ")");
-		ftruncate_or_die(new_fd, m_size);
-		BEESLOGNOTE("truncating new hash table '" << tmp_filename << "' -> '" << m_filename << "'");
-		renameat_or_die(m_ctx->home_fd(), tmp_filename, m_ctx->home_fd(), m_filename);
-	}
-
-	Stat st(new_fd);
-	off_t new_size = st.st_size;
-
-	THROW_CHECK1(invalid_argument, new_size, new_size > 0);
-	THROW_CHECK1(invalid_argument, new_size, (new_size % BLOCK_SIZE_HASHTAB_EXTENT) == 0);
-	m_size = new_size;
-	m_fd = new_fd;
 }
 
-BeesHashTable::BeesHashTable(shared_ptr<BeesContext> ctx, string filename, off_t size) :
+BeesHashTable::BeesHashTable(shared_ptr<BeesContext> ctx, string filename) :
 	m_ctx(ctx),
 	m_size(0),
 	m_void_ptr(nullptr),
@@ -591,29 +587,35 @@ BeesHashTable::BeesHashTable(shared_ptr<BeesContext> ctx, string filename, off_t
 	m_buckets(0),
 	m_cells(0),
 	m_writeback_thread("hash_writeback"),
-	m_prefetch_thread("hash_prefetch"),
+	m_prefetch_thread("hash_prefetch " + m_ctx->root_path()),
 	m_flush_rate_limit(BEES_FLUSH_RATE),
+	m_prefetch_rate_limit(BEES_FLUSH_RATE),
 	m_stats_file(m_ctx->home_fd(), "beesstats.txt")
 {
-	// Sanity checks to protect the implementation from its weaknesses
+	BEESNOTE("opening hash table " << filename);
+
+	m_fd = openat_or_die(m_ctx->home_fd(), filename, FLAGS_OPEN_FILE_RW, 0700);
+	Stat st(m_fd);
+	m_size = st.st_size;
+
+	BEESTRACE("hash table size " << m_size);
+	BEESTRACE("hash table bucket size " << BLOCK_SIZE_HASHTAB_BUCKET);
+	BEESTRACE("hash table extent size " << BLOCK_SIZE_HASHTAB_EXTENT);
+
 	THROW_CHECK2(invalid_argument, BLOCK_SIZE_HASHTAB_BUCKET, BLOCK_SIZE_HASHTAB_EXTENT, (BLOCK_SIZE_HASHTAB_EXTENT % BLOCK_SIZE_HASHTAB_BUCKET) == 0);
 
+	// Does the union work?
+	THROW_CHECK2(runtime_error, m_void_ptr, m_cell_ptr, m_void_ptr == m_cell_ptr);
+	THROW_CHECK2(runtime_error, m_void_ptr, m_byte_ptr, m_void_ptr == m_byte_ptr);
+	THROW_CHECK2(runtime_error, m_void_ptr, m_bucket_ptr, m_void_ptr == m_bucket_ptr);
+	THROW_CHECK2(runtime_error, m_void_ptr, m_extent_ptr, m_void_ptr == m_extent_ptr);
+
 	// There's more than one union
 	THROW_CHECK2(runtime_error, sizeof(Bucket), BLOCK_SIZE_HASHTAB_BUCKET, BLOCK_SIZE_HASHTAB_BUCKET == sizeof(Bucket));
 	THROW_CHECK2(runtime_error, sizeof(Bucket::p_byte), BLOCK_SIZE_HASHTAB_BUCKET, BLOCK_SIZE_HASHTAB_BUCKET == sizeof(Bucket::p_byte));
 	THROW_CHECK2(runtime_error, sizeof(Extent), BLOCK_SIZE_HASHTAB_EXTENT, BLOCK_SIZE_HASHTAB_EXTENT == sizeof(Extent));
 	THROW_CHECK2(runtime_error, sizeof(Extent::p_byte), BLOCK_SIZE_HASHTAB_EXTENT, BLOCK_SIZE_HASHTAB_EXTENT == sizeof(Extent::p_byte));
 
-	m_filename = filename;
-	m_size = size;
-	open_file();
-
-	// Now we know size we can compute stuff
-
-	BEESTRACE("hash table size " << m_size);
-	BEESTRACE("hash table bucket size " << BLOCK_SIZE_HASHTAB_BUCKET);
-	BEESTRACE("hash table extent size " << BLOCK_SIZE_HASHTAB_EXTENT);
-
 	BEESLOG("opened hash table filename '" << filename << "' length " << m_size);
 	m_buckets = m_size / BLOCK_SIZE_HASHTAB_BUCKET;
 	m_cells = m_buckets * c_cells_per_bucket;
@@ -622,38 +624,27 @@ BeesHashTable::BeesHashTable(shared_ptr<BeesContext> ctx, string filename, off_t
 
 	BEESLOG("\tflush rate limit " << BEES_FLUSH_RATE);
 
-	// Try to mmap that much memory
+	if (using_shared_map()) {
-	try_mmap_flags(MAP_PRIVATE | MAP_ANONYMOUS);
+		try_mmap_flags(MAP_SHARED);
+	} else {
+		try_mmap_flags(MAP_PRIVATE | MAP_ANONYMOUS);
+	}
 
 	if (!m_cell_ptr) {
-		THROW_ERRNO("unable to mmap " << filename);
+		THROW_ERROR(runtime_error, "unable to mmap " << filename);
 	}
 
-	// Do unions work the way we think (and rely on)?
+	if (!using_shared_map()) {
-	THROW_CHECK2(runtime_error, m_void_ptr, m_cell_ptr, m_void_ptr == m_cell_ptr);
+		// madvise fails if MAP_SHARED
-	THROW_CHECK2(runtime_error, m_void_ptr, m_byte_ptr, m_void_ptr == m_byte_ptr);
+		if (using_any_madvise()) {
-	THROW_CHECK2(runtime_error, m_void_ptr, m_bucket_ptr, m_void_ptr == m_bucket_ptr);
+			// DONTFORK because we sometimes do fork,
-	THROW_CHECK2(runtime_error, m_void_ptr, m_extent_ptr, m_void_ptr == m_extent_ptr);
+			// but the child doesn't touch any of the many, many pages
-
+			BEESTOOLONG("madvise(MADV_HUGEPAGE | MADV_DONTFORK)");
-	// Give all the madvise hints that the kernel understands
+			DIE_IF_NON_ZERO(madvise(m_byte_ptr, m_size, MADV_HUGEPAGE | MADV_DONTFORK));
-	const struct madv_flag {
+		}
-		const char *name;
+		for (uint64_t i = 0; i < m_size / sizeof(Extent); ++i) {
-		int value;
+			m_buckets_missing.insert(i);
-	} madv_flags[] = {
-		{ .name = "MADV_HUGEPAGE", .value = MADV_HUGEPAGE },
-		{ .name = "MADV_DONTFORK", .value = MADV_DONTFORK },
-		{ .name = "MADV_DONTDUMP", .value = MADV_DONTDUMP },
-		{ .name = "", .value = 0 },
-	};
-	for (auto fp = madv_flags; fp->value; ++fp) {
-		BEESTOOLONG("madvise(" << fp->name << ")");
-		if (madvise(m_byte_ptr, m_size, fp->value)) {
-			BEESLOG("madvise(..., " << fp->name << "): " << strerror(errno) << " (ignored)");
 		}
-	}
-
-	for (uint64_t i = 0; i < m_size / sizeof(Extent); ++i) {
-		m_buckets_missing.insert(i);
 	}
 
 	m_writeback_thread.exec([&]() {

src/bees-resolve.cc

@@ -105,7 +105,7 @@ BeesResolver::adjust_offset(const BeesFileRange &haystack, const BeesBlockData &
 	bool is_legacy = false;
 	if (m_addr.is_compressed()) {
 		BtrfsExtentWalker ew(haystack.fd(), haystack.begin(), m_ctx->root_fd());
-		BEESTRACE("haystack extent data " << ew);
+		BEESTRACE("haystack extent data " << ew); 
 		Extent e = ew.current();
 		if (m_addr.has_compressed_offset()) {
 			off_t coff = m_addr.get_compressed_offset();
@@ -196,7 +196,7 @@ BeesResolver::chase_extent_ref(const BtrfsInodeOffsetRoot &bior, BeesBlockData &
 
 	Fd file_fd = m_ctx->roots()->open_root_ino(bior.m_root, bior.m_inum);
 	if (!file_fd) {
-		// Deleted snapshots generate craptons of these
+		// Delete snapshots generate craptons of these
 		// BEESINFO("No FD in chase_extent_ref " << bior);
 		BEESCOUNT(chase_no_fd);
 		return BeesFileRange();
@@ -378,10 +378,7 @@ BeesResolver::for_each_extent_ref(BeesBlockData bbd, function<bool(const BeesFil
 				// We have reliable block addresses now, so we guarantee we can hit the desired block.
 				// Failure in chase_extent_ref means we are done, and don't need to look up all the
 				// other references.
-				// Or...not?  If we have a compressed extent, some refs will not match
+				stop_now = true;
-				// if there is are two references to the same extent with a reference
-				// to a different extent between them.
-				// stop_now = true;
 			}
 		});
 
@@ -480,6 +477,11 @@ BeesResolver::find_all_matches(BeesBlockData &bbd)
 bool
 BeesResolver::operator<(const BeesResolver &that) const
 {
-	// Lowest count, highest address
+	if (that.m_bior_count < m_bior_count) {
-	return tie(that.m_bior_count, m_addr) < tie(m_bior_count, that.m_addr);
+		return true;
+	} else if (m_bior_count < that.m_bior_count) {
+		return false;
+	}
+	return m_addr < that.m_addr;
 }
+

src/bees-roots.cc

@@ -42,26 +42,17 @@ BeesCrawlState::BeesCrawlState() :
 bool
 BeesCrawlState::operator<(const BeesCrawlState &that) const
 {
-	return tie(m_objectid, m_offset, m_root, m_min_transid, m_max_transid)
+	return tie(m_root, m_objectid, m_offset, m_min_transid, m_max_transid)
-		< tie(that.m_objectid, that.m_offset, that.m_root, that.m_min_transid, that.m_max_transid);
+		< tie(that.m_root, that.m_objectid, that.m_offset, that.m_min_transid, that.m_max_transid);
 }
 
 string
 BeesRoots::crawl_state_filename() const
 {
 	string rv;
-
-	// Legacy filename included UUID
 	rv += "beescrawl.";
 	rv += m_ctx->root_uuid();
 	rv += ".dat";
-
-	struct stat buf;
-	if (fstatat(m_ctx->home_fd(), rv.c_str(), &buf, AT_SYMLINK_NOFOLLOW)) {
-		// Use new filename
-		rv = "beescrawl.dat";
-	}
-
 	return rv;
 }
 
@@ -110,12 +101,6 @@ BeesRoots::state_save()
 
 	m_crawl_state_file.write(ofs.str());
 
-	// Renaming things is hard after release
-	if (m_crawl_state_file.name() != "beescrawl.dat") {
-		renameat(m_ctx->home_fd(), m_crawl_state_file.name().c_str(), m_ctx->home_fd(), "beescrawl.dat");
-		m_crawl_state_file.name("beescrawl.dat");
-	}
-
 	BEESNOTE("relocking crawl state");
 	lock.lock();
 	// Not really correct but probably close enough
@@ -208,15 +193,15 @@ BeesRoots::crawl_roots()
 	auto crawl_map_copy = m_root_crawl_map;
 	lock.unlock();
 
-#if 0
-	// Scan the same inode/offset tuple in each subvol (good for snapshots)
 	BeesFileRange first_range;
 	shared_ptr<BeesCrawl> first_crawl;
 	for (auto i : crawl_map_copy) {
 		auto this_crawl = i.second;
 		auto this_range = this_crawl->peek_front();
 		if (this_range) {
-			if (!first_range || this_range < first_range) {
+			auto tuple_this = make_tuple(this_range.fid().ino(), this_range.fid().root(), this_range.begin());
+			auto tuple_first = make_tuple(first_range.fid().ino(), first_range.fid().root(), first_range.begin());
+			if (!first_range || tuple_this < tuple_first) {
 				first_crawl = this_crawl;
 				first_range = this_range;
 			}
@@ -234,27 +219,6 @@ BeesRoots::crawl_roots()
 		THROW_CHECK2(runtime_error, first_range, first_range_popped, first_range == first_range_popped);
 		return;
 	}
-#else
-	// Scan each subvol one extent at a time (good for continuous forward progress)
-	bool crawled = false;
-	for (auto i : crawl_map_copy) {
-		auto this_crawl = i.second;
-		auto this_range = this_crawl->peek_front();
-		if (this_range) {
-			catch_all([&]() {
-				// BEESINFO("scan_forward " << this_range);
-				m_ctx->scan_forward(this_range);
-			});
-			crawled = true;
-			BEESCOUNT(crawl_scan);
-			m_crawl_current = this_crawl->get_state();
-			auto this_range_popped = this_crawl->pop_front();
-			THROW_CHECK2(runtime_error, this_range, this_range_popped, this_range == this_range_popped);
-		}
-	}
-
-	if (crawled) return;
-#endif
 
 	BEESLOG("Crawl ran out of data after " << m_crawl_timer.lap() << "s, waiting for more...");
 	BEESCOUNT(crawl_done);
@@ -379,8 +343,8 @@ BeesRoots::state_load()
 BeesRoots::BeesRoots(shared_ptr<BeesContext> ctx) :
 	m_ctx(ctx),
 	m_crawl_state_file(ctx->home_fd(), crawl_state_filename()),
-	m_crawl_thread("crawl"),
+	m_crawl_thread("crawl " + ctx->root_path()),
-	m_writeback_thread("crawl_writeback")
+	m_writeback_thread("crawl_writeback " + ctx->root_path())
 {
 	m_crawl_thread.exec([&]() {
 		catch_all([&]() {
@@ -665,7 +629,7 @@ BeesCrawl::fetch_extents()
 
 	Timer crawl_timer;
 
-	BtrfsIoctlSearchKey sk(BEES_MAX_CRAWL_SIZE * (sizeof(btrfs_file_extent_item) + sizeof(btrfs_ioctl_search_header)));
+	BtrfsIoctlSearchKey sk;
 	sk.tree_id = old_state.m_root;
 	sk.min_objectid = old_state.m_objectid;
 	sk.min_type = sk.max_type = BTRFS_EXTENT_DATA_KEY;
@@ -682,9 +646,7 @@ BeesCrawl::fetch_extents()
 	{
 		BEESNOTE("searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));
 		BEESTOOLONG("Searching crawl sk " << static_cast<btrfs_ioctl_search_key&>(sk));
-		Timer crawl_timer;
 		ioctl_ok = sk.do_ioctl_nothrow(m_ctx->root_fd());
-		BEESCOUNTADD(crawl_ms, crawl_timer.age() * 1000);
 	}
 
 	if (ioctl_ok) {
@@ -768,7 +730,6 @@ BeesCrawl::fetch_extents()
 				break;
 			case BTRFS_FILE_EXTENT_PREALLOC:
 				BEESCOUNT(crawl_prealloc);
-				// fallthrough
 			case BTRFS_FILE_EXTENT_REG: {
 				auto physical = call_btrfs_get(btrfs_stack_file_extent_disk_bytenr, i.m_data);
 				auto ram = call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data);
@@ -823,8 +784,7 @@ BeesCrawl::peek_front()
 	if (m_extents.empty()) {
 		return BeesFileRange();
 	}
-	auto rv = *m_extents.begin();
+	return *m_extents.begin();
-	return rv;
 }
 
 BeesFileRange
@@ -850,8 +810,7 @@ BeesCrawlState
 BeesCrawl::get_state()
 {
 	unique_lock<mutex> lock(m_state_mutex);
-	auto rv = m_state;
+	return m_state;
-	return rv;
 }
 
 void

src/bees-types.cc

@@ -286,8 +286,7 @@ Fd
 BeesFileRange::fd() const
 {
 	unique_lock<mutex> lock(s_mutex);
-	auto rv = m_fd;
+	return m_fd;
-	return rv;
 }
 
 Fd
@@ -311,8 +310,7 @@ BeesFileRange::fd(const shared_ptr<BeesContext> &ctx) const
 		}
 	}
 	// We either had a fid and opened it, or we didn't and we're just stuck with our fd
-	auto rv = m_fd;
+	return m_fd;
-	return rv;
 }
 
 BeesFileRange

src/bees.cc

@@ -1,5 +1,6 @@
 #include "bees.h"
 
+#include "crucible/interp.h"
 #include "crucible/limits.h"
 #include "crucible/process.h"
 #include "crucible/string.h"
@@ -23,7 +24,7 @@ using namespace crucible;
 using namespace std;
 
 int
-do_cmd_help(const char **argv)
+do_cmd_help(const ArgList &argv)
 {
 	cerr << "Usage: " << argv[0] << " fs-root-path [fs-root-path-2...]\n"
 		"Performs best-effort extent-same deduplication on btrfs.\n"
@@ -31,12 +32,15 @@ do_cmd_help(const char **argv)
 		"fs-root-path MUST be the root of a btrfs filesystem tree (id 5).\n"
 		"Other directories will be rejected.\n"
 		"\n"
-		"Optional environment variables:\n"
+		"Multiple filesystems can share a single hash table (BEESHOME)\n"
-		"\tBEESHOME\tPath to hash table and configuration files\n"
+		"but this only works well if the content of each filesystem\n"
-		"\t\t\t(default is .beeshome/ in the root of each filesystem).\n"
+		"is distinct from all the others.\n"
 		"\n"
-		"\tBEESSTATUS\tFile to write status to (tmpfs recommended, e.g. /run).\n"
+		"Required environment variables:\n"
-		"\t\t\tNo status is written if this variable is unset.\n"
+		"\tBEESHOME\tPath to hash table and configuration files\n"
+		"\n"
+		"Optional environment variables:\n"
+		"\tBEESSTATUS\tFile to write status to (tmpfs recommended, e.g. /run)\n"
 		"\n"
 	<< endl;
 	return 0;
@@ -46,7 +50,7 @@ do_cmd_help(const char **argv)
 
 RateLimiter bees_info_rate_limit(BEES_INFO_RATE, BEES_INFO_BURST);
 
-thread_local BeesTracer *BeesTracer::tl_next_tracer = nullptr;
+thread_local BeesTracer *BeesTracer::s_next_tracer = nullptr;
 
 BeesTracer::~BeesTracer()
 {
@@ -56,20 +60,20 @@ BeesTracer::~BeesTracer()
 			BEESLOG("---  END  TRACE --- exception ---");
 		}
 	}
-	tl_next_tracer = m_next_tracer;
+	s_next_tracer = m_next_tracer;
 }
 
 BeesTracer::BeesTracer(function<void()> f) :
 	m_func(f)
 {
-	m_next_tracer = tl_next_tracer;
+	m_next_tracer = s_next_tracer;
-	tl_next_tracer = this;
+	s_next_tracer = this;
 }
 
 void
 BeesTracer::trace_now()
 {
-	BeesTracer *tp = tl_next_tracer;
+	BeesTracer *tp = s_next_tracer;
 	BEESLOG("--- BEGIN TRACE ---");
 	while (tp) {
 		tp->m_func();
@@ -78,17 +82,17 @@ BeesTracer::trace_now()
 	BEESLOG("---  END  TRACE ---");
 }
 
-thread_local BeesNote *BeesNote::tl_next = nullptr;
+thread_local BeesNote *BeesNote::s_next = nullptr;
 mutex BeesNote::s_mutex;
 map<pid_t, BeesNote*> BeesNote::s_status;
-thread_local string BeesNote::tl_name;
+thread_local string BeesNote::s_name;
 
 BeesNote::~BeesNote()
 {
-	tl_next = m_prev;
 	unique_lock<mutex> lock(s_mutex);
-	if (tl_next) {
+	s_next = m_prev;
-		s_status[gettid()] = tl_next;
+	if (s_next) {
+		s_status[gettid()] = s_next;
 	} else {
 		s_status.erase(gettid());
 	}
@@ -97,26 +101,28 @@ BeesNote::~BeesNote()
 BeesNote::BeesNote(function<void(ostream &os)> f) :
 	m_func(f)
 {
-	m_name = tl_name;
-	m_prev = tl_next;
-	tl_next = this;
 	unique_lock<mutex> lock(s_mutex);
-	s_status[gettid()] = tl_next;
+	m_name = s_name;
+	m_prev = s_next;
+	s_next = this;
+	s_status[gettid()] = s_next;
 }
 
 void
 BeesNote::set_name(const string &name)
 {
-	tl_name = name;
+	unique_lock<mutex> lock(s_mutex);
+	s_name = name;
 }
 
 string
 BeesNote::get_name()
 {
-	if (tl_name.empty()) {
+	unique_lock<mutex> lock(s_mutex);
+	if (s_name.empty()) {
 		return "bees";
 	} else {
-		return tl_name;
+		return s_name;
 	}
 }
 
@@ -202,10 +208,11 @@ template <class T>
 T&
 BeesStatTmpl<T>::at(string idx)
 {
-	if (!m_stats_map.count(idx)) {
+	unique_lock<mutex> lock(m_mutex);
+    if (!m_stats_map.count(idx)) {
 		m_stats_map[idx] = 0;
 	}
-	return m_stats_map[idx];
+    return m_stats_map[idx];
 }
 
 template <class T>
@@ -213,8 +220,7 @@ T
 BeesStatTmpl<T>::at(string idx) const
 {
 	unique_lock<mutex> lock(m_mutex);
-	auto rv = m_stats_map.at(idx);
+    return m_stats_map.at(idx);
-	return rv;
 }
 
 template <class T>
@@ -222,7 +228,7 @@ void
 BeesStatTmpl<T>::add_count(string idx, size_t amount)
 {
 	unique_lock<mutex> lock(m_mutex);
-	if (!m_stats_map.count(idx)) {
+    if (!m_stats_map.count(idx)) {
 		m_stats_map[idx] = 0;
 	}
 	m_stats_map.at(idx) += amount;
@@ -254,17 +260,14 @@ BeesStats
 BeesStats::operator-(const BeesStats &that) const
 {
 	if (&that == this) return BeesStats();
-
 	unique_lock<mutex> this_lock(m_mutex);
 	BeesStats this_copy;
 	this_copy.m_stats_map = m_stats_map;
-	this_lock.unlock();
-
 	unique_lock<mutex> that_lock(that.m_mutex);
 	BeesStats that_copy;
 	that_copy.m_stats_map = that.m_stats_map;
+	this_lock.unlock();
 	that_lock.unlock();
-
 	for (auto i : that.m_stats_map) {
 		if (i.second != 0) {
 			this_copy.at(i.first) -= i.second;
@@ -348,18 +351,6 @@ BeesStringFile::BeesStringFile(Fd dir_fd, string name, size_t limit) :
 	BEESLOG("BeesStringFile " << name_fd(m_dir_fd) << "/" << m_name << " max size " << pretty(m_limit));
 }
 
-void
-BeesStringFile::name(const string &new_name)
-{
-	m_name = new_name;
-}
-
-string
-BeesStringFile::name() const
-{
-	return m_name;
-}
-
 string
 BeesStringFile::read()
 {
@@ -393,13 +384,8 @@ BeesStringFile::write(string contents)
 		Fd ofd = openat_or_die(m_dir_fd, tmpname, FLAGS_CREATE_FILE, S_IRUSR | S_IWUSR);
 		BEESNOTE("writing " << tmpname << " in " << name_fd(m_dir_fd));
 		write_or_die(ofd, contents);
-#if 0
-		// This triggers too many btrfs bugs.  I wish I was kidding.
-		// Forget snapshots, balance, compression, and dedup:
-		// the system call you have to fear on btrfs is fsync().
 		BEESNOTE("fsyncing " << tmpname << " in " << name_fd(m_dir_fd));
 		DIE_IF_NON_ZERO(fsync(ofd));
-#endif
 	}
 	BEESNOTE("renaming " << tmpname << " to " << m_name << " in FD " << name_fd(m_dir_fd));
 	BEESTRACE("renaming " << tmpname << " to " << m_name << " in FD " << name_fd(m_dir_fd));
@@ -461,7 +447,7 @@ BeesTempFile::BeesTempFile(shared_ptr<BeesContext> ctx) :
 
 void
 BeesTempFile::realign()
-{
+{ 
 	if (m_end_offset > BLOCK_SIZE_MAX_TEMP_FILE) {
 		BEESLOG("temporary file size " << to_hex(m_end_offset) << " > max " << BLOCK_SIZE_MAX_TEMP_FILE);
 		BEESCOUNT(tmp_trunc);
@@ -503,13 +489,8 @@ BeesTempFile::make_copy(const BeesFileRange &src)
 
 	THROW_CHECK1(invalid_argument, src, src.size() > 0);
 
-	// FIEMAP used to give us garbage data, e.g. distinct adjacent
+	// FIXME:  don't know where these come from, but we can't handle them.
-	// extents merged into a single entry in the FIEMAP output.
+	// Grab a trace for the log.
-	// FIEMAP didn't stop giving us garbage data, we just stopped
-	// using FIEMAP.
-	// We shouldn't get absurdly large extents any more; however,
-	// it's still a problem if we do, so bail out and leave a trace
-	// in the log.
 	THROW_CHECK1(invalid_argument, src, src.size() < BLOCK_SIZE_MAX_TEMP_FILE);
 
 	realign();
@@ -554,7 +535,7 @@ BeesTempFile::make_copy(const BeesFileRange &src)
 }
 
 int
-bees_main(int argc, const char **argv)
+bees_main(ArgList args)
 {
 	set_catch_explainer([&](string s) {
 		BEESLOG("\n\n*** EXCEPTION ***\n\t" << s << "\n***\n");
@@ -567,10 +548,7 @@ bees_main(int argc, const char **argv)
 	list<shared_ptr<BeesContext>> all_contexts;
 	shared_ptr<BeesContext> bc;
 
-	THROW_CHECK1(invalid_argument, argc, argc >= 0);
+	// Subscribe to fanotify events
-	vector<string> args(argv + 1, argv + argc);
-
-	// Create a context and start crawlers
 	bool did_subscription = false;
 	for (string arg : args) {
 		catch_all([&]() {
@@ -598,16 +576,16 @@ bees_main(int argc, const char **argv)
 int
 main(int argc, const char **argv)
 {
-	cerr << "bees version " << BEES_VERSION << endl;
-
 	if (argc < 2) {
 		do_cmd_help(argv);
 		return 2;
 	}
 
+	ArgList args(argv + 1);
+
 	int rv = 1;
 	catch_and_explain([&]() {
-		rv = bees_main(argc, argv);
+		rv = bees_main(args);
 	});
 	return rv;
 }

src/bees.h

@@ -1,6 +1,7 @@
 #ifndef BEES_H
 #define BEES_H
 
+#include "crucible/bool.h"
 #include "crucible/cache.h"
 #include "crucible/chatter.h"
 #include "crucible/error.h"
@@ -135,8 +136,6 @@ const int FLAGS_OPEN_FANOTIFY = O_RDWR | O_NOATIME | O_CLOEXEC | O_LARGEFILE;
 	} \
 } while (0)
 
-#define BEESLOGNOTE(x) BEESLOG(x); BEESNOTE(x)
-
 #define BEESCOUNT(stat) do { \
 	BeesStats::s_global.add_count(#stat); \
 } while (0)
@@ -155,12 +154,12 @@ class BeesStatTmpl {
 	map<string, T>	m_stats_map;
 	mutable mutex	m_mutex;
 
-	T& at(string idx);
 public:
 	BeesStatTmpl() = default;
 	BeesStatTmpl(const BeesStatTmpl &that);
 	BeesStatTmpl &operator=(const BeesStatTmpl &that);
 	void add_count(string idx, size_t amount = 1);
+	T& at(string idx);
 	T at(string idx) const;
 
 friend ostream& operator<< <>(ostream &os, const BeesStatTmpl<T> &bs);
@@ -184,7 +183,7 @@ class BeesTracer {
 	function<void()> m_func;
 	BeesTracer *m_next_tracer = 0;
 	
-	thread_local static BeesTracer *tl_next_tracer;
+	thread_local static BeesTracer *s_next_tracer;
 public:
 	BeesTracer(function<void()> f);
 	~BeesTracer();
@@ -200,8 +199,8 @@ class BeesNote {
 	static mutex			s_mutex;
 	static map<pid_t, BeesNote*>	s_status;
 
-	thread_local static BeesNote	*tl_next;
+	thread_local static BeesNote	*s_next;
-	thread_local static string	tl_name;
+	thread_local static string	s_name;
 
 public:
 	BeesNote(function<void(ostream &)> f);
@@ -375,8 +374,6 @@ public:
 	BeesStringFile(Fd dir_fd, string name, size_t limit = 1024 * 1024);
 	string read();
 	void write(string contents);
-	void name(const string &new_name);
-	string name() const;
 };
 
 class BeesHashTable {
@@ -410,7 +407,7 @@ public:
 		uint8_t	p_byte[BLOCK_SIZE_HASHTAB_EXTENT];
 	} __attribute__((packed));
 
-	BeesHashTable(shared_ptr<BeesContext> ctx, string filename, off_t size = BLOCK_SIZE_HASHTAB_EXTENT);
+	BeesHashTable(shared_ptr<BeesContext> ctx, string filename);
 	~BeesHashTable();
 
 	vector<Cell>	find_cell(HashType hash);
@@ -418,6 +415,8 @@ public:
 	void		erase_hash_addr(HashType hash, AddrType addr);
 	bool		push_front_hash_addr(HashType hash, AddrType addr);
 
+	void		set_shared(bool shared);
+
 private:
 	string		m_filename;
 	Fd		m_fd;
@@ -444,6 +443,7 @@ private:
 	BeesThread  		m_writeback_thread;
 	BeesThread	        m_prefetch_thread;
 	RateLimiter		m_flush_rate_limit;
+	RateLimiter		m_prefetch_rate_limit;
 	mutex			m_extent_mutex;
 	mutex			m_bucket_mutex;
 	condition_variable	m_condvar;
@@ -452,7 +452,8 @@ private:
 
 	LockSet<uint64_t> 	m_extent_lock_set;
 
-	void open_file();
+	DefaultBool		m_shared;
+
 	void writeback_loop();
 	void prefetch_loop();
 	void try_mmap_flags(int flags);
@@ -463,6 +464,8 @@ private:
 	void flush_dirty_extents();
 	bool is_toxic_hash(HashType h) const;
 
+	bool using_shared_map() const { return false; }
+
 	BeesHashTable(const BeesHashTable &) = delete;
 	BeesHashTable &operator=(const BeesHashTable &) = delete;
 };
@@ -485,7 +488,7 @@ class BeesCrawl {
 
 	mutex					m_mutex;
 	set<BeesFileRange>			m_extents;
-	bool					m_deferred = false;
+	DefaultBool				m_deferred;
 
 	mutex					m_state_mutex;
 	BeesCrawlState				m_state;
@@ -510,7 +513,7 @@ class BeesRoots {
 	map<uint64_t, shared_ptr<BeesCrawl>>	m_root_crawl_map;
 	mutex					m_mutex;
 	condition_variable			m_condvar;
-	bool					m_crawl_dirty = false;
+	DefaultBool				m_crawl_dirty;
 	Timer					m_crawl_timer;
 	BeesThread				m_crawl_thread;
 	BeesThread				m_writeback_thread;
@@ -566,7 +569,7 @@ class BeesBlockData {
 	mutable BeesAddress	m_addr;
 	mutable Blob		m_data;
 	mutable BeesHash	m_hash;
-	mutable bool		m_hash_done = false;
+	mutable DefaultBool	m_hash_done;
 
 public:
 	// Constructor with the immutable fields
@@ -605,7 +608,7 @@ friend ostream & operator<<(ostream &os, const BeesRangePair &brp);
 };
 
 class BeesWorkQueueBase {
-	string 				m_name;
+	string 				m_name; 
 
 protected:
 	static mutex			s_mutex;
@@ -620,7 +623,7 @@ public:
 
 	virtual size_t active_size() const = 0;
 	virtual list<string> peek_active(size_t count) const = 0;
-
+ 
 	static void for_each_work_queue(function<void(BeesWorkQueueBase *)> f);
 };
 
@@ -659,7 +662,6 @@ class BeesFdCache {
 	LRUCache<Fd, shared_ptr<BeesContext>, uint64_t>			m_root_cache;
 	LRUCache<Fd, shared_ptr<BeesContext>, uint64_t, uint64_t>	m_file_cache;
 	Timer								m_root_cache_timer;
-	Timer								m_file_cache_timer;
 
 public:
 	BeesFdCache();
@@ -671,7 +673,7 @@ public:
 struct BeesResolveAddrResult {
 	BeesResolveAddrResult();
 	vector<BtrfsInodeOffsetRoot> m_biors;
-	bool m_is_toxic = false;
+	DefaultBool m_is_toxic;
 	bool is_toxic() const { return m_is_toxic; }
 };
 
@@ -712,7 +714,7 @@ public:
 	void set_root_path(string path);
 
 	Fd root_fd() const { return m_root_fd; }
-	Fd home_fd();
+	Fd home_fd() const { return m_home_fd; }
 	string root_path() const { return m_root_path; }
 	string root_uuid() const { return m_root_uuid; }
 
@@ -749,22 +751,22 @@ class BeesResolver {
 	unsigned				m_bior_count;
 
 	// We found matching data, so we can dedup
-	bool					m_found_data = false;
+	DefaultBool				m_found_data;
 
 	// We found matching data, so we *did* dedup
-	bool					m_found_dup = false;
+	DefaultBool				m_found_dup;
 
 	// We found matching hash, so the hash table is still correct
-	bool					m_found_hash = false;
+	DefaultBool				m_found_hash;
 
 	// We found matching physical address, so the hash table isn't totally wrong
-	bool					m_found_addr = false;
+	DefaultBool				m_found_addr;
 
 	// We found matching physical address, but data did not match
-	bool					m_wrong_data = false;
+	DefaultBool				m_wrong_data;
 
 	// The whole thing is a placebo to avoid crippling btrfs performance bugs
-	bool					m_is_toxic = false;
+	DefaultBool				m_is_toxic;
 
 	BeesFileRange chase_extent_ref(const BtrfsInodeOffsetRoot &bior, BeesBlockData &needle_bbd);
 	BeesBlockData adjust_offset(const BeesFileRange &haystack, const BeesBlockData &needle);
@@ -818,7 +820,6 @@ public:
 };
 
 // And now, a giant pile of extern declarations
-extern const char *BEES_VERSION;
 string pretty(double d);
 extern RateLimiter bees_info_rate_limit;
 void bees_sync(int fd);

src/fanotify-watch.cc

@@ -0,0 +1,91 @@
+#include <crucible/error.h>
+#include <crucible/fd.h>
+#include <crucible/ntoa.h>
+
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+#include <unistd.h>
+#include <sys/fanotify.h>
+
+using namespace crucible;
+using namespace std;
+
+static
+void
+usage(const char *name)
+{
+	cerr << "Usage: " << name << " directory" << endl;
+	cerr << "Reports fanotify events from directory" << endl;
+}
+
+struct fan_read_block {
+	struct fanotify_event_metadata fem;
+	// more here in the future.  Maybe.
+};
+
+static inline
+string
+fan_flag_ntoa(uint64_t ui)
+{
+	static const bits_ntoa_table flag_names[] = {
+		NTOA_TABLE_ENTRY_BITS(FAN_ACCESS),
+		NTOA_TABLE_ENTRY_BITS(FAN_OPEN),
+		NTOA_TABLE_ENTRY_BITS(FAN_MODIFY),
+		NTOA_TABLE_ENTRY_BITS(FAN_CLOSE),
+		NTOA_TABLE_ENTRY_BITS(FAN_CLOSE_WRITE),
+		NTOA_TABLE_ENTRY_BITS(FAN_CLOSE_NOWRITE),
+		NTOA_TABLE_ENTRY_BITS(FAN_Q_OVERFLOW),
+		NTOA_TABLE_ENTRY_BITS(FAN_ACCESS_PERM),
+		NTOA_TABLE_ENTRY_BITS(FAN_OPEN_PERM),
+		NTOA_TABLE_ENTRY_END()
+	};
+	return bits_ntoa(ui, flag_names);
+}
+
+int
+main(int argc, char **argv)
+{
+	if (argc < 1) {
+		usage(argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	Fd fd;
+
+	DIE_IF_MINUS_ONE(fd = fanotify_init(FAN_CLASS_NOTIF, O_RDONLY | O_LARGEFILE | O_CLOEXEC | O_NOATIME));
+
+	for (char **argvp = argv + 1; *argvp; ++argvp) {
+		cerr << "fanotify_mark(" << *argvp << ")..." << flush;
+		DIE_IF_MINUS_ONE(fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT, FAN_CLOSE_WRITE | FAN_CLOSE_NOWRITE | FAN_OPEN, FAN_NOFD, *argvp));
+		cerr << endl;
+	}
+
+	while (1) {
+		struct fan_read_block frb;
+		read_or_die(fd, frb);
+
+#if 0
+		cout << "event_len\t= " << frb.fem.event_len << endl;
+		cout << "vers\t= " << static_cast<int>(frb.fem.vers) << endl;
+		cout << "reserved\t= " << static_cast<int>(frb.fem.reserved) << endl;
+		cout << "metadata_len\t= " << frb.fem.metadata_len << endl;
+		cout << "mask\t= " << hex << frb.fem.mask << dec << "\t" << fan_flag_ntoa(frb.fem.mask) << endl;
+		cout << "fd\t= " << frb.fem.fd << endl;
+		cout << "pid\t= " << frb.fem.pid << endl;
+#endif
+
+		cout << "flags " << fan_flag_ntoa(frb.fem.mask) << " pid " << frb.fem.pid << ' ' << flush;
+
+		Fd event_fd(frb.fem.fd);
+		ostringstream oss;
+		oss << "/proc/self/fd/" << event_fd;
+		cout << "file " << readlink_or_die(oss.str()) << endl;
+
+		// cout << endl;
+	}
+	
+	return EXIT_SUCCESS;
+}

test/Makefile

@@ -1,7 +1,9 @@
 PROGRAMS = \
 	chatter \
 	crc64 \
+	execpipe \
 	fd \
+	interp \
 	limits \
 	path \
 	process \
@@ -19,7 +21,7 @@ LDFLAGS = -L../lib -Wl,-rpath=$(shell realpath ../lib)
 depends.mk: *.cc
 	for x in *.cc; do $(CXX) $(CXXFLAGS) -M "$$x"; done >> depends.mk.new
 	mv -fv depends.mk.new depends.mk
-
+                                       
 -include depends.mk
 
 %.o: %.cc %.h ../makeflags

test/crc64.cc

@@ -5,6 +5,18 @@
 
 using namespace crucible;
 
+static
+void
+test_getcrc64_strings()
+{
+	assert(Digest::CRC::crc64("John") == 5942451273432301568);
+	assert(Digest::CRC::crc64("Paul") == 5838402100630913024);
+	assert(Digest::CRC::crc64("George") == 6714394476893704192);
+	assert(Digest::CRC::crc64("Ringo") == 6038837226071130112);
+	assert(Digest::CRC::crc64("") == 0);
+	assert(Digest::CRC::crc64("\377\277\300\200") == 15615382887346470912ULL);
+}
+
 static
 void
 test_getcrc64_byte_arrays()
@@ -20,6 +32,7 @@ test_getcrc64_byte_arrays()
 int
 main(int, char**)
 {
+	RUN_A_TEST(test_getcrc64_strings());
 	RUN_A_TEST(test_getcrc64_byte_arrays());
 
 	exit(EXIT_SUCCESS);

test/execpipe.cc

@@ -0,0 +1,64 @@
+#include "tests.h"
+
+#include "crucible/execpipe.h"
+
+#include <ios>
+#include <cassert>
+#include <cstring>
+#include <cstdlib>
+#include <stdexcept>
+
+#include <unistd.h>
+
+using namespace crucible;
+using namespace std;
+
+#if 1 // Needs rework
+static inline
+void
+test_hello_world()
+{
+	// alarm(9);
+	Fd fd = popen([]() { return system("echo Hello, World!"); });
+	char buf[1024];
+	size_t rv = -1;
+	read_partial_or_die(fd, buf, rv);
+	assert(rv > 0);
+	string b(buf, buf + rv - 1);
+	// cerr << "hello_world says: '" << b << "'" << endl;
+	assert(b == "Hello, World!");
+}
+
+static inline
+void
+test_read_limit(size_t limit = 4096)
+{
+	alarm(9);
+	Fd fd = popen([]() { return system("yes Hello!"); });
+	try {
+		string b = read_all(fd, limit);
+	} catch (out_of_range &re) {
+		return;
+	}
+	assert(!"no exception thrown by read_all");
+}
+#endif
+
+namespace crucible {
+	extern bool assert_no_leaked_fds();
+};
+
+int
+main(int, char**)
+{
+#if 1
+	RUN_A_TEST(test_hello_world());
+	assert(assert_no_leaked_fds());
+	RUN_A_TEST(test_read_limit(4095));
+	RUN_A_TEST(test_read_limit(4096));
+	RUN_A_TEST(test_read_limit(4097));
+	assert(assert_no_leaked_fds());
+#endif
+
+	exit(EXIT_SUCCESS);
+}

test/interp.cc

@@ -0,0 +1,88 @@
+#include "tests.h"
+
+#include "crucible/interp.h"
+
+using namespace crucible;
+using namespace std;
+
+/***********************************************************************
+
+How this should work:
+
+Interpreter reads an arg list:
+
+	argv[0] --method0args --method1arg arg1 --method1arg=arg1 -- args...
+
+argv[0] should look up a shared_ptr<Command> which creates an object of
+type shared_ptr<Process>.  This object is used to receive args by
+method calls or one at a time.
+
+<Command> and <Process> can be the same object, or not.
+
+Process p methods:
+
+	p->spawn(Interp*) -> Process
+	p->exec(ArgList) -> Process / Result
+	p->method (from ArgParser<>)
+		p->finish() -> void (destroys object without early destruction warnings...?)
+		p->~Process() -> complains loudly if finish() not called first...?
+
+Result might be a pair of Process, string.  Or just string.
+
+ArgParser should be more like GetOpt:
+
+	build a dictionary and an arg list from arguments
+	Process methods should interrogate ArgParser
+	ArgParser might have a table of boolean and string option names so it can reject invalid options
+		but if it had that, we could also pass in Process and have it call methods on it
+		...but that is a _lot_ of pointer-hiding when we could KISS
+		...but if we had that solved, argparser tables look like lists of method names
+	ArgParser<T> has a table of names and methods on object of type T
+		ArgParser hides everything behind void* and hands off to a compiled implementation to do callbacks
+
+Extreme simplification:  arguments are themselves executable
+
+	so '--method_foo arg' really means construct MethodFoo(arg) and cast to shared_ptr<ProcArg>
+	then Process->invokeSomething(ProcArg)
+	too extreme, use argparser instead
+
+***********************************************************************/
+
+void
+test_arg_parser()
+{
+	ArgParser ap;
+	ArgList al( { "abc", "--def", "ghi" } );
+	ap.parse(NULL, al);
+}
+
+struct Thing {
+	int m_i;
+	double m_d;
+	string m_s;
+
+	void set_i(int i) { cerr << "i = " << i << endl; m_i = i; }
+	void set_d(double d) { cerr << "d = " << d << endl; m_d = d; }
+	void set_s(string s) { cerr << "s = " << s << endl; m_s = s; }
+};
+
+template <typename F, typename T, typename A>
+void
+assign(T& t, F f, A a)
+{
+	cerr << __PRETTY_FUNCTION__ << " - a = " << a << endl;
+	(t.*f)(a);
+}
+
+int
+main(int, char**)
+{
+	RUN_A_TEST(test_arg_parser());
+
+	Thing p;
+	assign(p, &Thing::set_i, 5);
+
+	cerr << "p.m_i = " << p.m_i << endl;
+
+	exit(EXIT_SUCCESS);
+}

test/limits.cc

@@ -141,13 +141,7 @@ test_cast_0x80000000_to_things()
 	SHOULD_FAIL(ranged_cast<unsigned short>(uv));
 	SHOULD_FAIL(ranged_cast<unsigned char>(uv));
 	SHOULD_PASS(ranged_cast<signed long long>(sv), sv);
-	if (sizeof(long) == 4) {
+	SHOULD_PASS(ranged_cast<signed long>(sv), sv);
-		SHOULD_FAIL(ranged_cast<signed long>(sv));
-	} else if (sizeof(long) == 8) {
-		SHOULD_PASS(ranged_cast<signed long>(sv), sv);
-	} else {
-		assert(!"unhandled case, please add code for long here");
-	}
 	SHOULD_FAIL(ranged_cast<signed short>(sv));
 	SHOULD_FAIL(ranged_cast<signed char>(sv));
 	if (sizeof(int) == 4) {
@@ -155,7 +149,7 @@ test_cast_0x80000000_to_things()
 	} else if (sizeof(int) == 8) {
 		SHOULD_PASS(ranged_cast<signed int>(sv), sv);
 	} else {
-		assert(!"unhandled case, please add code for int here");
+		assert(!"unhandled case, please add code here");
 	}
 }
 
@@ -180,13 +174,7 @@ test_cast_0xffffffff_to_things()
 	SHOULD_FAIL(ranged_cast<unsigned short>(uv));
 	SHOULD_FAIL(ranged_cast<unsigned char>(uv));
 	SHOULD_PASS(ranged_cast<signed long long>(sv), sv);
-	if (sizeof(long) == 4) {
+	SHOULD_PASS(ranged_cast<signed long>(sv), sv);
-		SHOULD_FAIL(ranged_cast<signed long>(sv));
-	} else if (sizeof(long) == 8) {
-		SHOULD_PASS(ranged_cast<signed long>(sv), sv);
-	} else {
-		assert(!"unhandled case, please add code for long here");
-	}
 	SHOULD_FAIL(ranged_cast<signed short>(sv));
 	SHOULD_FAIL(ranged_cast<signed char>(sv));
 	if (sizeof(int) == 4) {
@@ -194,7 +182,7 @@ test_cast_0xffffffff_to_things()
 	} else if (sizeof(int) == 8) {
 		SHOULD_PASS(ranged_cast<signed int>(sv), sv);
 	} else {
-		assert(!"unhandled case, please add code for int here");
+		assert(!"unhandled case, please add code here");
 	}
 }