bees: remove local cruft, throw at github

include/crucible/backtrace.h

@@ -0,0 +1,29 @@
+#ifndef CRUCIBLE_BACKTRACE_H
+#define CRUCIBLE_BACKTRACE_H
+
+#include <string>
+#include <vector>
+
+#include <execinfo.h>
+
+namespace crucible {
+	using namespace std;
+
+	class Backtrace {
+		vector<void *> m_buffer;
+		mutable vector<string> m_result_stringvec;
+		mutable char **m_result_cpp;
+		int m_result_size;
+		int m_desired_size;
+	public:
+		Backtrace(int size = 99);
+		~Backtrace();
+		const vector<string> &strings() const;
+		const vector<void *> &voids() const;
+		void symbols_fd(int fd) const;
+		bool overflowed() const;
+	};
+
+}
+
+#endif // CRUCIBLE_BACKTRACE_H

include/crucible/bencode.h

@@ -0,0 +1,76 @@
+#ifndef CRUCIBLE_BENCODE_H
+#define CRUCIBLE_BENCODE_H
+
+#include "crucible/error.h"
+
+#include <cctype>
+
+#include <fstream>
+#include <map>
+#include <memory>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace crucible {
+	using namespace std;
+
+	// So...much...forward declaration...
+	struct bencode_variant;
+	typedef shared_ptr<bencode_variant> bencode_variant_ptr;
+
+	struct bencode_variant {
+		virtual ~bencode_variant();
+		virtual ostream& print(ostream &os, const string &parent = "") const = 0;
+		virtual bencode_variant_ptr at(size_t i) const;
+		virtual bencode_variant_ptr at(const string &s) const;
+		virtual operator string() const;
+	};
+
+	ostream& operator<<(ostream &os, const bencode_variant_ptr &p);
+
+	// i<base-10-ascii>e
+	struct bencode_int : public bencode_variant {
+		~bencode_int();
+		bencode_int(int64_t i);
+		ostream & print(ostream &os, const string &parent = "") const override;
+	private:
+		int64_t m_i;
+	};
+
+	// <length>:contents
+	struct bencode_string : public bencode_variant {
+		~bencode_string();
+		bencode_string(string s);
+		ostream & print(ostream &os, const string &parent = "") const override;
+		operator string() const override;
+	private:
+		string m_s;
+	};
+
+	// l<contents>e
+	struct bencode_list : public bencode_variant {
+		~bencode_list();
+		bencode_list(const vector<bencode_variant_ptr> &l);
+		ostream & print(ostream &os, const string &parent = "") const override;
+		using bencode_variant::at;
+		bencode_variant_ptr at(size_t i) const override;
+	private:
+		vector<bencode_variant_ptr> m_l;
+	};
+
+	// d<contents>e (lexicographically sorted pairs of <key><value>, key is a string)
+	struct bencode_dict : public bencode_variant {
+		~bencode_dict();
+		bencode_dict(const map<string, bencode_variant_ptr> &m);
+		ostream& print(ostream &os, const string &parent = "") const override;
+		using bencode_variant::at;
+		bencode_variant_ptr at(const string &key) const override;
+	private:
+		map<string, bencode_variant_ptr> m_m;
+	};
+
+	bencode_variant_ptr bencode_decode_stream(istream &is);
+};
+
+#endif

include/crucible/bool.h

@@ -0,0 +1,13 @@
+#ifndef CRUCIBLE_BOOL_H
+#define CRUCIBLE_BOOL_H
+
+namespace crucible {
+	struct DefaultBool {
+		bool m_b;
+		DefaultBool(bool init = false) : m_b(init) {}
+		operator bool() const { return m_b; }
+		bool &operator=(const bool &that) { return m_b = that; }
+	};
+}
+
+#endif // CRUCIBLE_BOOL_H

include/crucible/btrfs.h

@@ -0,0 +1,205 @@
+#ifndef CRUCIBLE_BTRFS_H
+#define CRUCIBLE_BTRFS_H
+
+// Copied from Linux kernel sources as of 3.15 or so.
+// These are probably missing from /usr/include at the moment.
+
+// NULL
+#include <cstdio>
+
+// _IOWR macro and friends
+#include <asm-generic/ioctl.h>
+
+// __u64 typedef and friends
+#include <linux/types.h>
+
+// try Linux headers first
+#include <btrfs/ioctl.h>
+
+// Supply any missing definitions
+#define mutex not_mutex
+#include <btrfs/ctree.h>
+// Repair the damage
+#undef min
+#undef max
+#undef mutex
+
+#ifndef BTRFS_FIRST_FREE_OBJECTID
+
+	#define BTRFS_ROOT_TREE_OBJECTID 1ULL
+	#define BTRFS_EXTENT_TREE_OBJECTID 2ULL
+	#define BTRFS_CHUNK_TREE_OBJECTID 3ULL
+	#define BTRFS_DEV_TREE_OBJECTID 4ULL
+	#define BTRFS_FS_TREE_OBJECTID 5ULL
+	#define BTRFS_ROOT_TREE_DIR_OBJECTID 6ULL
+	#define BTRFS_CSUM_TREE_OBJECTID 7ULL
+	#define BTRFS_QUOTA_TREE_OBJECTID 8ULL
+	#define BTRFS_UUID_TREE_OBJECTID 9ULL
+	#define BTRFS_FREE_SPACE_TREE_OBJECTID 10ULL
+	#define BTRFS_BALANCE_OBJECTID -4ULL
+	#define BTRFS_ORPHAN_OBJECTID -5ULL
+	#define BTRFS_TREE_LOG_OBJECTID -6ULL
+	#define BTRFS_TREE_LOG_FIXUP_OBJECTID -7ULL
+	#define BTRFS_TREE_RELOC_OBJECTID -8ULL
+	#define BTRFS_DATA_RELOC_TREE_OBJECTID -9ULL
+	#define BTRFS_EXTENT_CSUM_OBJECTID -10ULL
+	#define BTRFS_FREE_SPACE_OBJECTID -11ULL
+	#define BTRFS_FREE_INO_OBJECTID -12ULL
+	#define BTRFS_MULTIPLE_OBJECTIDS -255ULL
+	#define BTRFS_FIRST_FREE_OBJECTID 256ULL
+	#define BTRFS_LAST_FREE_OBJECTID -256ULL
+	#define BTRFS_FIRST_CHUNK_TREE_OBJECTID 256ULL
+	#define BTRFS_DEV_ITEMS_OBJECTID 1ULL
+
+	#define BTRFS_INODE_ITEM_KEY            1
+	#define BTRFS_INODE_REF_KEY             12
+	#define BTRFS_INODE_EXTREF_KEY          13
+	#define BTRFS_XATTR_ITEM_KEY            24
+	#define BTRFS_ORPHAN_ITEM_KEY           48
+	#define BTRFS_DIR_LOG_ITEM_KEY  60
+	#define BTRFS_DIR_LOG_INDEX_KEY 72
+	#define BTRFS_DIR_ITEM_KEY      84
+	#define BTRFS_DIR_INDEX_KEY     96
+	#define BTRFS_EXTENT_DATA_KEY   108
+	#define BTRFS_CSUM_ITEM_KEY     120
+	#define BTRFS_EXTENT_CSUM_KEY   128
+	#define BTRFS_ROOT_ITEM_KEY     132
+	#define BTRFS_ROOT_BACKREF_KEY  144
+	#define BTRFS_ROOT_REF_KEY      156
+	#define BTRFS_EXTENT_ITEM_KEY   168
+	#define BTRFS_METADATA_ITEM_KEY 169
+	#define BTRFS_TREE_BLOCK_REF_KEY        176
+	#define BTRFS_EXTENT_DATA_REF_KEY       178
+	#define BTRFS_EXTENT_REF_V0_KEY         180
+	#define BTRFS_SHARED_BLOCK_REF_KEY      182
+	#define BTRFS_SHARED_DATA_REF_KEY       184
+	#define BTRFS_BLOCK_GROUP_ITEM_KEY 192
+	#define BTRFS_FREE_SPACE_INFO_KEY 198
+	#define BTRFS_FREE_SPACE_EXTENT_KEY 199
+	#define BTRFS_FREE_SPACE_BITMAP_KEY 200
+	#define BTRFS_DEV_EXTENT_KEY    204
+	#define BTRFS_DEV_ITEM_KEY      216
+	#define BTRFS_CHUNK_ITEM_KEY    228
+	#define BTRFS_BALANCE_ITEM_KEY  248
+	#define BTRFS_QGROUP_STATUS_KEY         240
+	#define BTRFS_QGROUP_INFO_KEY           242
+	#define BTRFS_QGROUP_LIMIT_KEY          244
+	#define BTRFS_QGROUP_RELATION_KEY       246
+	#define BTRFS_DEV_STATS_KEY     249
+	#define BTRFS_DEV_REPLACE_KEY   250
+	#define BTRFS_UUID_KEY_SUBVOL   251
+	#define BTRFS_UUID_KEY_RECEIVED_SUBVOL  252
+	#define BTRFS_STRING_ITEM_KEY   253
+
+#endif
+
+#ifndef BTRFS_DEFRAG_RANGE_START_IO
+
+	// For some reason uapi has BTRFS_DEFRAG_RANGE_COMPRESS and
+	// BTRFS_DEFRAG_RANGE_START_IO but not btrfs_ioctl_defrag_range_args
+	// Never mind, it's too broken to be useful anyway
+	struct btrfs_ioctl_defrag_range_args {
+		/* start of the defrag operation */
+		__u64 start;
+
+		/* number of bytes to defrag, use (u64)-1 to say all */
+		__u64 len;
+
+		/*
+		 * flags for the operation, which can include turning
+		 * on compression for this one defrag
+		 */
+		__u64 flags;
+
+		/*
+		 * any extent bigger than this will be considered
+		 * already defragged.  Use 0 to take the kernel default
+		 * Use 1 to say every single extent must be rewritten
+		 */
+		__u32 extent_thresh;
+
+		/*
+		 * which compression method to use if turning on compression
+		 * for this defrag operation.  If unspecified, zlib will
+		 * be used
+		 */
+		__u32 compress_type;
+
+		/* spare for later */
+		__u32 unused[4];
+	};
+
+#endif
+	 
+#ifndef BTRFS_IOC_CLONE_RANGE
+
+	struct btrfs_ioctl_clone_range_args {
+	  __s64 src_fd;
+	  __u64 src_offset, src_length;
+	  __u64 dest_offset;
+	};
+
+	// We definitely have this
+	#define BTRFS_IOCTL_MAGIC 0x94
+
+	#define BTRFS_IOC_CLONE        _IOW(BTRFS_IOCTL_MAGIC, 9, int)
+
+	#define BTRFS_IOC_CLONE_RANGE _IOW(BTRFS_IOCTL_MAGIC, 13, \
+					  struct btrfs_ioctl_clone_range_args)
+#endif
+
+#ifndef BTRFS_SAME_DATA_DIFFERS
+
+	#define BTRFS_SAME_DATA_DIFFERS 1
+	/* For extent-same ioctl */
+	struct btrfs_ioctl_same_extent_info {
+		__s64 fd;               /* in - destination file */
+		__u64 logical_offset;   /* in - start of extent in destination */
+		__u64 bytes_deduped;    /* out - total # of bytes we were able
+					 * to dedupe from this file */
+		/* status of this dedupe operation:
+		 * 0 if dedup succeeds
+		 * < 0 for error
+		 * == BTRFS_SAME_DATA_DIFFERS if data differs
+		 */
+		__s32 status;           /* out - see above description */
+		__u32 reserved;
+	};
+
+	struct btrfs_ioctl_same_args {
+		__u64 logical_offset;   /* in - start of extent in source */
+		__u64 length;           /* in - length of extent */
+		__u16 dest_count;       /* in - total elements in info array */
+		__u16 reserved1;
+		__u32 reserved2;
+		struct btrfs_ioctl_same_extent_info info[0];
+	};
+
+	#define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
+						 struct btrfs_ioctl_same_args)
+
+#endif
+
+#ifndef BTRFS_MAX_DEDUPE_LEN
+	#define BTRFS_MAX_DEDUPE_LEN    (16 * 1024 * 1024)
+#endif
+
+#ifndef BTRFS_IOC_TREE_SEARCH_V2
+
+	/*
+	 * Extended version of TREE_SEARCH ioctl that can return more than 4k of bytes.
+	 * The allocated size of the buffer is set in buf_size.
+	 */
+	struct btrfs_ioctl_search_args_v2 {
+		struct btrfs_ioctl_search_key key; /* in/out - search parameters */
+		__u64 buf_size;                    /* in - size of buffer
+						    * out - on EOVERFLOW: needed size
+						    *       to store item */
+		__u64 buf[0];                      /* out - found items */
+	};
+
+	#define BTRFS_IOC_TREE_SEARCH_V2 _IOWR(BTRFS_IOCTL_MAGIC, 17, \
+					   struct btrfs_ioctl_search_args_v2)
+#endif
+
+#endif // CRUCIBLE_BTRFS_H

include/crucible/cache.h

@@ -0,0 +1,221 @@
+#ifndef CRUCIBLE_CACHE_H
+#define CRUCIBLE_CACHE_H
+
+#include "crucible/lockset.h"
+
+#include <algorithm>
+#include <functional>
+#include <map>
+#include <mutex>
+#include <tuple>
+
+namespace crucible {
+	using namespace std;
+
+	template <class Return, class... Arguments>
+	class LRUCache {
+	public:
+		using Key = tuple<Arguments...>;
+		using Func = function<Return(Arguments...)>;
+		using Time = unsigned;
+		using Value = pair<Time, Return>;
+	private:
+		Func		m_fn;
+		Time		m_ctr;
+		map<Key, Value>	m_map;
+		LockSet<Key>	m_lockset;
+		size_t		m_max_size;
+		mutex		m_mutex;
+
+		void check_overflow();
+	public:
+		LRUCache(Func f = Func(), size_t max_size = 100);
+
+		void func(Func f);
+		void max_size(size_t new_max_size);
+
+		Return operator()(Arguments... args);
+		Return refresh(Arguments... args);
+		void expire(Arguments... args);
+		void prune(function<bool(const Return &)> predicate);
+		void insert(const Return &r, Arguments... args);
+		void clear();
+	};
+
+	template <class Return, class... Arguments>
+	LRUCache<Return, Arguments...>::LRUCache(Func f, size_t max_size) :
+		m_fn(f),
+		m_ctr(0),
+		m_max_size(max_size)
+	{
+	}
+
+	template <class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::check_overflow()
+	{
+		if (m_map.size() <= m_max_size) return;
+		vector<pair<Key, Time>> map_contents;
+		map_contents.reserve(m_map.size());
+		for (auto i : m_map) {
+			map_contents.push_back(make_pair(i.first, i.second.first));
+		}
+		sort(map_contents.begin(), map_contents.end(), [](const pair<Key, Time> &a, const pair<Key, Time> &b) {
+			return a.second < b.second;
+		});
+		for (size_t i = 0; i < map_contents.size() / 2; ++i) {
+			m_map.erase(map_contents[i].first);
+		}
+	}
+
+	template <class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::max_size(size_t new_max_size)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		m_max_size = new_max_size;
+		check_overflow();
+	}
+
+	template <class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::func(Func func)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		m_fn = func;
+	}
+
+	template <class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::clear()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		m_map.clear();
+	}
+
+	template <class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::prune(function<bool(const Return &)> pred)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		for (auto it = m_map.begin(); it != m_map.end(); ) {
+			auto next_it = ++it;
+			if (pred(it.second.second)) {
+				m_map.erase(it);
+			}
+			it = next_it;
+		}
+	}
+
+	template<class Return, class... Arguments>
+	Return
+	LRUCache<Return, Arguments...>::operator()(Arguments... args)
+	{
+		Key k(args...);
+		bool inserted = false;
+
+		// Do we have it cached?
+		unique_lock<mutex> lock(m_mutex);
+		auto found = m_map.find(k);
+		if (found == m_map.end()) {
+			// No, release cache lock and acquire key lock
+			lock.unlock();
+			typename LockSet<Key>::Lock key_lock(m_lockset, k);
+
+			// Did item appear in cache while we were waiting for key?
+			lock.lock();
+			found = m_map.find(k);
+			if (found == m_map.end()) {
+
+				// No, we hold key and cache locks, but item not in cache.
+				// Release cache lock and call function
+				auto ctr_copy = m_ctr++;
+				lock.unlock();
+				Value v(ctr_copy, m_fn(args...));
+
+				// Reacquire cache lock and insert return value
+				lock.lock();
+				tie(found, inserted) = m_map.insert(make_pair(k, v));
+
+				// We hold a lock on this key so we are the ones to insert it
+				THROW_CHECK0(runtime_error, inserted);
+
+				// Release key lock and clean out overflow
+				key_lock.unlock();
+				check_overflow();
+			}
+		}
+
+		// Item should be in cache now
+		THROW_CHECK0(runtime_error, found != m_map.end());
+
+		// We are using this object so update the timestamp
+		if (!inserted) {
+			found->second.first = m_ctr++;
+		}
+		return found->second.second;
+	}
+
+	template<class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::expire(Arguments... args)
+	{
+		Key k(args...);
+		unique_lock<mutex> lock(m_mutex);
+		m_map.erase(k);
+	}
+
+	template<class Return, class... Arguments>
+	Return
+	LRUCache<Return, Arguments...>::refresh(Arguments... args)
+	{
+		expire(args...);
+		return operator()(args...);
+	}
+
+	template<class Return, class... Arguments>
+	void
+	LRUCache<Return, Arguments...>::insert(const Return &r, Arguments... args)
+	{
+		Key k(args...);
+		bool inserted = false;
+
+		// Do we have it cached?
+		unique_lock<mutex> lock(m_mutex);
+		auto found = m_map.find(k);
+		if (found == m_map.end()) {
+			// No, release cache lock and acquire key lock
+			lock.unlock();
+			typename LockSet<Key>::Lock key_lock(m_lockset, k);
+
+			// Did item appear in cache while we were waiting for key?
+			lock.lock();
+			found = m_map.find(k);
+			if (found == m_map.end()) {
+
+				// No, we hold key and cache locks, but item not in cache.
+				// Release cache lock and insert the provided return value
+				auto ctr_copy = m_ctr++;
+				Value v(ctr_copy, r);
+				tie(found, inserted) = m_map.insert(make_pair(k, v));
+
+				// We hold a lock on this key so we are the ones to insert it
+				THROW_CHECK0(runtime_error, inserted);
+
+				// Release key lock and clean out overflow
+				key_lock.unlock();
+				check_overflow();
+			}
+		}
+
+		// Item should be in cache now
+		THROW_CHECK0(runtime_error, found != m_map.end());
+
+		// We are using this object so update the timestamp
+		if (!inserted) {
+			found->second.first = m_ctr++;
+		}
+	}
+}
+
+#endif // CRUCIBLE_CACHE_H

include/crucible/chatter.h

@@ -0,0 +1,156 @@
+#ifndef CRUCIBLE_CHATTER_H
+#define CRUCIBLE_CHATTER_H
+
+#include <functional>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <string>
+#include <typeinfo>
+
+  /** \brief Chatter wraps a std::ostream reference with a destructor that
+      writes a newline, and inserts timestamp, pid, and tid prefixes on output.
+
+      Typical usage is expressions like the following:
+
+      int six = 6, nine = 9; \n
+      Chatter() << "What you get when you multiply" << six
+	<< "by" << nine << '?'; \n
+      Chatter() << "forty two!";
+
+      which results in output like the following:
+
+      What you get when you multiply 6 by 9 ?\n
+      forty-two!
+
+      Note that newlines and timestamps are injected automatically in
+      the output by the Chatter destructor.  You can also use std::endl
+      explicitly, although it will not have the effect of flushing the
+      buffer.
+   */
+
+namespace crucible {
+	using namespace std;
+
+	class Chatter {
+		string m_name;
+		ostream &m_os;
+		ostringstream m_oss;
+
+	public:
+		Chatter(string name, ostream &os = cerr);
+		Chatter(Chatter &&c);
+		ostream &get_os() { return m_oss; }
+
+		template <class T> Chatter &operator<<(const T& arg);
+
+		~Chatter();
+	};
+
+	template <class Argument>
+	struct ChatterTraits {
+		Chatter &operator()(Chatter &c, const Argument &arg)
+		{
+			c.get_os() << arg;
+			return c;
+		}
+	};
+
+	template <class T>
+	Chatter &
+	Chatter::operator<<(const T& arg)
+	{
+		return ChatterTraits<T>()(*this, arg);
+	}
+
+	template <class Argument>
+	struct ChatterTraits<const Argument *> {
+		Chatter &operator()(Chatter &c, const Argument *arg)
+		{
+			if (arg) {
+				c.get_os() << "(pointer to " << typeid(*arg).name() << ")(" << reinterpret_cast<const void *>(arg) << ")";
+			} else {
+				c.get_os() << "(NULL pointer to " << typeid(arg).name() << ')';
+			}
+			return c;
+		}
+	};
+
+	template <>
+	struct ChatterTraits<const char *> {
+		Chatter &
+		operator()(Chatter &c, const char *arg)
+		{
+			c.get_os() << arg;
+			return c;
+		}
+	};
+
+	template <>
+	struct ChatterTraits<ostream &> {
+		Chatter &
+		operator()(Chatter &c, ostream & arg)
+		{
+			c.get_os() << arg;
+			return c;
+		}
+	};
+
+	class ChatterBox {
+		string m_file;
+		int m_line;
+		string m_pretty_function;
+		bool m_enabled;
+		ostream& m_os;
+
+		static set<ChatterBox*> s_boxes;
+
+	public:
+		ChatterBox(string file, int line, string pretty_function, ostream &os = cerr);
+		~ChatterBox();
+
+		template <class T> Chatter operator<<(const T &t)
+		{
+			Chatter c(m_pretty_function, m_os);
+			c << t;
+			return c;
+		}
+
+		bool enabled() const { return m_enabled; }
+		void set_enable(bool en);
+
+		static set<ChatterBox*>& all_boxes();
+	};
+
+	class ChatterUnwinder {
+		function<void()> m_func;
+	public:
+		ChatterUnwinder(function<void()> f);
+		~ChatterUnwinder();
+	};
+};
+
+#define CHATTER(x) do { \
+	using namespace crucible; \
+	static ChatterBox crucible_chatterbox_cb(__FILE__, __LINE__, __func__); \
+	if (crucible_chatterbox_cb.enabled()) { \
+		crucible_chatterbox_cb << x; \
+	} \
+} while (0)
+
+#define CHATTER_TRACE(x) do { \
+	using namespace crucible; \
+	static ChatterBox crucible_chatterbox_cb(__FILE__, __LINE__, __func__); \
+	if (crucible_chatterbox_cb.enabled()) { \
+		crucible_chatterbox_cb << __FILE__ << ":" << __LINE__ << ": " << x; \
+	} \
+} while (0)
+
+#define WTF_C(x, y) x##y
+#define SRSLY_WTF_C(x, y) WTF_C(x, y)
+#define CHATTER_UNWIND(x) \
+	crucible::ChatterUnwinder SRSLY_WTF_C(chatterUnwinder_, __LINE__) ([&]() { \
+		CHATTER_TRACE(x); \
+	})
+
+#endif // CRUCIBLE_CHATTER_H

include/crucible/crc64.h

@@ -0,0 +1,16 @@
+#ifndef CRUCIBLE_CRC64_H
+#define CRUCIBLE_CRC64_H
+
+#include <cstdint>
+#include <cstdlib>
+
+namespace crucible {
+	namespace Digest {
+		namespace CRC {
+			uint64_t crc64(const char *s);
+			uint64_t crc64(const void *p, size_t len);
+		};
+	};
+};
+
+#endif

include/crucible/error.h

@@ -0,0 +1,161 @@
+#ifndef CRUCIBLE_ERROR_H
+#define CRUCIBLE_ERROR_H
+
+// Common error-handling idioms for C library calls
+
+#include <cerrno>
+#include <cstring>
+#include <functional>
+#include <sstream>
+#include <stdexcept>
+#include <system_error>
+
+#include <unistd.h>
+
+namespace crucible {
+        using namespace std;
+
+	// Common error-handling idioms for C library calls
+
+        template <class T> T die_if_minus_errno(const char *expr, T rv)
+	{
+		if (rv < 0) {
+			throw system_error(error_code(-rv, system_category()), expr);
+		}
+		return rv;
+	}
+
+        template <class T> T die_if_minus_one(const char *expr, T rv)
+	{
+		if (rv == -1) {
+			throw system_error(error_code(errno, system_category()), expr);
+		}
+		return rv;
+	}
+
+        template <class T> T die_if_zero(const char *expr, T rv)
+	{
+		if (rv == 0) {
+			throw system_error(error_code(errno, system_category()), expr);
+		}
+		return rv;
+	}
+
+        template <class T> T die_if_non_zero(const char *expr, T rv)
+	{
+		if (rv != 0) {
+			throw system_error(error_code(errno, system_category()), expr);
+		}
+		return rv;
+	}
+
+	// Usage:  catch_all([&]() { /* insert body here */ } );
+	// Executes body with exceptions caught and reported to cerr.
+	// Returns:
+	//     0 if f() returns
+	//     non-zero if f() throws an exception
+	//	-1 for unknown exception
+	//	1 for std::exception or class derived thereof
+
+	void set_catch_explainer(function<void(string s)> f);
+	void default_catch_explainer(string s);
+	int catch_all(const function<void()> &f, const function<void(string)> &explainer = default_catch_explainer);
+
+	// catch_and_explain traps the exception, calls the explainer, then rethrows the original exception
+	void catch_and_explain(const function<void()> &f, const function<void(string)> &explainer = default_catch_explainer);
+};
+
+// 0 on success, -errno on error.
+// Covers most pthread functions.
+#define DIE_IF_MINUS_ERRNO(expr) crucible::die_if_minus_errno(#expr, expr)
+
+// -1 on error, all other values mean success.
+#define DIE_IF_MINUS_ONE(expr)   crucible::die_if_minus_one(#expr, expr)
+
+// 0 (or NULL) on error, all other values mean success.
+#define DIE_IF_ZERO(expr)        crucible::die_if_zero(#expr, expr)
+
+// 0 (or NULL) on success, all other values mean error.
+#define DIE_IF_NON_ZERO(expr)    crucible::die_if_non_zero(#expr, expr)
+
+// macro for throwing an error
+#define THROW_ERROR(type, expr) do { \
+	std::ostringstream _te_oss; \
+	_te_oss << expr; \
+	throw type(_te_oss.str()); \
+} while (0)
+
+// macro for throwing a system_error with errno
+#define THROW_ERRNO(expr) do { \
+	std::ostringstream _te_oss; \
+	_te_oss << expr; \
+	throw std::system_error(std::error_code(errno, std::system_category()), _te_oss.str()); \
+} while (0)
+
+// macro for throwing a system_error with some other variable
+#define THROW_ERRNO_VALUE(value, expr) do { \
+	std::ostringstream _te_oss; \
+	_te_oss << expr; \
+	throw std::system_error(std::error_code((value), std::system_category()), _te_oss.str()); \
+} while (0)
+
+// macros for checking a constraint
+#define CHECK_CONSTRAINT(value, expr) do { \
+	if (!(expr)) { \
+		THROW_ERROR(out_of_range, #value << " = " << value << " failed constraint check (" << #expr << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK0(type, expr) do { \
+	if (!(expr)) { \
+		THROW_ERROR(type, "failed constraint check (" << #expr << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK1(type, value, expr) do { \
+	if (!(expr)) { \
+		THROW_ERROR(type, #value << " = " << (value) << " failed constraint check (" << #expr << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK2(type, value1, value2, expr) do { \
+	if (!(expr)) { \
+		THROW_ERROR(type, #value1 << " = " << (value1) << ", " #value2 << " = " << (value2) \
+			<< " failed constraint check (" << #expr << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK3(type, value1, value2, value3, expr) do { \
+	if (!(expr)) { \
+		THROW_ERROR(type, #value1 << " = " << (value1) << ", " #value2 << " = " << (value2) << ", " #value3 << " = " << (value3) \
+			<< " failed constraint check (" << #expr << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK_BIN_OP(type, value1, op, value2) do { \
+	if (!((value1) op (value2))) { \
+		THROW_ERROR(type, "failed constraint check " << #value1 << " (" << (value1) << ") " << #op << " " << #value2 << " (" << (value2) << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK_PREFIX_OP(type, op, value1) do { \
+	if (!(op (value1))) { \
+		THROW_ERROR(type, "failed constraint check " << #op << " " << #value1 << " (" << (value1) << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK_RANGE(type, value_min, value_test, value_max) do { \
+	if ((value_test) < (value_min) || (value_max) < (value_test)) { \
+		THROW_ERROR(type, "failed constraint check " << #value_min << " (" << (value_min) << ") <= " #value_test << " (" << (value_test) \
+			<< ") <= " << #value_max << " (" << (value_max) << ")"); \
+	} \
+} while(0)
+
+#define THROW_CHECK_ARRAY_RANGE(type, value_min, value_test, value_max) do { \
+	if ((value_test) < (value_min) || !((value_test) < (value_max))) { \
+		THROW_ERROR(type, "failed constraint check " << #value_min << " (" << (value_min) << ") <= " #value_test << " (" << (value_test) \
+			<< ") < " << #value_max << " (" << (value_max) << ")"); \
+	} \
+} while(0)
+
+#endif // CRUCIBLE_ERROR_H

include/crucible/execpipe.h

@@ -0,0 +1,28 @@
+#ifndef CRUCIBLE_EXECPIPE_H
+#define CRUCIBLE_EXECPIPE_H
+
+#include "crucible/fd.h"
+
+#include <functional>
+#include <limits>
+#include <string>
+
+namespace crucible {
+	using namespace std;
+
+	void redirect_stdin(const Fd &child_fd);
+	void redirect_stdin_stdout(const Fd &child_fd);
+	void redirect_stdin_stdout_stderr(const Fd &child_fd);
+	void redirect_stdout(const Fd &child_fd);
+	void redirect_stdout_stderr(const Fd &child_fd);
+
+	// Open a pipe (actually socketpair) to child process, then execute code in that process.
+	// e.g. popen([] () { system("echo Hello, World!"); });
+	// Forked process will exit when function returns.
+	Fd popen(function<int()> f, function<void(const Fd &child_fd)> import_fd_fn = redirect_stdin_stdout);
+
+	// Read all the data from fd into a string
+        string read_all(Fd fd, size_t max_bytes = numeric_limits<size_t>::max(), size_t chunk_bytes = 4096);
+};
+
+#endif // CRUCIBLE_EXECPIPE_H

include/crucible/extentwalker.h

@@ -0,0 +1,101 @@
+#ifndef CRUCIBLE_EXTENTWALKER_H
+#define CRUCIBLE_EXTENTWALKER_H
+
+#include "crucible/fd.h"
+
+namespace crucible {
+	using namespace std;
+
+	// FIXME:  ExtentCursor is probably a better name
+	struct Extent {
+		off_t		m_begin;
+		off_t		m_end;
+		uint64_t	m_physical;
+		uint64_t	m_flags;
+
+		// Btrfs extent reference details
+		off_t		m_physical_len;
+		off_t		m_logical_len;
+		off_t		m_offset;
+
+		// fiemap flags are uint32_t, so bits 32..63 are OK for us
+
+		// no extent here
+		static const uint64_t HOLE      = (1ULL << 32);
+
+		// extent is physical space full of zeros
+		static const uint64_t PREALLOC  = (1ULL << 33);
+
+		// extent's physical (RAM) size does not match logical (can we know this?)
+		static const uint64_t OBSCURED  = (1ULL << 34);
+
+		operator bool() const;
+		off_t size() const;
+		off_t begin() const { return m_begin; }
+		off_t end() const { return m_end; }
+		uint64_t flags() const { return m_flags; }
+		uint64_t physical() const { return m_physical; }
+		off_t physical_len() const { return m_physical_len; }
+		off_t logical_len() const { return m_logical_len; }
+		off_t offset() const { return m_offset; }
+		bool operator==(const Extent &that) const;
+		bool operator!=(const Extent &that) const { return !(*this == that); }
+
+		Extent();
+		Extent(const Extent &e) = default;
+	};
+
+	class ExtentWalker {
+	public:
+		using Vec = vector<Extent>;
+		using Itr = Vec::iterator;
+
+	protected:
+		Fd	m_fd;
+		Stat	m_stat;
+
+		virtual Vec get_extent_map(off_t pos);
+
+		static const unsigned sc_extent_fetch_max = 64;
+		static const unsigned sc_extent_fetch_min = 4;
+		static const off_t sc_step_size = 0x1000 * (sc_extent_fetch_max / 2);
+
+	private:
+		Vec	m_extents;
+		Itr	m_current;
+
+		Itr find_in_cache(off_t pos);
+		void run_fiemap(off_t pos);
+
+	public:
+		ExtentWalker(Fd fd = Fd());
+		ExtentWalker(Fd fd, off_t initial_pos);
+		virtual ~ExtentWalker();
+
+		void reset();
+		Extent current();
+		bool next();
+		bool prev();
+		void seek(off_t new_pos);
+
+	friend ostream & operator<<(ostream &os, const ExtentWalker &ew);
+	};
+
+	class BtrfsExtentWalker : public ExtentWalker {
+		uint64_t m_tree_id;
+		Fd m_root_fd;
+
+	protected:
+		Vec get_extent_map(off_t pos) override;
+
+	public:
+		BtrfsExtentWalker(Fd fd);
+		BtrfsExtentWalker(Fd fd, off_t initial_pos);
+		BtrfsExtentWalker(Fd fd, off_t initial_pos, Fd root_fd);
+		void set_root_fd(Fd fd);
+	};
+
+	ostream &operator<<(ostream &os, const Extent &e);
+};
+
+#endif // CRUCIBLE_EXTENTWALKER_H

include/crucible/fd.h

@@ -0,0 +1,178 @@
+#ifndef CRUCIBLE_FD_H
+#define CRUCIBLE_FD_H
+
+#include "crucible/resource.h"
+
+#include <cstring>
+
+#include <string>
+#include <vector>
+
+// open
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+// socket
+#include <sys/socket.h>
+
+// pread/pwrite
+#include <unistd.h>
+
+namespace crucible {
+	using namespace std;
+
+	// IOHandle is a file descriptor owner object.  It closes them when destroyed.
+	// Most of the functions here don't use it because these functions don't own FDs.
+	// All good names for such objects are taken.
+	class IOHandle {
+		IOHandle(const IOHandle &) = delete;
+		IOHandle(IOHandle &&) = delete;
+		IOHandle& operator=(IOHandle &&) = delete;
+		IOHandle& operator=(const IOHandle &) = delete;
+	protected:
+		int	m_fd;
+		IOHandle& operator=(int that) { m_fd = that; return *this; }
+	public:
+		virtual ~IOHandle();
+		IOHandle(int fd);
+		IOHandle();
+
+		void close();
+		int get_fd() const { return m_fd; }
+		int release_fd();
+	};
+
+        template <>
+        struct ResourceTraits<int, IOHandle> {
+                int get_key(const IOHandle &res) const { return res.get_fd(); }
+                shared_ptr<IOHandle> make_resource(int fd) const { return make_shared<IOHandle>(fd); }
+                bool is_null_key(const int &key) const { return key < 0; }
+                int get_null_key() const { return -1; }
+        };
+
+        typedef ResourceHandle<int, IOHandle> Fd;
+
+	// Functions named "foo_or_die" throw exceptions on failure.
+
+	// Attempt to open the file with the given mode
+	int open_or_die(const string &file, int flags = O_RDONLY, mode_t mode = 0777);
+	int openat_or_die(int dir_fd, const string &file, int flags = O_RDONLY, mode_t mode = 0777);
+
+	// Decode open parameters
+	string o_flags_ntoa(int flags);
+	string o_mode_ntoa(mode_t mode);
+
+	// mmap with its one weird error case
+	void *mmap_or_die(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
+	// Decode mmap parameters
+	string mmap_prot_ntoa(int prot);
+	string mmap_flags_ntoa(int flags);
+
+	// Unlink, rename
+	void unlink_or_die(const string &file);
+	void rename_or_die(const string &from, const string &to);
+	void renameat_or_die(int fromfd, const string &frompath, int tofd, const string &topath);
+
+	// Read or write structs:
+	// There is a template specialization to read or write strings
+	// Three-arg version of read_or_die/write_or_die throws an error on incomplete read/writes
+	// Four-arg version returns number of bytes read/written through reference arg
+
+	void read_or_die(int fd, void *buf, size_t size);
+	template <class T> void read_or_die(int fd, T& buf)
+	{
+		return read_or_die(fd, static_cast<void *>(&buf), sizeof(buf));
+	}
+
+	void read_partial_or_die(int fd, void *buf, size_t size_wanted, size_t &size_read);
+	template <class T> void read_partial_or_die(int fd, T& buf, size_t &size_read)
+	{
+		return read_partial_or_die(fd, static_cast<void *>(&buf), sizeof(buf), size_read);
+	}
+
+	void pread_or_die(int fd, void *buf, size_t size, off_t offset);
+	template <class T> void pread_or_die(int fd, T& buf, off_t offset)
+	{
+		return pread_or_die(fd, static_cast<void *>(&buf), sizeof(buf), offset);
+	}
+
+	void write_or_die(int fd, const void *buf, size_t size);
+	template <class T> void write_or_die(int fd, const T& buf)
+	{
+		return write_or_die(fd, static_cast<const void *>(&buf), sizeof(buf));
+	}
+
+	void write_partial_or_die(int fd, const void *buf, size_t size_wanted, size_t &size_written);
+	template <class T> void write_partial_or_die(int fd, const T& buf, size_t &size_written)
+	{
+		return write_partial_or_die(fd, static_cast<const void *>(&buf), sizeof(buf), size_written);
+	}
+
+	void pwrite_or_die(int fd, const void *buf, size_t size, off_t offset);
+	template <class T> void pwrite_or_die(int fd, const T& buf, off_t offset)
+	{
+		return pwrite_or_die(fd, static_cast<const void *>(&buf), sizeof(buf), offset);
+	}
+
+	// Specialization for strings which reads/writes the string content, not the struct string
+	template<> void write_or_die<string>(int fd, const string& str);
+	template<> void pread_or_die<string>(int fd, string& str, off_t offset);
+	template<> void pread_or_die<vector<char>>(int fd, vector<char>& str, off_t offset);
+	template<> void pread_or_die<vector<uint8_t>>(int fd, vector<uint8_t>& str, off_t offset);
+
+	// A different approach to reading a simple string
+	string read_string(int fd, size_t size);
+
+	// A lot of Unix API wants you to initialize a struct and call
+	// one function to fill it, another function to throw it away,
+	// and has some unknown third thing you have to do when there's
+	// an error.  That's also a C++ object with an exception-throwing
+	// constructor.
+	struct Stat : public stat {
+		Stat();
+		Stat(int f);
+		Stat(const string &filename);
+		Stat &fstat(int fd);
+		Stat &lstat(const string &filename);
+	};
+
+	string st_mode_ntoa(mode_t mode);
+
+	// Because it's not trivial to do correctly
+	string readlink_or_die(const string &path);
+
+	// Determine the name of a FD by readlink through /proc/self/fd/
+	string name_fd(int fd);
+
+	// Returns Fd objects because it does own them.
+	pair<Fd, Fd> socketpair_or_die(int domain = AF_UNIX, int type = SOCK_STREAM, int protocol = 0);
+
+	// like unique_lock but for flock instead of mutexes...and not trying
+	// to hide the many and subtle differences between those two things *at all*.
+	class Flock {
+		int	m_fd;
+		bool	m_locked;
+		Flock(const Flock &) = delete;
+		Flock(Flock &&) = delete;
+		Flock &operator=(const Flock &) = delete;
+		Flock &operator=(Flock &&) = delete;
+	public:
+		Flock();
+		Flock(int fd);
+		Flock(int fd, bool init_locked_state);
+		~Flock();
+		void lock();
+		void try_lock();
+		void unlock();
+		bool owns_lock();
+		operator bool();
+		int fd();
+	};
+
+	// Doesn't use Fd objects because it's usually just used to replace stdin/stdout/stderr.
+	void dup2_or_die(int fd_in, int fd_out);
+
+}
+
+#endif // CRUCIBLE_FD_H

include/crucible/fs.h

@@ -0,0 +1,246 @@
+#ifndef CRUCIBLE_FS_H
+#define CRUCIBLE_FS_H
+
+#include "crucible/error.h"
+
+// Terribly Linux-specific FS-wrangling functions
+
+// BTRFS
+#include "crucible/btrfs.h"
+
+// FIEMAP_* structs and flags
+#include <linux/fiemap.h>
+
+#include <cstdint>
+#include <iosfwd>
+#include <vector>
+
+#include <fcntl.h>
+#include <sys/statvfs.h>
+
+namespace crucible {
+	using namespace std;
+
+	// wrapper around fallocate(...FALLOC_FL_PUNCH_HOLE...)
+	void punch_hole(int fd, off_t offset, off_t len);
+
+	struct BtrfsExtentInfo : public btrfs_ioctl_same_extent_info {
+		BtrfsExtentInfo(int dst_fd, off_t dst_offset);
+	};
+
+	struct BtrfsExtentSame : public btrfs_ioctl_same_args {
+		virtual ~BtrfsExtentSame();
+		BtrfsExtentSame(int src_fd, off_t src_offset, off_t src_length);
+		void add(int fd, off_t offset);
+		virtual void do_ioctl();
+
+		int m_fd;
+		vector<BtrfsExtentInfo> m_info;
+	};
+
+	struct BtrfsExtentSameByClone : public BtrfsExtentSame {
+		using BtrfsExtentSame::BtrfsExtentSame;
+		void do_ioctl() override;
+	};
+
+	ostream & operator<<(ostream &os, const btrfs_ioctl_same_extent_info *info);
+	ostream & operator<<(ostream &os, const btrfs_ioctl_same_args *info);
+	ostream & operator<<(ostream &os, const BtrfsExtentSame &bes);
+
+	struct BtrfsInodeOffsetRoot {
+		uint64_t m_inum;
+		uint64_t m_offset;
+		uint64_t m_root;
+	};
+
+	ostream & operator<<(ostream &os, const BtrfsInodeOffsetRoot &p);
+
+	struct BtrfsDataContainer : public btrfs_data_container {
+		BtrfsDataContainer(size_t size = 64 * 1024);
+		void *prepare();
+
+		size_t get_size() const;
+		decltype(bytes_left) get_bytes_left() const;
+		decltype(bytes_missing) get_bytes_missing() const;
+		decltype(elem_cnt) get_elem_cnt() const;
+		decltype(elem_missed) get_elem_missed() const;
+
+		vector<char> m_data;
+	};
+
+	struct BtrfsIoctlLogicalInoArgs : public btrfs_ioctl_logical_ino_args {
+		BtrfsIoctlLogicalInoArgs(uint64_t logical, size_t buf_size = 64 * 1024);
+		virtual void do_ioctl(int fd);
+		virtual bool do_ioctl_nothrow(int fd);
+
+		BtrfsDataContainer m_container;
+		vector<BtrfsInodeOffsetRoot> m_iors;
+	};
+
+	ostream & operator<<(ostream &os, const BtrfsIoctlLogicalInoArgs &p);
+
+	struct BtrfsIoctlInoPathArgs : public btrfs_ioctl_ino_path_args {
+		BtrfsIoctlInoPathArgs(uint64_t inode, size_t buf_size = 64 * 1024);
+		virtual void do_ioctl(int fd);
+		virtual bool do_ioctl_nothrow(int fd);
+
+		BtrfsDataContainer m_container;
+		vector<string> m_paths;
+	};
+
+	ostream & operator<<(ostream &os, const BtrfsIoctlInoPathArgs &p);
+
+	struct BtrfsIoctlInoLookupArgs : public btrfs_ioctl_ino_lookup_args {
+		BtrfsIoctlInoLookupArgs(uint64_t objectid);
+		virtual void do_ioctl(int fd);
+		virtual bool do_ioctl_nothrow(int fd);
+		// use objectid = BTRFS_FIRST_FREE_OBJECTID
+		// this->treeid is the rootid for the path (we get the path too)
+	};
+
+	struct BtrfsIoctlDefragRangeArgs : public btrfs_ioctl_defrag_range_args {
+		BtrfsIoctlDefragRangeArgs();
+		virtual void do_ioctl(int fd);
+		virtual bool do_ioctl_nothrow(int fd);
+	};
+
+	ostream & operator<<(ostream &os, const BtrfsIoctlDefragRangeArgs *p);
+
+	// in btrfs/ctree.h, but that's a nightmare to #include here
+	typedef enum {
+		BTRFS_COMPRESS_NONE  = 0,
+		BTRFS_COMPRESS_ZLIB  = 1,
+		BTRFS_COMPRESS_LZO   = 2,
+		BTRFS_COMPRESS_TYPES = 2,
+		BTRFS_COMPRESS_LAST  = 3,
+	} btrfs_compression_type;
+
+	struct FiemapExtent : public fiemap_extent {
+		FiemapExtent();
+		FiemapExtent(const fiemap_extent &that);
+		operator bool() const;
+		off_t begin() const;
+		off_t end() const;
+	};
+
+	struct Fiemap : public fiemap {
+
+		// Get entire file
+		Fiemap(uint64_t start = 0, uint64_t length = FIEMAP_MAX_OFFSET);
+
+		void do_ioctl(int fd);
+
+		vector<FiemapExtent> m_extents;
+		uint64_t m_min_count = (4096 - sizeof(fiemap)) / sizeof(fiemap_extent);
+		uint64_t m_max_count = 16 * 1024 * 1024 / sizeof(fiemap_extent);
+	};
+
+	ostream & operator<<(ostream &os, const fiemap_extent *info);
+	ostream & operator<<(ostream &os, const FiemapExtent &info);
+	ostream & operator<<(ostream &os, const fiemap *info);
+	ostream & operator<<(ostream &os, const Fiemap &info);
+
+	string fiemap_extent_flags_ntoa(unsigned long flags);
+
+	// Helper functions
+	void btrfs_clone_range(int src_fd, off_t src_offset, off_t src_length, int dst_fd, off_t dst_offset);
+	bool btrfs_extent_same(int src_fd, off_t src_offset, off_t src_length, int dst_fd, off_t dst_offset);
+
+	struct BtrfsIoctlSearchHeader : public btrfs_ioctl_search_header {
+		BtrfsIoctlSearchHeader();
+		vector<char> m_data;
+		size_t set_data(const vector<char> &v, size_t offset);
+	};
+
+	ostream & operator<<(ostream &os, const btrfs_ioctl_search_header &hdr);
+	ostream & operator<<(ostream &os, const BtrfsIoctlSearchHeader &hdr);
+
+	struct BtrfsIoctlSearchKey : public btrfs_ioctl_search_key {
+		BtrfsIoctlSearchKey(size_t buf_size = 1024 * 1024);
+		virtual bool do_ioctl_nothrow(int fd);
+		virtual void do_ioctl(int fd);
+
+		// Copy objectid/type/offset so we move forward
+		void next_min(const BtrfsIoctlSearchHeader& ref);
+
+		size_t m_buf_size;
+		vector<BtrfsIoctlSearchHeader> m_result;
+	};
+
+	ostream & operator<<(ostream &os, const btrfs_ioctl_search_key &key);
+	ostream & operator<<(ostream &os, const BtrfsIoctlSearchKey &key);
+
+	string btrfs_search_type_ntoa(unsigned type);
+	string btrfs_search_objectid_ntoa(unsigned objectid);
+
+	uint64_t btrfs_get_root_id(int fd);
+	uint64_t btrfs_get_root_transid(int fd);
+
+	template<class T>
+	const T*
+	get_struct_ptr(vector<char> &v, size_t offset = 0)
+	{
+		// OK so sometimes btrfs overshoots a little
+		if (offset + sizeof(T) > v.size()) {
+			v.resize(offset + sizeof(T), 0);
+		}
+		THROW_CHECK2(invalid_argument, v.size(), offset + sizeof(T), offset + sizeof(T) <= v.size());
+		return reinterpret_cast<const T*>(v.data() + offset);
+	}
+
+	template<class A, class R>
+	R
+	call_btrfs_get(R (*func)(const A*), vector<char> &v, size_t offset = 0)
+	{
+		return func(get_struct_ptr<A>(v, offset));
+	}
+
+	template <class T> struct btrfs_get_le;
+
+	template<> struct btrfs_get_le<__le64> {
+		uint64_t operator()(const void *p) { return get_unaligned_le64(p); }
+	};
+
+	template<> struct btrfs_get_le<__le32> {
+		uint32_t operator()(const void *p) { return get_unaligned_le32(p); }
+	};
+
+	template<> struct btrfs_get_le<__le16> {
+		uint16_t operator()(const void *p) { return get_unaligned_le16(p); }
+	};
+
+	template<> struct btrfs_get_le<__le8> {
+		uint8_t operator()(const void *p) { return get_unaligned_le8(p); }
+	};
+
+	template<class S, class T>
+	T
+	btrfs_get_member(T S::* member, vector<char> &v, size_t offset = 0)
+	{
+		const S *sp = reinterpret_cast<const S*>(NULL);
+		const T *spm = &(sp->*member);
+		auto member_offset = reinterpret_cast<const char *>(spm) - reinterpret_cast<const char *>(sp);
+		return btrfs_get_le<T>()(get_struct_ptr<S>(v, offset + member_offset));
+	}
+
+	struct Statvfs : public statvfs {
+		Statvfs();
+		Statvfs(string path);
+		Statvfs(int fd);
+		unsigned long size() const;
+		unsigned long free() const;
+		unsigned long available() const;
+	};
+
+	ostream &hexdump(ostream &os, const vector<char> &v);
+
+	struct BtrfsIoctlFsInfoArgs : public btrfs_ioctl_fs_info_args {
+		BtrfsIoctlFsInfoArgs();
+		void do_ioctl(int fd);
+		string uuid() const;
+	};
+
+	ostream & operator<<(ostream &os, const BtrfsIoctlFsInfoArgs &a);
+};
+
+#endif // CRUCIBLE_FS_H

include/crucible/interp.h

@@ -0,0 +1,106 @@
+#ifndef CRUCIBLE_INTERP_H
+#define CRUCIBLE_INTERP_H
+
+#include "crucible/error.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace crucible {
+	using namespace std;
+
+	struct ArgList : public vector<string> {
+		ArgList(const char **argv);
+		// using vector<string>::vector ... doesn't work:
+		// error: ‘std::vector<std::basic_string<char> >::vector’ names constructor
+		// Still doesn't work in 4.9 because it can't manage a conversion
+		ArgList(const vector<string> &&that);
+	};
+
+	struct ArgActor {
+		struct ArgActorBase {
+			virtual void predicate(void *obj, string arg);
+		};
+
+		template <class T>
+		struct ArgActorDerived {
+			function<void(T, string)> m_func;
+
+			ArgActorDerived(decltype(m_func) func) :
+				m_func(func)
+			{
+			}
+
+			void predicate(void *obj, string arg) override
+			{
+				T &op = *(reinterpret_cast<T*>(obj));
+				m_func(op, obj);
+			}
+		};
+
+		template <class T>
+		ArgActor(T, function<void(T, string)> func) :
+			m_actor(make_shared(ArgActorDerived<T>(func)))
+		{
+		}
+
+		ArgActor() = default;
+
+		void predicate(void *t, string arg)
+		{
+			if (m_actor) {
+				m_actor->predicate(t, arg);
+			} else {
+				THROW_ERROR(invalid_argument, "null m_actor for predicate arg '" << arg << "'");
+			}
+		}
+
+	private:
+		shared_ptr<ArgActorBase> m_actor;
+	};
+
+	struct ArgParser {
+		~ArgParser();
+		ArgParser();
+
+		void add_opt(string opt, ArgActor actor);
+
+		template <class T>
+		void
+		parse(T t, const ArgList &args)
+		{
+			void *vt = &t;
+			parse_backend(vt, args);
+		}
+		
+	private:
+		void parse_backend(void *t, const ArgList &args);
+		map<string, ArgActor>	m_string_opts;
+	};
+
+	struct Command {
+		virtual ~Command();
+		virtual int exec(const ArgList &args) = 0;
+	};
+
+	struct Proc : public Command {
+		int exec(const ArgList &args) override;
+		Proc(const function<int(const ArgList &)> &f);
+	private:
+		function<int(const ArgList &)> m_cmd;
+	};
+
+	struct Interp {
+		virtual ~Interp();
+		Interp(const map<string, shared_ptr<Command> > &cmdlist);
+		void add_command(const string &name, const shared_ptr<Command> &command);
+		int exec(const ArgList &args);
+	private:
+		Interp(const Interp &) = delete;
+		map<string, shared_ptr<Command> > m_commands;
+	};
+
+};
+#endif // CRUCIBLE_INTERP_H

include/crucible/limits.h

@@ -0,0 +1,51 @@
+#ifndef CRUCIBLE_LIMITS_H
+#define CRUCIBLE_LIMITS_H
+
+#include "crucible/error.h"
+
+#include <limits>
+#include <typeinfo>
+
+namespace crucible {
+	using namespace std;
+
+	template <class To, class From>
+	To
+	ranged_cast(From f)
+	{
+		if (typeid(From) == typeid(To)) {
+			return f;
+		}
+
+		To t;
+		static string f_info = typeid(f).name();
+		static string t_info = typeid(t).name();
+
+		if (numeric_limits<From>::max() > numeric_limits<To>::max() && numeric_limits<From>::max() < numeric_limits<To>::max()) {
+			THROW_ERROR(out_of_range,
+				"ranged_cast: can't compare limits of types " << f_info << " and " << t_info << ", template specialization required");
+		}
+
+		if (numeric_limits<From>::max() > numeric_limits<To>::max() && f > static_cast<From>(numeric_limits<To>::max())) {
+			THROW_ERROR(out_of_range,
+				"ranged_cast: " << f_info << "(" << f << ") out of range of target type " << t_info);
+		}
+
+		if (!numeric_limits<To>::is_signed && numeric_limits<From>::is_signed && f < 0) {
+			THROW_ERROR(out_of_range,
+				"ranged_cast: " << f_info << "(" << f << ") out of range of unsigned target type " << t_info);
+		}
+
+		t = static_cast<To>(f);
+
+		From f2 = static_cast<From>(t);
+		if (f2 != f) {
+			THROW_ERROR(out_of_range,
+				"ranged_cast: " << f_info << "(" << f << ") -> " << t_info << " failed: result value " << f2);
+		}
+
+		return t;
+	}
+};
+
+#endif // CRUCIBLE_LIMITS_H

include/crucible/lockset.h

@@ -0,0 +1,210 @@
+#ifndef CRUCIBLE_LOCKSET_H
+#define CRUCIBLE_LOCKSET_H
+
+#include <crucible/error.h>
+
+#include <cassert>
+
+#include <condition_variable>
+#include <iostream>
+#include <mutex>
+#include <set>
+
+namespace crucible {
+	using namespace std;
+
+	template <class T>
+	class LockSet {
+
+	public:
+		using key_type = T;
+		using set_type = set<T>;
+
+	private:
+
+		set_type			m_set;
+		mutex				m_mutex;
+		condition_variable		m_condvar;
+
+	public:
+		~LockSet();
+		LockSet() = default;
+
+		void lock(const key_type &name);
+		void unlock(const key_type &name);
+		bool try_lock(const key_type &name);
+		size_t size();
+		bool empty();
+		set_type copy();
+		void wait_unlock(double interval);
+
+		class Lock {
+			LockSet		&m_lockset;
+			key_type	m_name;
+			bool		m_locked;
+
+			Lock() = delete;
+			Lock(const Lock &) = delete;
+			Lock& operator=(const Lock &) = delete;
+		public:
+			~Lock();
+			Lock(LockSet &lockset, const key_type &m_name, bool start_locked = true);
+			Lock(Lock &&that);
+			Lock& operator=(Lock &&that);
+			void lock();
+			void unlock();
+			bool try_lock();
+		};
+
+	};
+
+	template <class T>
+	LockSet<T>::~LockSet()
+	{
+		if (!m_set.empty()) {
+			cerr << "ERROR: " << m_set.size() << " locked items still in set at destruction" << endl;
+		}
+		// We will crash later.  Might as well crash now.
+		assert(m_set.empty());
+	}
+
+	template <class T>
+	void
+	LockSet<T>::lock(const key_type &name)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		while (m_set.count(name)) {
+			m_condvar.wait(lock);
+		}
+		auto rv = m_set.insert(name);
+		THROW_CHECK0(runtime_error, rv.second);
+	}
+
+	template <class T>
+	bool
+	LockSet<T>::try_lock(const key_type &name)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		if (m_set.count(name)) {
+			return false;
+		}
+		auto rv = m_set.insert(name);
+		THROW_CHECK1(runtime_error, name, rv.second);
+		return true;
+	}
+
+	template <class T>
+	void
+	LockSet<T>::unlock(const key_type &name)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		m_condvar.notify_all();
+		auto erase_count = m_set.erase(name);
+		THROW_CHECK1(invalid_argument, erase_count, erase_count == 1);
+	}
+
+	template <class T>
+	void
+	LockSet<T>::wait_unlock(double interval)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		if (m_set.empty()) return;
+		m_condvar.wait_for(lock, chrono::duration<double>(interval));
+	}
+
+	template <class T>
+	size_t
+	LockSet<T>::size()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set.size();
+	}
+
+	template <class T>
+	bool
+	LockSet<T>::empty()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set.empty();
+	}
+
+	template <class T>
+	typename LockSet<T>::set_type
+	LockSet<T>::copy()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set;
+	}
+
+	template <class T>
+	void
+	LockSet<T>::Lock::lock()
+	{
+		if (m_locked) return;
+		m_lockset.lock(m_name);
+		m_locked = true;
+	}
+
+	template <class T>
+	bool
+	LockSet<T>::Lock::try_lock()
+	{
+		if (m_locked) return true;
+		m_locked = m_lockset.try_lock(m_name);
+		return m_locked;
+	}
+
+	template <class T>
+	void
+	LockSet<T>::Lock::unlock()
+	{
+		if (!m_locked) return;
+		m_lockset.unlock(m_name);
+		m_locked = false;
+	}
+
+	template <class T>
+	LockSet<T>::Lock::~Lock()
+	{
+		if (m_locked) {
+			unlock();
+		}
+	}
+
+	template <class T>
+	LockSet<T>::Lock::Lock(LockSet &lockset, const key_type &name, bool start_locked) :
+		m_lockset(lockset),
+		m_name(name),
+		m_locked(false)
+	{
+		if (start_locked) {
+			lock();
+		}
+	}
+
+	template <class T>
+	LockSet<T>::Lock::Lock(Lock &&that) :
+		m_lockset(that.lockset),
+		m_name(that.m_name),
+		m_locked(that.m_locked)
+	{
+		that.m_locked = false;
+	}
+
+	template <class T>
+	typename LockSet<T>::Lock &
+	LockSet<T>::Lock::operator=(Lock &&that)
+	{
+		THROW_CHECK2(invalid_argument, &m_lockset, &that.m_lockset, &m_lockset == &that.m_lockset);
+		if (m_locked && that.m_name != m_name) {
+			unlock();
+		}
+		m_name = that.m_name;
+		m_locked = that.m_locked;
+		that.m_locked = false;
+		return *this;
+	}
+
+}
+
+#endif // CRUCIBLE_LOCKSET_H

include/crucible/ntoa.h

@@ -0,0 +1,28 @@
+#ifndef CRUCIBLE_NTOA_H
+#define CRUCIBLE_NTOA_H
+
+#include <string>
+
+namespace crucible {
+	using namespace std;
+
+	struct bits_ntoa_table {
+		unsigned long n;
+		unsigned long mask;
+		const char *a;
+	};
+
+	string bits_ntoa(unsigned long n, const bits_ntoa_table *a);
+
+};
+
+// Combinations of bits (list multiple-bit entries first)
+#define NTOA_TABLE_ENTRY_BITS(x) { .n = (x), .mask = (x), .a = (#x) }
+
+// Enumerations (entire value matches all bits)
+#define NTOA_TABLE_ENTRY_ENUM(x) { .n = (x), .mask = ~0UL,  .a = (#x) }
+
+// End of table (sorry, gcc doesn't implement this)
+#define NTOA_TABLE_ENTRY_END() { .n = 0, .mask = 0, .a = nullptr }
+
+#endif // CRUCIBLE_NTOA_H

include/crucible/path.h

@@ -0,0 +1,13 @@
+#ifndef CRUCIBLE_PATH_H
+#define CRUCIBLE_PATH_H
+
+#include <string>
+
+namespace crucible {
+	using namespace std;
+
+	string basename(string s);
+	string join(string dir, string base);
+};
+
+#endif // CRUCIBLE_PATH_H

include/crucible/process.h

@@ -0,0 +1,78 @@
+#ifndef CRUCIBLE_PROCESS_H
+#define CRUCIBLE_PROCESS_H
+
+#include "crucible/resource.h"
+
+#include <functional>
+#include <memory>
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+namespace crucible {
+	using namespace std;
+
+	// Like thread, but for processes.
+	// TODO:  thread has a few warts for this usage:
+	// - can't create one from its native_handle,
+	// - can't destroy one without joining/detaching it first
+	// - can't implement detach correctly without crossing threshold of insanity
+	// - WTF is native_handle() not const?
+	struct Process {
+		// These parts are for compatibility with std::thread
+
+		using id = ::pid_t;
+		using native_handle_type = ::pid_t;
+
+		~Process();
+		Process();
+
+		template <class Fn, class... Args>
+		Process(Fn fn, Args... args) :
+			Process()
+		{
+			do_fork(function<int()>([&]() { return fn(args...); }));
+		}
+
+		Process(const Process &) = delete;
+		Process(Process &&move_from);
+
+		bool joinable();
+		void detach();
+		native_handle_type native_handle();
+		id get_id();
+
+		// Modified thread members for Process
+
+		// join() calls waitpid(), returns status or exception (std::thread returns void)
+		using status_type = int;
+		status_type join();
+
+		// New members for Process
+
+		// kill() terminates a process in the usual Unix way
+		void kill(int sig = SIGTERM);
+
+		// take over ownership of an already-forked native process handle
+		Process(id pid);
+
+	private:
+		id m_pid;
+
+		void do_fork(function<int()>);
+	};
+
+	template <>
+	struct ResourceTraits<Process::id, Process> {
+		Process::id get_key(const Process &res) const { return (const_cast<Process&>(res)).native_handle(); }
+		shared_ptr<Process> make_resource(const Process::id &id) const { return make_shared<Process>(id); }
+		bool is_null_key(const Process::id &key) const { return !key; }
+		Process::id get_null_key() const { return 0; }
+	};
+
+	typedef ResourceHandle<Process::id, Process> Pid;
+
+	pid_t gettid();
+}
+#endif // CRUCIBLE_PROCESS_H

include/crucible/resource.h

@@ -0,0 +1,387 @@
+#ifndef CRUCIBLE_RESOURCE_H
+#define CRUCIBLE_RESOURCE_H
+
+#include "crucible/error.h"
+
+#include <cassert>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <iostream>
+
+namespace crucible {
+	using namespace std;
+
+	// Template classes for non-copiable resource owner objects
+	// for objects with process-wide unique names.
+
+	// Everything we need to know about Key and Resource.
+	// Specialize this template for your Resource class.
+	template <class Key, class Resource>
+	struct ResourceTraits {
+		// How to get the Key out of a Resource owner.
+		// If the owner owns no resource, returns "null" for "no Resource."
+		Key get_key(const Resource &res) const;
+
+		// How to construct a new Resource owner given _only_ the key.
+		// Usually just calls make_shared<Resource>(key).
+		shared_ptr<Resource> make_resource(const Key &key) const;
+
+		// Test a Key value to see if it is null (no active Resource has this Key value).
+		// Usually an equality test with get_null_key(), but sometimes many Key values are equivalent to null.
+		bool is_null_key(const Key &key) const;
+
+		// is_null_key(get_null_key()) == true
+		Key get_null_key() const;
+	};
+
+	template <class Key, class Resource>
+	class ResourceHandle {
+	public:
+		using key_type = Key;
+		using resource_type = Resource;
+		using resource_ptr_type = shared_ptr<Resource>;
+
+	private:
+		using traits_type = ResourceTraits<Key, Resource>;
+
+		class ResourceHolder {
+			resource_ptr_type m_ptr;
+		public:
+			~ResourceHolder();
+			ResourceHolder(resource_ptr_type that);
+			ResourceHolder(const ResourceHolder &that) = default;
+			ResourceHolder(ResourceHolder &&that) = default;
+			ResourceHolder& operator=(ResourceHolder &&that) = default;
+			ResourceHolder& operator=(const ResourceHolder &that) = default;
+			resource_ptr_type get_resource_ptr() const;
+		};
+
+		using holder_ptr_type = shared_ptr<ResourceHolder>;
+		using weak_holder_ptr_type = weak_ptr<ResourceHolder>;
+		using map_type = map<key_type, weak_holder_ptr_type>;
+
+		// The only instance variable
+		holder_ptr_type m_ptr;
+
+		// A bunch of static variables and functions
+		static mutex &s_mutex();
+		static shared_ptr<map_type> s_map();
+		static holder_ptr_type insert(const key_type &key);
+		static holder_ptr_type insert(const resource_ptr_type &res);
+		static void erase(const key_type &key);
+		static ResourceTraits<Key, Resource> s_traits;
+
+	public:
+
+		// test for resource.  A separate operator because key_type could be confused with bool.
+		bool operator!() const;
+
+		// get key_type for an active resource or null
+		key_type get_key() const;
+
+		// conversion/assignment to and from key_type
+		operator key_type() const;
+		ResourceHandle(const key_type &key);
+		ResourceHandle& operator=(const key_type &key);
+
+		// conversion to/from resource_ptr_type
+		ResourceHandle(const resource_ptr_type &res);
+		ResourceHandle& operator=(const resource_ptr_type &res);
+
+		// default constructor is public
+		ResourceHandle() = default;
+
+		// forward anything else to the Resource constructor
+		// if we can do so unambiguously
+		template<class A1, class A2, class... Args>
+		ResourceHandle(A1 a1, A2 a2, Args... args) : ResourceHandle( make_shared<Resource>(a1, a2, args...) )
+		{
+		}
+
+		// forward anything else to a Resource factory method
+		template<class... Args>
+		static
+		ResourceHandle
+		make(Args... args) {
+			return ResourceHandle( make_shared<Resource>(args...) );
+		}
+
+		// get pointer to Resource object (nothrow, result may be null)
+		resource_ptr_type get_resource_ptr() const;
+		// this version throws and is probably not thread safe
+		resource_ptr_type operator->() const;
+
+		// dynamic casting of the resource (throws if cast fails)
+		template <class T> shared_ptr<T> cast() const;
+	};
+
+	template <class Key, class Resource>
+	Key
+	ResourceTraits<Key, Resource>::get_key(const Resource &res) const
+	{
+		return res.get_key();
+	}
+
+	template <class Key, class Resource>
+	shared_ptr<Resource>
+	ResourceTraits<Key, Resource>::make_resource(const Key &key) const
+	{
+		return make_shared<Resource>(key);
+	}
+
+	template <class Key, class Resource>
+	bool
+	ResourceTraits<Key, Resource>::is_null_key(const Key &key) const
+	{
+		return !key;
+	}
+
+	template <class Key, class Resource>
+	Key
+	ResourceTraits<Key, Resource>::get_null_key() const
+	{
+		return NULL;
+	}
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>::ResourceHolder::ResourceHolder(resource_ptr_type that) :
+		m_ptr(that)
+	{
+		// Cannot insert ourselves here since our shared_ptr does not exist yet.
+	}
+
+	template <class Key, class Resource>
+	mutex &
+	ResourceHandle<Key, Resource>::s_mutex()
+	{
+		static mutex gcc_won_t_instantiate_this_either;
+		return gcc_won_t_instantiate_this_either;
+	}
+
+	template <class Key, class Resource>
+	shared_ptr<typename ResourceHandle<Key, Resource>::map_type>
+	ResourceHandle<Key, Resource>::s_map()
+	{
+		static shared_ptr<map_type> gcc_won_t_instantiate_the_damn_static_vars;
+		if (!gcc_won_t_instantiate_the_damn_static_vars) {
+			gcc_won_t_instantiate_the_damn_static_vars = make_shared<map_type>();
+		}
+		return gcc_won_t_instantiate_the_damn_static_vars;
+	}
+
+	template <class Key, class Resource>
+	void
+	ResourceHandle<Key, Resource>::erase(const key_type &key)
+	{
+		unique_lock<mutex> lock(s_mutex());
+		// Resources are allowed to set their Keys to null.
+		if (s_traits.is_null_key(key)) {
+			// Clean out any dead weak_ptr objects.
+			for (auto i = s_map()->begin(); i != s_map()->end(); ) {
+				if (! (*i).second.lock()) {
+					i = s_map()->erase(i);
+				} else {
+					++i;
+				}
+			}
+			return;
+		}
+		auto erased = s_map()->erase(key);
+		if (erased != 1) {
+			cerr << __PRETTY_FUNCTION__ << ": WARNING: s_map()->erase(" << key << ") returned " << erased << " != 1" << endl;
+		}
+	}
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>::ResourceHolder::~ResourceHolder()
+	{
+		if (!m_ptr) {
+			// Probably something harmless like a failed constructor.
+			cerr << __PRETTY_FUNCTION__ << ": WARNING: destroying null m_ptr" << endl;
+			return;
+		}
+		Key key = s_traits.get_key(*m_ptr);
+		ResourceHandle::erase(key);
+	}
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::holder_ptr_type
+	ResourceHandle<Key, Resource>::insert(const key_type &key)
+	{
+		// no Resources for null keys
+		if (s_traits.is_null_key(key)) {
+			return holder_ptr_type();
+		}
+		unique_lock<mutex> lock(s_mutex());
+		// find ResourceHolder for non-null key
+		auto found = s_map()->find(key);
+		if (found != s_map()->end()) {
+			holder_ptr_type rv = (*found).second.lock();
+			// a weak_ptr may have expired
+			if (rv) {
+				return rv;
+			}
+		}
+		// not found or expired, throw any existing ref away and make a new one
+		resource_ptr_type rpt = s_traits.make_resource(key);
+		holder_ptr_type hpt = make_shared<ResourceHolder>(rpt);
+		// store weak_ptr in map
+		(*s_map())[key] = hpt;
+		// return shared_ptr
+		return hpt;
+	};
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::holder_ptr_type
+	ResourceHandle<Key, Resource>::insert(const resource_ptr_type &res)
+	{
+		// no Resource, no ResourceHolder.
+		if (!res) {
+			return holder_ptr_type();
+		}
+		// no ResourceHolders for null keys either.
+		key_type key = s_traits.get_key(*res);
+		if (s_traits.is_null_key(key)) {
+			return holder_ptr_type();
+		}
+		unique_lock<mutex> lock(s_mutex());
+		// find ResourceHolder for non-null key
+		auto found = s_map()->find(key);
+		if (found != s_map()->end()) {
+			holder_ptr_type rv = (*found).second.lock();
+			// The map doesn't own the ResourceHolders, the ResourceHandles do.
+			// It's OK for the map to contain an expired weak_ptr to some dead ResourceHolder...
+			if (rv) {
+				// found ResourceHolder, look at pointer
+				resource_ptr_type rp = rv->get_resource_ptr();
+				// We do not store references to null Resources.
+				assert(rp);
+				// Key retrieved for an existing object must match key searched or be null.
+				key_type found_key = s_traits.get_key(*rp);
+				bool found_key_is_null = s_traits.is_null_key(found_key);
+				assert(found_key_is_null || found_key == key);
+				if (!found_key_is_null) {
+					// We do not store references to duplicate resources.
+					if (rp.owner_before(res) || res.owner_before(rp)) {
+						cerr << "inserting new Resource with existing Key " << key << " not allowed at " << __PRETTY_FUNCTION__ << endl;;
+						abort();
+						// THROW_ERROR(out_of_range, "inserting new Resource with existing Key " << key << " not allowed at " << __PRETTY_FUNCTION__);
+					}
+					// rv is good, return it
+					return rv;
+				}
+			}
+		}
+		// not found or expired, make a new one
+		holder_ptr_type rv = make_shared<ResourceHolder>(res);
+		s_map()->insert(make_pair(key, weak_holder_ptr_type(rv)));
+		// no need to check s_map result, we are either replacing a dead weak_ptr or adding a new one
+		return rv;
+	};
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>::ResourceHandle(const key_type &key)
+	{
+		m_ptr = insert(key);
+	}
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>&
+	ResourceHandle<Key, Resource>::operator=(const key_type &key)
+	{
+		m_ptr = insert(key);
+		return *this;
+	}
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>::ResourceHandle(const resource_ptr_type &res)
+	{
+		m_ptr = insert(res);
+	}
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>&
+	ResourceHandle<Key, Resource>::operator=(const resource_ptr_type &res)
+	{
+		m_ptr = insert(res);
+		return *this;
+	}
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::resource_ptr_type
+	ResourceHandle<Key, Resource>::ResourceHolder::get_resource_ptr() const
+	{
+		return m_ptr;
+	}
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::resource_ptr_type
+	ResourceHandle<Key, Resource>::get_resource_ptr() const
+	{
+		if (!m_ptr) {
+			return resource_ptr_type();
+		}
+		return m_ptr->get_resource_ptr();
+	}
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::resource_ptr_type
+	ResourceHandle<Key, Resource>::operator->() const
+	{
+		resource_ptr_type rp = get_resource_ptr();
+		if (!rp) {
+			THROW_ERROR(out_of_range, __PRETTY_FUNCTION__ << " called on null Resource");
+		}
+		return rp;
+	}
+
+	template <class Key, class Resource>
+	template <class T>
+	shared_ptr<T>
+	ResourceHandle<Key, Resource>::cast() const
+	{
+		shared_ptr<T> dp;
+		resource_ptr_type rp = get_resource_ptr();
+		if (!rp) {
+			return dp;
+		}
+		dp = dynamic_pointer_cast<T>(rp);
+		if (!dp) {
+			throw bad_cast();
+		}
+		return dp;
+	}
+
+	template <class Key, class Resource>
+	typename ResourceHandle<Key, Resource>::key_type
+	ResourceHandle<Key, Resource>::get_key() const
+	{
+		resource_ptr_type rp = get_resource_ptr();
+		if (!rp) {
+			return s_traits.get_null_key();
+		} else {
+			return s_traits.get_key(*rp);
+		}
+	}
+
+	template <class Key, class Resource>
+	ResourceHandle<Key, Resource>::operator key_type() const
+	{
+		return get_key();
+	}
+
+	template <class Key, class Resource>
+	bool
+	ResourceHandle<Key, Resource>::operator!() const
+	{
+		return s_traits.is_null_key(operator key_type());
+	}
+
+	template <class Key, class Resource>
+	ResourceTraits<Key, Resource> ResourceHandle<Key, Resource>::s_traits;
+
+
+}
+
+#endif // RESOURCE_H

include/crucible/string.h

@@ -0,0 +1,67 @@
+#ifndef CRUCIBLE_STRING_H
+#define CRUCIBLE_STRING_H
+
+#include "crucible/error.h"
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <vector>
+
+namespace crucible {
+	using namespace std;
+
+	// Zero-initialize a base class object (usually a C struct)
+	template <class Base>
+	void
+	memset_zero(Base *that)
+	{
+		memset(that, 0, sizeof(Base));
+	}
+
+	// Copy a base class object (usually a C struct) into a vector<char>
+	template <class Base>
+	vector<char>
+	vector_copy_struct(Base *that)
+	{
+		const char *begin_that = reinterpret_cast<const char *>(static_cast<const Base *>(that));
+		return vector<char>(begin_that, begin_that + sizeof(Base));
+	}
+
+	// int->hex conversion with sprintf
+	string to_hex(uint64_t i);
+
+	// hex->int conversion with stoull
+	uint64_t from_hex(const string &s);
+
+	// asprintf with string output and exceptions
+	template<class... Args>
+	string
+	astringprintf(const char *fmt, Args... args)
+	{
+		char *rv = NULL;
+		DIE_IF_MINUS_ONE(asprintf(&rv, fmt, args...));
+		string rv_string = rv;
+		free(rv);
+		return rv_string;
+	}
+
+	template<class... Args>
+	string
+	astringprintf(const string &fmt, Args... args)
+	{
+		return astringprintf(fmt.c_str(), args...);
+	}
+
+	vector<string> split(string delim, string s);
+
+	// Shut up and give me the difference between two pointers
+	template <class P1, class P2>
+	ptrdiff_t
+	pointer_distance(const P1 *a, const P2 *b)
+	{
+		return reinterpret_cast<const char *>(a) - reinterpret_cast<const char *>(b);
+	}
+};
+
+#endif // CRUCIBLE_STRING_H

include/crucible/time.h

@@ -0,0 +1,49 @@
+#ifndef CRUCIBLE_TIME_H
+#define CRUCIBLE_TIME_H
+
+#include "crucible/error.h"
+
+#include <chrono>
+#include <mutex>
+#include <ostream>
+
+namespace crucible {
+
+	double nanosleep(double secs);
+
+	class Timer {
+		chrono::high_resolution_clock::time_point m_start;
+
+	public:
+		Timer();
+		double age() const;
+		double report(int precision = 1000) const;
+		void reset();
+		void set(const chrono::high_resolution_clock::time_point &start);
+		void set(double delta);
+		double lap();
+		bool operator<(double d) const;
+		bool operator>(double d) const;
+	};
+
+	ostream &operator<<(ostream &os, const Timer &t);
+
+	class RateLimiter {
+		Timer	m_timer;
+		double	m_rate;
+		double	m_burst;
+		double  m_tokens;
+		mutex	m_mutex;
+
+		void update_tokens();
+	public:
+		RateLimiter(double rate, double burst);
+		RateLimiter(double rate);
+		void sleep_for(double cost = 1.0);
+		bool is_ready();
+		void borrow(double cost = 1.0);
+	};
+
+}
+
+#endif // CRUCIBLE_TIME_H

include/crucible/timequeue.h

@@ -0,0 +1,188 @@
+#ifndef CRUCIBLE_TIMEQUEUE_H
+#define CRUCIBLE_TIMEQUEUE_H
+
+#include <crucible/error.h>
+#include <crucible/time.h>
+
+#include <condition_variable>
+#include <limits>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <set>
+
+namespace crucible {
+	using namespace std;
+
+	template <class Task>
+	class TimeQueue {
+
+	public:
+		using Timestamp = chrono::high_resolution_clock::time_point;
+
+	private:
+		struct Item {
+			Timestamp m_time;
+			unsigned m_id;
+			Task m_task;
+
+			bool operator<(const Item &that) const {
+				if (m_time < that.m_time) return true;
+				if (that.m_time < m_time) return false;
+				return m_id < that.m_id;
+			}
+			static unsigned s_id;
+
+			Item(const Timestamp &time, const Task& task) :
+				m_time(time),
+				m_id(++s_id),
+				m_task(task)
+			{
+			}
+
+		};
+
+		set<Item>		m_set;
+		mutable mutex		m_mutex;
+		condition_variable	m_cond_full, m_cond_empty;
+		size_t			m_max_queue_depth;
+
+	public:
+		~TimeQueue();
+		TimeQueue(size_t max_queue_depth = numeric_limits<size_t>::max());
+
+		void push(const Task &task, double delay = 0);
+		void push_nowait(const Task &task, double delay = 0);
+		Task pop();
+		bool pop_nowait(Task &t);
+		double when() const;
+
+		size_t size() const;
+		bool empty() const;
+
+		list<Task> peek(size_t count) const;
+	};
+
+	template <class Task> unsigned TimeQueue<Task>::Item::s_id = 0;
+
+	template <class Task>
+	TimeQueue<Task>::~TimeQueue()
+	{
+		if (!m_set.empty()) {
+			cerr << "ERROR: " << m_set.size() << " locked items still in TimeQueue at destruction" << endl;
+		}
+	}
+
+	template <class Task>
+	void
+	TimeQueue<Task>::push(const Task &task, double delay)
+	{
+		Timestamp time = chrono::high_resolution_clock::now() + 
+			chrono::duration_cast<chrono::high_resolution_clock::duration>(chrono::duration<double>(delay));
+		unique_lock<mutex> lock(m_mutex);
+		while (m_set.size() > m_max_queue_depth) {
+			m_cond_full.wait(lock);
+		}
+		m_set.insert(Item(time, task));
+		m_cond_empty.notify_all();
+	}
+
+	template <class Task>
+	void
+	TimeQueue<Task>::push_nowait(const Task &task, double delay)
+	{
+		Timestamp time = chrono::high_resolution_clock::now() + 
+			chrono::duration_cast<chrono::high_resolution_clock::duration>(chrono::duration<double>(delay));
+		unique_lock<mutex> lock(m_mutex);
+		m_set.insert(Item(time, task));
+		m_cond_empty.notify_all();
+	}
+
+	template <class Task>
+	Task
+	TimeQueue<Task>::pop()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		while (1) {
+			while (m_set.empty()) {
+				m_cond_empty.wait(lock);
+			}
+			Timestamp now = chrono::high_resolution_clock::now();
+			if (now > m_set.begin()->m_time) {
+				Task rv = m_set.begin()->m_task;
+				m_set.erase(m_set.begin());
+				m_cond_full.notify_all();
+				return rv;
+			}
+			m_cond_empty.wait_until(lock, m_set.begin()->m_time);
+		}
+	}
+
+	template <class Task>
+	bool
+	TimeQueue<Task>::pop_nowait(Task &t)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		if (m_set.empty()) {
+			return false;
+		}
+		Timestamp now = chrono::high_resolution_clock::now();
+		if (now <= m_set.begin()->m_time) {
+			return false;
+		}
+		t = m_set.begin()->m_task;
+		m_set.erase(m_set.begin());
+		m_cond_full.notify_all();
+		return true;
+	}
+
+	template <class Task>
+	double
+	TimeQueue<Task>::when() const
+	{
+		unique_lock<mutex> lock(m_mutex);
+		if (m_set.empty()) {
+			return numeric_limits<double>::infinity();
+		}
+		return chrono::duration<double>(m_set.begin()->m_time - chrono::high_resolution_clock::now()).count();
+	}
+
+	template <class Task>
+	size_t
+	TimeQueue<Task>::size() const
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set.size();
+	}
+
+	template <class Task>
+	bool
+	TimeQueue<Task>::empty() const
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set.empty();
+	}
+
+	template <class Task>
+	list<Task>
+	TimeQueue<Task>::peek(size_t count) const
+	{
+		unique_lock<mutex> lock(m_mutex);
+		list<Task> rv;
+		auto it = m_set.begin();
+		while (count-- && it != m_set.end()) {
+			rv.push_back(it->m_task);
+			++it;
+		}
+		return rv;
+	}
+
+	template <class Task>
+	TimeQueue<Task>::TimeQueue(size_t max_depth) :
+		m_max_queue_depth(max_depth)
+	{
+	}
+
+}
+
+#endif // CRUCIBLE_TIMEQUEUE_H

include/crucible/uuid.h

@@ -0,0 +1,14 @@
+#ifndef CRUCIBLE_UUID_H
+#define CRUCIBLE_UUID_H
+
+#include <string>
+
+#include <uuid/uuid.h>
+
+namespace crucible {
+	using namespace std;
+
+	string uuid_unparse(const unsigned char a[16]);
+}
+
+#endif // CRUCIBLE_UUID_H

include/crucible/workqueue.h

@@ -0,0 +1,189 @@
+#ifndef CRUCIBLE_WORKQUEUE_H
+#define CRUCIBLE_WORKQUEUE_H
+
+#include <crucible/error.h>
+
+#include <condition_variable>
+#include <limits>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <set>
+
+namespace crucible {
+	using namespace std;
+
+	template <class Task>
+	class WorkQueue {
+
+	public:
+		using set_type = set<Task>;
+		using key_type = Task;
+
+	private:
+
+		set_type		m_set;
+		mutable mutex			m_mutex;
+		condition_variable	m_cond_full, m_cond_empty;
+		size_t			m_max_queue_depth;
+
+	public:
+		~WorkQueue();
+		template <class... Args> WorkQueue(size_t max_queue_depth, Args... args);
+		template <class... Args> WorkQueue(Args... args);
+
+		void push(const key_type &name);
+		void push_wait(const key_type &name, size_t limit);
+		void push_nowait(const key_type &name);
+
+		key_type pop();
+		bool pop_nowait(key_type &rv);
+		key_type peek();
+
+		size_t size() const;
+		bool empty();
+		set_type copy();
+		list<Task> peek(size_t count) const;
+
+	};
+
+	template <class Task>
+	WorkQueue<Task>::~WorkQueue()
+	{
+		if (!m_set.empty()) {
+			cerr << "ERROR: " << m_set.size() << " locked items still in WorkQueue " << this << " at destruction" << endl;
+		}
+	}
+
+	template <class Task>
+	void
+	WorkQueue<Task>::push(const key_type &name)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		while (!m_set.count(name) && m_set.size() > m_max_queue_depth) {
+			m_cond_full.wait(lock);
+		}
+		m_set.insert(name);
+		m_cond_empty.notify_all();
+	}
+
+	template <class Task>
+	void
+	WorkQueue<Task>::push_wait(const key_type &name, size_t limit)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		while (!m_set.count(name) && m_set.size() >= limit) {
+			m_cond_full.wait(lock);
+		}
+		m_set.insert(name);
+		m_cond_empty.notify_all();
+	}
+
+	template <class Task>
+	void
+	WorkQueue<Task>::push_nowait(const key_type &name)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		m_set.insert(name);
+		m_cond_empty.notify_all();
+	}
+
+	template <class Task>
+	typename WorkQueue<Task>::key_type
+	WorkQueue<Task>::pop()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		while (m_set.empty()) {
+			m_cond_empty.wait(lock);
+		}
+		key_type rv = *m_set.begin();
+		m_set.erase(m_set.begin());
+		m_cond_full.notify_all();
+		return rv;
+	}
+
+	template <class Task>
+	bool
+	WorkQueue<Task>::pop_nowait(key_type &rv)
+	{
+		unique_lock<mutex> lock(m_mutex);
+		if (m_set.empty()) {
+			return false;
+		}
+		rv = *m_set.begin();
+		m_set.erase(m_set.begin());
+		m_cond_full.notify_all();
+		return true;
+	}
+
+	template <class Task>
+	typename WorkQueue<Task>::key_type
+	WorkQueue<Task>::peek()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		if (m_set.empty()) {
+			return key_type();
+		} else {
+			return *m_set.begin();
+		}
+	}
+
+	template <class Task>
+	size_t
+	WorkQueue<Task>::size() const
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set.size();
+	}
+
+	template <class Task>
+	bool
+	WorkQueue<Task>::empty()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set.empty();
+	}
+
+	template <class Task>
+	typename WorkQueue<Task>::set_type
+	WorkQueue<Task>::copy()
+	{
+		unique_lock<mutex> lock(m_mutex);
+		return m_set;
+	}
+
+	template <class Task>
+	list<Task>
+	WorkQueue<Task>::peek(size_t count) const
+	{
+		unique_lock<mutex> lock(m_mutex);
+		list<Task> rv;
+		for (auto i : m_set) {
+			if (count--) {
+				rv.push_back(i);
+			} else {
+				break;
+			}
+		}
+		return rv;
+	}
+
+	template <class Task>
+	template <class... Args>
+	WorkQueue<Task>::WorkQueue(Args... args) :
+		m_set(args...),
+		m_max_queue_depth(numeric_limits<size_t>::max())
+	{
+	}
+
+	template <class Task>
+	template <class... Args>
+	WorkQueue<Task>::WorkQueue(size_t max_depth, Args... args) :
+		m_set(args...),
+		m_max_queue_depth(max_depth)
+	{
+	}
+
+}
+
+#endif // CRUCIBLE_WORKQUEUE_H