GGLinnk/bees
1
0
Fork 0
mirror of https://github.com/Zygo/bees.git synced 2025-08-03 14:23:29 +02:00
Code Packages Activity

bees: remove local cruft, throw at github

Browse Source
This commit is contained in:
Zygo Blaxell
2016-11-15 23:32:44 -05:00
commit cca0ee26a8
66 changed files with 12785 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
lib/Makefile Normal file
View File

@@ -0,0 +1,37 @@
default: libcrucible.so
OBJS = \
crc64.o \
chatter.o \
error.o \
execpipe.o \
extentwalker.o \
fd.o \
fs.o \
interp.o \
ntoa.o \
path.o \
process.o \
string.o \
time.o \
uuid.o \
include ../makeflags
LDFLAGS = -shared -luuid
depends.mk: *.c *.cc
for x in *.c; do $(CC) $(CFLAGS) -M "$$x"; done > depends.mk.new
for x in *.cc; do $(CXX) $(CXXFLAGS) -M "$$x"; done >> depends.mk.new
mv -fv depends.mk.new depends.mk
-include depends.mk
%.o: %.c
$(CC) $(CFLAGS) -o $@ -c $<
%.o: %.cc ../include/crucible/%.h
$(CXX) $(CXXFLAGS) -o $@ -c $<
libcrucible.so: $(OBJS) Makefile
$(CXX) $(LDFLAGS) -o $@ $(OBJS)

140
lib/chatter.cc Normal file
View File

@@ -0,0 +1,140 @@
#include "crucible/chatter.h"
#include "crucible/error.h"
#include "crucible/path.h"
#include "crucible/process.h"
#include <cassert>
#include <ctime>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <pthread.h>
namespace crucible {
using namespace std;
static auto_ptr<set<string>> chatter_names;
static const char *SPACETAB = " \t";
static
void
init_chatter_names()
{
if (!chatter_names.get()) {
chatter_names.reset(new set<string>);
const char *sp = ::getenv("CRUCIBLE_CHATTER");
if (sp) {
cerr << "CRUCIBLE_CHATTER = '" << sp << "'" << endl;
string s(sp);
while (!s.empty()) {
s.erase(0, s.find_first_not_of(SPACETAB));
if (s.empty()) {
break;
}
size_t last = s.find_first_of(SPACETAB);
string first_word = s.substr(0, last);
cerr << "\t'" << first_word << "'" << endl;
chatter_names->insert(first_word);
s.erase(0, last);
}
}
}
}
Chatter::Chatter(string name, ostream &os)
: m_name(name), m_os(os)
{
}
Chatter::~Chatter()
{
ostringstream header_stream;
time_t ltime;
DIE_IF_MINUS_ONE(time(&ltime));
struct tm ltm;
DIE_IF_ZERO(localtime_r(&ltime, &ltm));
char buf[1024];
DIE_IF_ZERO(strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &ltm));
header_stream << buf;
header_stream << " " << getpid() << "." << gettid();
if (!m_name.empty()) {
header_stream << " " << m_name;
}
header_stream << ": ";
string out = m_oss.str();
string header = header_stream.str();
string::size_type start = 0;
while (start < out.size()) {
size_t end_line = out.find_first_of("\n", start);
if (end_line != string::npos) {
assert(out[end_line] == '\n');
size_t end = end_line;
m_os << (header + out.substr(start, end - start) + "\n") << flush;
start = end_line + 1;
} else {
m_os << (header + out.substr(start) + "\n") << flush;
start = out.size();
}
}
}
Chatter::Chatter(Chatter &&c)
: m_name(c.m_name), m_os(c.m_os), m_oss(c.m_oss.str())
{
c.m_oss.str("");
}
set<ChatterBox*> ChatterBox::s_boxes;
set<ChatterBox*>& ChatterBox::all_boxes()
{
return s_boxes;
}
ChatterBox::ChatterBox(string file, int line, string pretty_function, ostream &os)
: m_file(basename(file)), m_line(line), m_pretty_function(pretty_function), m_enabled(false), m_os(os)
{
s_boxes.insert(this);
init_chatter_names();
if (chatter_names->find(m_file) != chatter_names->end()) {
m_enabled = true;
} else if (chatter_names->find(m_pretty_function) != chatter_names->end()) {
m_enabled = true;
} else if (!chatter_names->empty()) {
cerr << "CRUCIBLE_CHATTER does not list '" << m_file << "' or '" << m_pretty_function << "'" << endl;
}
// cerr << "ChatterBox " << reinterpret_cast<void*>(this) << " constructed" << endl;
}
ChatterBox::~ChatterBox()
{
s_boxes.erase(this);
// cerr << "ChatterBox " << reinterpret_cast<void*>(this) << " destructed" << endl;
}
void
ChatterBox::set_enable(bool en)
{
m_enabled = en;
}
ChatterUnwinder::ChatterUnwinder(function<void()> f) :
m_func(f)
{
}
ChatterUnwinder::~ChatterUnwinder()
{
if (uncaught_exception()) {
m_func();
}
}
};

59
lib/crc64.cc Normal file
View File

@@ -0,0 +1,59 @@
#include "crucible/crc64.h"
#define POLY64REV 0xd800000000000000ULL
namespace crucible {
static bool init = false;
static uint64_t CRCTable[256];
static void init_crc64_table()
{
if (!init) {
for (int i = 0; i <= 255; i++) {
uint64_t part = i;
for (int j = 0; j < 8; j++) {
if (part & 1) {
part = (part >> 1) ^ POLY64REV;
} else {
part >>= 1;
}
}
CRCTable[i] = part;
}
init = true;
}
}
uint64_t
Digest::CRC::crc64(const char *s)
{
init_crc64_table();
uint64_t crc = 0;
for (; *s; s++) {
uint64_t temp1 = crc >> 8;
uint64_t temp2 = CRCTable[(crc ^ static_cast<uint64_t>(*s)) & 0xff];
crc = temp1 ^ temp2;
}
return crc;
}
uint64_t
Digest::CRC::crc64(const void *p, size_t len)
{
init_crc64_table();
uint64_t crc = 0;
for (const unsigned char *s = static_cast<const unsigned char *>(p); len; --len) {
uint64_t temp1 = crc >> 8;
uint64_t temp2 = CRCTable[(crc ^ *s++) & 0xff];
crc = temp1 ^ temp2;
}
return crc;
}
};

74
lib/error.cc Normal file
View File

@@ -0,0 +1,74 @@
#include "crucible/error.h"
#include <cstdarg>
#include <iostream>
#include <cxxabi.h>
namespace crucible {
using namespace std;
static
string
analyze_exception(const exception &e)
{
// Let's ignore all the potential memory allocation exceptions for now, K?
ostringstream oss;
int status;
char *realname = abi::__cxa_demangle(typeid(e).name(), 0, 0, &status);
oss << "exception type ";
// This is questionable since anything that would cause
// cxa_demangle to fail will probably cause an exception anyway.
if (realname) {
oss << realname;
free(realname);
} else {
oss << typeid(e).name();
}
oss << ": " << e.what();
return oss.str();
}
// FIXME: could probably avoid some of these levels of indirection
static
function<void(string s)> current_catch_explainer = [&](string s) {
cerr << s << endl;
};
void
set_catch_explainer(function<void(string s)> f)
{
current_catch_explainer = f;
}
void
default_catch_explainer(string s)
{
current_catch_explainer(s);
}
int
catch_all(const function<void()> &f, const function<void(string)> &explainer)
{
try {
f();
return 0;
} catch (const exception &e) {
explainer(analyze_exception(e));
return 1;
}
}
void
catch_and_explain(const function<void()> &f, const function<void(string)> &explainer)
{
try {
f();
} catch (const exception &e) {
explainer(analyze_exception(e));
throw;
}
}
};

104
lib/execpipe.cc Normal file
View File

@@ -0,0 +1,104 @@
#include "crucible/execpipe.h"
#include "crucible/chatter.h"
#include "crucible/error.h"
#include "crucible/process.h"
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <unistd.h>
namespace crucible {
using namespace std;
void
redirect_stdin(const Fd &child_fd)
{
dup2_or_die(child_fd, STDIN_FILENO);
}
void
redirect_stdin_stdout(const Fd &child_fd)
{
dup2_or_die(child_fd, STDOUT_FILENO);
dup2_or_die(child_fd, STDIN_FILENO);
}
void
redirect_stdin_stdout_stderr(const Fd &child_fd)
{
dup2_or_die(child_fd, STDERR_FILENO);
dup2_or_die(child_fd, STDOUT_FILENO);
dup2_or_die(child_fd, STDIN_FILENO);
}
void
redirect_stdout_stderr(const Fd &child_fd)
{
dup2_or_die(child_fd, STDERR_FILENO);
dup2_or_die(child_fd, STDOUT_FILENO);
}
void
redirect_stdout(const Fd &child_fd)
{
dup2_or_die(child_fd, STDOUT_FILENO);
}
void
redirect_stderr(const Fd &child_fd)
{
dup2_or_die(child_fd, STDERR_FILENO);
}
Fd popen(function<int()> f, function<void(const Fd &child_fd)> import_fd_fn)
{
Fd parent_fd, child_fd;
{
pair<Fd, Fd> fd_pair = socketpair_or_die();
parent_fd = fd_pair.first;
child_fd = fd_pair.second;
}
pid_t fv;
DIE_IF_MINUS_ONE(fv = fork());
if (fv) {
child_fd->close();
return parent_fd;
} else {
int rv = EXIT_FAILURE;
catch_all([&]() {
parent_fd->close();
import_fd_fn(child_fd);
// system("ls -l /proc/$$/fd/ >&2");
rv = f();
});
_exit(rv);
cerr << "PID " << getpid() << " TID " << gettid() << "STILL ALIVE" << endl;
system("ls -l /proc/$$/task/ >&2");
exit(EXIT_FAILURE);
}
}
string
read_all(Fd fd, size_t max_bytes, size_t chunk_bytes)
{
char buf[chunk_bytes];
string str;
size_t rv;
while (1) {
read_partial_or_die(fd, static_cast<void *>(buf), chunk_bytes, rv);
if (rv == 0) {
break;
}
if (max_bytes - str.size() < rv) {
THROW_ERROR(out_of_range, "Output size limit " << max_bytes << " exceeded by appending " << rv << " bytes read to " << str.size() << " already in string");
}
str.append(buf, rv);
}
return str;
}
}

630
lib/extentwalker.cc Normal file
View File

@@ -0,0 +1,630 @@
#include "crucible/extentwalker.h"
#include "crucible/chatter.h"
#include "crucible/error.h"
#include "crucible/fs.h"
#include "crucible/limits.h"
#include "crucible/string.h"
namespace crucible {
using namespace std;
const off_t ExtentWalker::sc_step_size;
// fm_start, fm_length, fm_flags, m_extents
// fe_logical, fe_physical, fe_length, fe_flags
static const off_t MAX_OFFSET = numeric_limits<off_t>::max();
static const off_t FIEMAP_BLOCK_SIZE = 4096;
static bool __ew_do_log = getenv("EXTENTWALKER_DEBUG");
#define EWLOG(x) do { \
if (__ew_do_log) { \
CHATTER(x); \
} \
} while (0)
ostream &
operator<<(ostream &os, const Extent &e)
{
os << "Extent {"
<< " begin = " << to_hex(e.m_begin)
<< ", end = " << to_hex(e.m_end)
<< ", physical = " << to_hex(e.m_physical)
<< ", flags = ";
if (e.m_flags & Extent::HOLE) {
os << "Extent::HOLE|";
}
if (e.m_flags & Extent::PREALLOC) {
os << "Extent::PREALLOC|";
}
if (e.m_flags & Extent::OBSCURED) {
os << "Extent::OBSCURED|";
}
if (e.m_flags & ~(Extent::HOLE | Extent::PREALLOC | Extent::OBSCURED)) {
os << fiemap_extent_flags_ntoa(e.m_flags & ~(Extent::HOLE | Extent::PREALLOC | Extent::OBSCURED));
}
if (e.m_physical_len) {
os << ", physical_len = " << to_hex(e.m_physical_len);
}
if (e.m_logical_len) {
os << ", logical_len = " << to_hex(e.m_logical_len);
}
if (e.m_offset) {
os << ", offset = " << to_hex(e.m_offset);
}
return os << " }";
}
ostream &
operator<<(ostream &os, const ExtentWalker::Vec &v)
{
os << "ExtentWalker::Vec {";
for (auto e : v) {
os << "\n\t" << e;
}
return os << "}";
}
ostream &
operator<<(ostream &os, const ExtentWalker &ew)
{
return os << "ExtentWalker {"
<< " fd = " << name_fd(ew.m_fd)
<< ", stat.st_size = " << to_hex(ew.m_stat.st_size)
<< ", extents = " << ew.m_extents
<< ", current = [" << ew.m_current - ew.m_extents.begin()
<< "] }";
}
Extent::Extent() :
m_begin(0),
m_end(0),
m_physical(0),
m_flags(0),
m_physical_len(0),
m_logical_len(0),
m_offset(0)
{
}
Extent::operator bool() const
{
THROW_CHECK2(invalid_argument, m_begin, m_end, m_end >= m_begin);
return m_end > m_begin;
}
off_t
Extent::size() const
{
THROW_CHECK2(invalid_argument, m_begin, m_end, m_end >= m_begin);
return m_end - m_begin;
}
bool
Extent::operator==(const Extent &that) const
{
return m_begin == that.m_begin && m_end == that.m_end && m_physical == that.m_physical && m_flags == that.m_flags;
}
ExtentWalker::ExtentWalker(Fd fd) :
m_fd(fd),
m_current(m_extents.begin())
{
}
ExtentWalker::ExtentWalker(Fd fd, off_t initial_pos) :
m_fd(fd),
m_current(m_extents.begin())
{
seek(initial_pos);
}
ExtentWalker::Itr
ExtentWalker::find_in_cache(off_t pos)
{
EWLOG("find_in_cache " << to_hex(pos));
// EOF is an annoying special case
if (pos >= m_stat.st_size) {
if (!m_extents.empty() && m_extents.rbegin()->m_end == m_stat.st_size) {
auto i = m_extents.end();
return --i;
}
}
for (auto vi = m_extents.begin(); vi != m_extents.end(); ++vi) {
if (pos >= vi->m_begin && pos < vi->m_end) {
EWLOG("pos " << to_hex(pos) << " in " << *vi);
if (vi == m_extents.begin() && !(m_extents.begin()->m_begin == 0)) {
// Must have an extent before pos, unless
// there can be no extent before pos because pos == 0
EWLOG("can't match first unless begin is BOF");
break;
}
auto ni = vi;
++ni;
if (ni == m_extents.end() && !(vi->m_end >= m_stat.st_size)) {
// Must have an extent after pos, unless
// there can be no extent after pos because pos >= EOF
EWLOG("can't match last unless end past EOF " << to_hex(m_stat.st_size));
break;
}
// Extent surrounded on either side by other known extents
return vi;
}
}
EWLOG("find_in_cache failed: " << *this);
return m_extents.end();
}
void
ExtentWalker::run_fiemap(off_t pos)
{
ostringstream log;
CHATTER_UNWIND("Log of run_fiemap: " << log.str());
EWLOG("pos = " << to_hex(pos));
THROW_CHECK1(invalid_argument, pos, (pos & (FIEMAP_BLOCK_SIZE - 1)) == 0);
Vec fm;
off_t step_size = pos;
off_t begin = pos - min(pos, sc_step_size);
// This loop should not run forever
int loop_count = 0;
int loop_limit = 99;
while (true) {
if (loop_count == 90) {
EWLOG(log.str());
}
THROW_CHECK1(runtime_error, loop_count, loop_count < loop_limit);
++loop_count;
// Get file size every time in case it changes under us
m_stat.fstat(m_fd);
// Get fiemap begin..EOF
fm = get_extent_map(begin);
EWLOG("fiemap result loop count #" << loop_count << ":" << fm);
// This algorithm seeks at least three extents: one before,
// one after, and one containing pos. Files which contain
// two or fewer extents will cause an obvious problem with that,
// so handle those cases separately.
// FIEMAP lies, and we catch it in a lie about the size of the
// second extent. To work around this, try getting more than 3.
// 0..2(ish) extents
if (fm.size() < sc_extent_fetch_min) {
// If we are not at beginning of file, move backward
if (begin > 0) {
step_size /= 2;
auto next_begin = (begin - min(step_size, begin)) & ~(FIEMAP_BLOCK_SIZE - 1);
EWLOG("step backward " << to_hex(begin) << " -> " << to_hex(next_begin) << " extents size " << fm.size());
if (begin == next_begin) {
EWLOG("step backward stopped");
break;
}
begin = next_begin;
continue;
}
// We are at beginning of file and have too few extents.
// Zero extents? Entire file is a hole.
if (fm.empty()) {
EWLOG("zero extents");
break;
}
// We know we have the beginning of the file and at least
// one extent. If the last extent is EOF then we have the
// whole file in the buffer. If the last extent is NOT
// EOF then fiemap did something we didn't expect.
THROW_CHECK1(runtime_error, fm.rbegin()->flags(), fm.rbegin()->flags() & FIEMAP_EXTENT_LAST);
break;
}
// We have at least three extents, so there is now a first and last.
// We want pos to be between first and last. There doesn't have
// to be an extent between these (it could be a hole).
auto &first_extent = fm.at(sc_extent_fetch_min - 2);
auto &last_extent = *fm.rbegin();
EWLOG("first_extent = " << first_extent);
EWLOG("last_extent = " << last_extent);
// First extent must end on or before pos
if (first_extent.end() > pos) {
// Can we move backward?
if (begin > 0) {
step_size /= 2;
auto next_begin = (begin - min(step_size, begin)) & ~(FIEMAP_BLOCK_SIZE - 1);
EWLOG("step backward " << to_hex(begin) << " -> " << to_hex(next_begin) << " extents size " << fm.size());
if (begin == next_begin) {
EWLOG("step backward stopped");
break;
}
begin = next_begin;
continue;
}
// We are as far back as we can go, so there must be no
// extent before pos (i.e. file starts with a hole).
EWLOG("no extent before pos");
break;
}
// First extent ends on or before pos.
// If last extent is EOF then we have the entire file in the buffer.
// pos could be in last extent, so skip the later checks that
// insist pos be located prior to the last extent.
if (last_extent.flags() & FIEMAP_EXTENT_LAST) {
break;
}
// Don't have EOF, must have an extent after pos.
if (last_extent.begin() <= pos) {
step_size /= 2;
auto new_begin = (begin + step_size) & ~(FIEMAP_BLOCK_SIZE - 1);
EWLOG("step forward " << to_hex(begin) << " -> " << to_hex(new_begin));
if (begin == new_begin) {
EWLOG("step forward stopped");
break;
}
begin = new_begin;
continue;
}
// Last extent begins after pos, first extent ends on or before pos.
// All other cases should have been handled before here.
THROW_CHECK2(runtime_error, pos, first_extent, first_extent.end() <= pos);
THROW_CHECK2(runtime_error, pos, last_extent, last_extent.begin() > pos);
// We should probably stop now
break;
}
// Fill in holes so there are Extent records over entire range
auto fmi = fm.begin();
off_t ipos = begin;
Vec new_vec;
// If we mapped the entire file and there are no extents,
// the entire file is a hole.
bool last_extent_is_last = (begin == 0 && fm.empty());
while (fmi != fm.end()) {
Extent new_extent(*fmi);
THROW_CHECK2(runtime_error, ipos, new_extent.m_begin, ipos <= new_extent.m_begin);
if (new_extent.m_begin > ipos) {
Extent hole_extent;
hole_extent.m_begin = ipos;
hole_extent.m_end = fmi->begin();
hole_extent.m_physical = 0;
hole_extent.m_flags = Extent::HOLE;
new_vec.push_back(hole_extent);
ipos += hole_extent.size();
}
THROW_CHECK2(runtime_error, ipos, new_extent.m_begin, ipos == new_extent.m_begin);
new_vec.push_back(new_extent);
ipos += new_extent.size();
last_extent_is_last = fmi->flags() & FIEMAP_EXTENT_LAST;
++fmi;
}
// If we have run out of extents before EOF, insert a hole at the end
if (last_extent_is_last && ipos < m_stat.st_size) {
Extent hole_extent;
hole_extent.m_begin = ipos;
hole_extent.m_end = m_stat.st_size;
hole_extent.m_physical = 0;
hole_extent.m_flags = Extent::HOLE;
if (!new_vec.empty() && new_vec.rbegin()->m_flags & FIEMAP_EXTENT_LAST) {
new_vec.rbegin()->m_flags &= ~(FIEMAP_EXTENT_LAST);
hole_extent.m_flags |= FIEMAP_EXTENT_LAST;
}
new_vec.push_back(hole_extent);
ipos += new_vec.size();
}
THROW_CHECK1(runtime_error, new_vec.size(), !new_vec.empty());
// Allow last extent to extend beyond desired range (e.g. at EOF)
THROW_CHECK2(runtime_error, ipos, new_vec.rbegin()->m_end, ipos <= new_vec.rbegin()->m_end);
// If we have the last extent in the file, truncate it to the file size.
if (ipos >= m_stat.st_size) {
THROW_CHECK2(runtime_error, new_vec.rbegin()->m_begin, m_stat.st_size, m_stat.st_size > new_vec.rbegin()->m_begin);
THROW_CHECK2(runtime_error, new_vec.rbegin()->m_end, m_stat.st_size, m_stat.st_size <= new_vec.rbegin()->m_end);
new_vec.rbegin()->m_end = m_stat.st_size;
}
// Verify contiguous, ascending order, at least one Extent
THROW_CHECK1(runtime_error, new_vec, !new_vec.empty());
ipos = new_vec.begin()->m_begin;
bool last_flag_last = false;
for (auto e : new_vec) {
THROW_CHECK1(runtime_error, new_vec, e.m_begin == ipos);
THROW_CHECK1(runtime_error, e, e.size() > 0);
THROW_CHECK1(runtime_error, new_vec, !last_flag_last);
ipos += e.size();
last_flag_last = e.m_flags & FIEMAP_EXTENT_LAST;
}
THROW_CHECK1(runtime_error, new_vec, !last_extent_is_last || new_vec.rbegin()->m_end == ipos);
m_extents = new_vec;
m_current = m_extents.begin();
}
void
ExtentWalker::reset()
{
m_extents.clear();
m_current = m_extents.begin();
}
void
ExtentWalker::seek(off_t pos)
{
CHATTER_UNWIND("seek " << to_hex(pos));
THROW_CHECK1(out_of_range, pos, pos >= 0);
Itr rv = find_in_cache(pos);
if (rv != m_extents.end()) {
m_current = rv;
return;
}
run_fiemap(pos);
m_current = find_in_cache(pos);
}
Extent
ExtentWalker::current()
{
THROW_CHECK2(invalid_argument, *this, m_extents.size(), m_current != m_extents.end());
CHATTER_UNWIND("current " << *m_current);
return *m_current;
}
bool
ExtentWalker::next()
{
CHATTER_UNWIND("next");
THROW_CHECK1(invalid_argument, (m_current != m_extents.end()), m_current != m_extents.end());
if (current().m_end >= m_stat.st_size) {
CHATTER_UNWIND("next EOF");
return false;
}
auto next_pos = current().m_end;
if (next_pos >= m_stat.st_size) {
CHATTER_UNWIND("next next_pos = " << next_pos << " m_stat.st_size = " << m_stat.st_size);
return false;
}
seek(next_pos);
THROW_CHECK1(runtime_error, (m_current != m_extents.end()), m_current != m_extents.end());
// FIEMAP is full of lies, so this check keeps failing
// THROW_CHECK2(runtime_error, current().m_begin, next_pos, current().m_begin == next_pos);
// Just ensure that pos is in the next extent somewhere.
THROW_CHECK2(runtime_error, current(), next_pos, current().m_begin <= next_pos);
THROW_CHECK2(runtime_error, current(), next_pos, current().m_end > next_pos);
return true;
}
bool
ExtentWalker::prev()
{
CHATTER_UNWIND("prev");
THROW_CHECK1(invalid_argument, (m_current != m_extents.end()), m_current != m_extents.end());
auto prev_iter = m_current;
if (prev_iter->m_begin == 0) {
CHATTER_UNWIND("prev BOF");
return false;
}
THROW_CHECK1(invalid_argument, (prev_iter != m_extents.begin()), prev_iter != m_extents.begin());
--prev_iter;
CHATTER_UNWIND("prev seeking to " << *prev_iter << "->m_begin");
auto prev_end = current().m_begin;
seek(prev_iter->m_begin);
THROW_CHECK1(runtime_error, (m_current != m_extents.end()), m_current != m_extents.end());
THROW_CHECK2(runtime_error, current().m_end, prev_end, current().m_end == prev_end);
return true;
}
ExtentWalker::~ExtentWalker()
{
}
BtrfsExtentWalker::BtrfsExtentWalker(Fd fd) :
ExtentWalker(fd),
m_tree_id(0)
{
}
BtrfsExtentWalker::BtrfsExtentWalker(Fd fd, off_t initial_pos) :
ExtentWalker(fd),
m_tree_id(0)
{
seek(initial_pos);
}
void
BtrfsExtentWalker::set_root_fd(Fd root_fd)
{
m_root_fd = root_fd;
}
BtrfsExtentWalker::BtrfsExtentWalker(Fd fd, off_t initial_pos, Fd root_fd) :
ExtentWalker(fd),
m_tree_id(0)
{
set_root_fd(root_fd);
seek(initial_pos);
}
BtrfsExtentWalker::Vec
BtrfsExtentWalker::get_extent_map(off_t pos)
{
BtrfsIoctlSearchKey sk;
if (!m_root_fd) {
m_root_fd = m_fd;
}
if (!m_tree_id) {
m_tree_id = btrfs_get_root_id(m_fd);
}
sk.tree_id = m_tree_id;
sk.min_objectid = m_stat.st_ino;
sk.max_objectid = numeric_limits<uint64_t>::max();
sk.min_offset = ranged_cast<uint64_t>(pos);
sk.max_offset = numeric_limits<uint64_t>::max();
sk.min_transid = 0;
sk.max_transid = numeric_limits<uint64_t>::max();
sk.min_type = sk.max_type = BTRFS_EXTENT_DATA_KEY;
sk.nr_items = sc_extent_fetch_max;
CHATTER_UNWIND("sk " << sk << " root_fd " << name_fd(m_root_fd));
sk.do_ioctl(m_root_fd);
Vec rv;
bool past_eof = false;
for (auto i : sk.m_result) {
// If we're seeing extents from the next file then we're past EOF on this file
if (i.objectid > m_stat.st_ino) {
past_eof = true;
break;
}
// Ignore things that aren't EXTENT_DATA_KEY
if (i.type != BTRFS_EXTENT_DATA_KEY) {
continue;
}
// Hmmmkay we shouldn't be seeing these
if (i.objectid < m_stat.st_ino) {
THROW_ERROR(out_of_range, "objectid " << i.objectid << " < m_stat.st_ino " << m_stat.st_ino);
continue;
}
Extent e;
e.m_begin = i.offset;
auto compressed = call_btrfs_get(btrfs_stack_file_extent_compression, i.m_data);
// FIEMAP told us about compressed extents and we can too
if (compressed) {
e.m_flags |= FIEMAP_EXTENT_ENCODED;
}
auto type = call_btrfs_get(btrfs_stack_file_extent_type, i.m_data);
off_t len = -1;
switch (type) {
default:
cerr << "Unhandled file extent type " << type << " in root " << m_tree_id << " ino " << m_stat.st_ino << endl;
break;
case BTRFS_FILE_EXTENT_INLINE:
len = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data));
e.m_flags |= FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
// Inline extents are never obscured, so don't bother filling in m_physical_len, etc.
break;
case BTRFS_FILE_EXTENT_PREALLOC:
e.m_flags |= Extent::PREALLOC;
case BTRFS_FILE_EXTENT_REG: {
e.m_physical = call_btrfs_get(btrfs_stack_file_extent_disk_bytenr, i.m_data);
// This is the length of the full extent (decompressed)
off_t ram = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_ram_bytes, i.m_data));
// This is the length of the part of the extent appearing in the file (decompressed)
len = ranged_cast<off_t>(call_btrfs_get(btrfs_stack_file_extent_num_bytes, i.m_data));
// This is the offset from start of on-disk extent to the part we see in the file (decompressed)
// May be negative due to the kind of bug we're stuck with forever, so no cast range check
off_t offset = call_btrfs_get(btrfs_stack_file_extent_offset, i.m_data);
// If there is a physical address there must be size too
if (e.m_physical) {
THROW_CHECK1(runtime_error, ram, ram > 0);
THROW_CHECK1(runtime_error, len, len > 0);
THROW_CHECK2(runtime_error, offset, ram, offset < ram);
} else {
// There are two kinds of hole in btrfs. This is the other one.
e.m_flags |= Extent::HOLE;
}
// Partially obscured extent
// FIXME: sometimes this happens:
// i.type == BTRFS_EXTENT_DATA_KEY
// type = 0x1
// compressed = 0x0
// REG start 0x0 offset 0x0 num 0x20000 ram 0x21000 gen 1101121
// btrfs_file_extent_item {
// generation = 1101121
// ram_bytes = 135168
// compression = 0x0
// encryption = 0x0
// other_encoding = 0x0
// type = 0x1
// disk_bytenr = 0x0
// disk_num_bytes = 0x0
// offset = 0x0
// num_bytes = 0x20000
// }
if (ram != len || offset != 0) {
e.m_flags |= Extent::OBSCURED;
// cerr << e << "\nram = " << ram << ", len = " << len << ", offset = " << offset << endl;
}
e.m_physical_len = ram;
e.m_logical_len = len;
e.m_offset = offset;
// To maintain compatibility with FIEMAP we ignore the offset for compressed extents.
// At some point we'll grow out of this.
if (!compressed) {
e.m_physical += offset;
}
break;
}
}
if (len > 0) {
e.m_end = e.m_begin + len;
if (e.m_end >= m_stat.st_size) {
e.m_flags |= FIEMAP_EXTENT_LAST;
}
// FIXME: no FIEMAP_EXTENT_SHARED
// WONTFIX: non-trivial to replicate LOGIAL_INO
rv.push_back(e);
}
}
// Plug a hole at EOF
if (past_eof && !rv.empty()) {
rv.rbegin()->m_flags |= FIEMAP_EXTENT_LAST;
}
return rv;
}
ExtentWalker::Vec
ExtentWalker::get_extent_map(off_t pos)
{
Fiemap fm;
fm.fm_start = ranged_cast<uint64_t>(pos);
fm.fm_length = ranged_cast<uint64_t>(numeric_limits<off_t>::max() - pos);
fm.m_max_count = fm.m_min_count = sc_extent_fetch_max;
fm.do_ioctl(m_fd);
Vec rv;
for (auto i : fm.m_extents) {
Extent e;
e.m_begin = ranged_cast<off_t>(i.fe_logical);
e.m_end = ranged_cast<off_t>(i.fe_logical + i.fe_length);
e.m_physical = i.fe_physical;
e.m_flags = i.fe_flags;
rv.push_back(e);
}
return rv;
}
};

575
lib/fd.cc Normal file
View File

@@ -0,0 +1,575 @@
#include "crucible/chatter.h"
#include "crucible/error.h"
#include "crucible/fd.h"
#include "crucible/ntoa.h"
#include "crucible/string.h"
#include <cstdio>
#include <cstring>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <vector>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
namespace crucible {
using namespace std;
static const struct bits_ntoa_table o_flags_table[] = {
NTOA_TABLE_ENTRY_BITS(O_APPEND),
NTOA_TABLE_ENTRY_BITS(O_ASYNC),
NTOA_TABLE_ENTRY_BITS(O_CLOEXEC),
NTOA_TABLE_ENTRY_BITS(O_CREAT),
NTOA_TABLE_ENTRY_BITS(O_DIRECT),
NTOA_TABLE_ENTRY_BITS(O_DIRECTORY),
NTOA_TABLE_ENTRY_BITS(O_EXCL),
NTOA_TABLE_ENTRY_BITS(O_LARGEFILE),
NTOA_TABLE_ENTRY_BITS(O_NOATIME),
NTOA_TABLE_ENTRY_BITS(O_NOCTTY),
NTOA_TABLE_ENTRY_BITS(O_NOFOLLOW),
NTOA_TABLE_ENTRY_BITS(O_NONBLOCK),
NTOA_TABLE_ENTRY_BITS(O_NDELAY), // NONBLOCK will prevent this
NTOA_TABLE_ENTRY_BITS(O_SYNC),
NTOA_TABLE_ENTRY_BITS(O_TRUNC),
// These aren't really bit values
NTOA_TABLE_ENTRY_BITS(O_RDWR),
NTOA_TABLE_ENTRY_BITS(O_WRONLY),
NTOA_TABLE_ENTRY_BITS(O_RDONLY),
NTOA_TABLE_ENTRY_END(),
};
static const struct bits_ntoa_table o_mode_table[] = {
NTOA_TABLE_ENTRY_BITS(S_IFMT),
NTOA_TABLE_ENTRY_BITS(S_IFSOCK),
NTOA_TABLE_ENTRY_BITS(S_IFLNK),
NTOA_TABLE_ENTRY_BITS(S_IFREG),
NTOA_TABLE_ENTRY_BITS(S_IFBLK),
NTOA_TABLE_ENTRY_BITS(S_IFDIR),
NTOA_TABLE_ENTRY_BITS(S_IFCHR),
NTOA_TABLE_ENTRY_BITS(S_IFIFO),
NTOA_TABLE_ENTRY_BITS(S_ISUID),
NTOA_TABLE_ENTRY_BITS(S_ISGID),
NTOA_TABLE_ENTRY_BITS(S_ISVTX),
NTOA_TABLE_ENTRY_BITS(S_IRWXU),
NTOA_TABLE_ENTRY_BITS(S_IRUSR),
NTOA_TABLE_ENTRY_BITS(S_IWUSR),
NTOA_TABLE_ENTRY_BITS(S_IXUSR),
NTOA_TABLE_ENTRY_BITS(S_IRWXG),
NTOA_TABLE_ENTRY_BITS(S_IRGRP),
NTOA_TABLE_ENTRY_BITS(S_IWGRP),
NTOA_TABLE_ENTRY_BITS(S_IXGRP),
NTOA_TABLE_ENTRY_BITS(S_IRWXO),
NTOA_TABLE_ENTRY_BITS(S_IROTH),
NTOA_TABLE_ENTRY_BITS(S_IWOTH),
NTOA_TABLE_ENTRY_BITS(S_IXOTH),
NTOA_TABLE_ENTRY_END(),
};
string o_flags_ntoa(int flags)
{
return bits_ntoa(flags, o_flags_table);
}
string o_mode_ntoa(mode_t mode)
{
return bits_ntoa(mode, o_mode_table);
}
void
IOHandle::close()
{
CHATTER_TRACE("close fd " << m_fd << " in " << this);
if (m_fd >= 0) {
// Assume that ::close always destroys the FD, even if errors are encountered;
int closing_fd = m_fd;
m_fd = -1;
CHATTER_UNWIND("closing fd " << closing_fd << " in " << this);
DIE_IF_MINUS_ONE(::close(closing_fd));
}
}
IOHandle::~IOHandle()
{
CHATTER_TRACE("destroy fd " << m_fd << " in " << this);
if (m_fd >= 0) {
catch_all([&](){
close();
});
}
}
IOHandle::IOHandle() :
m_fd(-1)
{
CHATTER_TRACE("open fd " << m_fd << " in " << this);
}
IOHandle::IOHandle(int fd) :
m_fd(fd)
{
CHATTER_TRACE("open fd " << m_fd << " in " << this);
}
int
IOHandle::release_fd()
{
CHATTER_TRACE("release fd " << m_fd << " in " << this);
int rv = m_fd;
m_fd = -1;
return rv;
}
// XXX: necessary? useful?
template <>
struct ChatterTraits<Fd> {
Chatter &operator()(Chatter &c, const Fd &fd) const
{
c << "Fd {this=" << &fd << " fd=" << static_cast<int>(fd) << "}";
return c;
}
};
int
open_or_die(const string &file, int flags, mode_t mode)
{
int fd(::open(file.c_str(), flags, mode));
if (fd < 0) {
THROW_ERRNO("open: name '" << file << "' mode " << oct << setfill('0') << setw(3) << mode << " flags " << o_flags_ntoa(flags));
}
return fd;
}
int
openat_or_die(int dir_fd, const string &file, int flags, mode_t mode)
{
int fd(::openat(dir_fd, file.c_str(), flags, mode));
if (fd < 0) {
THROW_ERRNO("openat: dir_fd " << dir_fd << " " << name_fd(dir_fd) << " name '" << file << "' mode " << oct << setfill('0') << setw(3) << mode << " flags " << o_flags_ntoa(flags));
}
return fd;
}
static const struct bits_ntoa_table mmap_prot_table[] = {
NTOA_TABLE_ENTRY_BITS(PROT_EXEC),
NTOA_TABLE_ENTRY_BITS(PROT_READ),
NTOA_TABLE_ENTRY_BITS(PROT_WRITE),
NTOA_TABLE_ENTRY_BITS(PROT_NONE),
NTOA_TABLE_ENTRY_END(),
};
string mmap_prot_ntoa(int prot)
{
return bits_ntoa(prot, mmap_prot_table);
}
static const struct bits_ntoa_table mmap_flags_table[] = {
NTOA_TABLE_ENTRY_BITS(MAP_SHARED),
NTOA_TABLE_ENTRY_BITS(MAP_PRIVATE),
NTOA_TABLE_ENTRY_BITS(MAP_32BIT),
NTOA_TABLE_ENTRY_BITS(MAP_ANONYMOUS),
NTOA_TABLE_ENTRY_BITS(MAP_DENYWRITE),
NTOA_TABLE_ENTRY_BITS(MAP_EXECUTABLE),
#if MAP_FILE
NTOA_TABLE_ENTRY_BITS(MAP_FILE),
#endif
NTOA_TABLE_ENTRY_BITS(MAP_FIXED),
NTOA_TABLE_ENTRY_BITS(MAP_GROWSDOWN),
NTOA_TABLE_ENTRY_BITS(MAP_HUGETLB),
NTOA_TABLE_ENTRY_BITS(MAP_LOCKED),
NTOA_TABLE_ENTRY_BITS(MAP_NONBLOCK),
NTOA_TABLE_ENTRY_BITS(MAP_NORESERVE),
NTOA_TABLE_ENTRY_BITS(MAP_POPULATE),
NTOA_TABLE_ENTRY_BITS(MAP_STACK),
#ifdef MAP_UNINITIALIZED
NTOA_TABLE_ENTRY_BITS(MAP_UNINITIALIZED),
#endif
NTOA_TABLE_ENTRY_END(),
};
string mmap_flags_ntoa(int flags)
{
return bits_ntoa(flags, mmap_flags_table);
}
void *
mmap_or_die(void *addr, size_t length, int prot, int flags, int fd, off_t offset)
{
void *rv = mmap(addr, length, prot, flags, fd, offset);
if (rv == MAP_FAILED) {
THROW_ERRNO("mmap: addr " << addr << " length " << length
<< " prot " << mmap_prot_ntoa(prot)
<< " flags " << mmap_flags_ntoa(flags)
<< " fd " << fd << " offset " << offset);
}
return rv;
}
void
rename_or_die(const string &from, const string &to)
{
if (::rename(from.c_str(), to.c_str())) {
THROW_ERRNO("rename: " << from << " -> " << to);
}
}
void
renameat_or_die(int fromfd, const string &frompath, int tofd, const string &topath)
{
if (::renameat(fromfd, frompath.c_str(), tofd, topath.c_str())) {
THROW_ERRNO("renameat: " << name_fd(fromfd) << "/" << frompath
<< " -> " << name_fd(tofd) << "/" << topath);
}
}
string
socket_domain_ntoa(int domain)
{
static const bits_ntoa_table table[] = {
NTOA_TABLE_ENTRY_ENUM(AF_UNIX),
NTOA_TABLE_ENTRY_ENUM(AF_LOCAL), // probably the same as AF_UNIX
NTOA_TABLE_ENTRY_ENUM(AF_INET),
NTOA_TABLE_ENTRY_ENUM(AF_INET6),
NTOA_TABLE_ENTRY_ENUM(AF_PACKET),
NTOA_TABLE_ENTRY_END()
};
return bits_ntoa(domain, table);
}
string
socket_type_ntoa(int type)
{
static const bits_ntoa_table table[] = {
NTOA_TABLE_ENTRY_BITS(SOCK_CLOEXEC),
NTOA_TABLE_ENTRY_BITS(SOCK_NONBLOCK),
NTOA_TABLE_ENTRY_ENUM(SOCK_STREAM),
NTOA_TABLE_ENTRY_ENUM(SOCK_DGRAM),
NTOA_TABLE_ENTRY_ENUM(SOCK_RAW),
NTOA_TABLE_ENTRY_ENUM(SOCK_PACKET),
NTOA_TABLE_ENTRY_END()
};
return bits_ntoa(type, table);
}
string
socket_protocol_ntoa(int protocol)
{
static const bits_ntoa_table table[] = {
// an empty table just prints the number
NTOA_TABLE_ENTRY_END()
};
return bits_ntoa(protocol, table);
}
Fd
socket_or_die(int domain, int type, int protocol)
{
Fd fd(::socket(domain, type, protocol));
if (fd < 0) {
THROW_ERRNO("socket: domain " << socket_domain_ntoa(domain)
<< " type " << socket_type_ntoa(type)
<< " protocol " << socket_protocol_ntoa(protocol));
}
return fd;
}
void
write_or_die_partial(int fd, const void *buf, size_t size_wanted, size_t &size_written)
{
if (size_wanted > (static_cast<size_t>(~0) >> 1)) {
THROW_ERROR(invalid_argument, "cannot read " << size_wanted << ", more than signed size allows");
}
if (fd < 0) {
THROW_ERROR(invalid_argument, "write: trying to write on a closed file descriptor");
}
int rv = write(fd, buf, size_wanted);
if (rv < 0) {
THROW_ERRNO("write: " << size_wanted << " bytes returned " << rv);
}
size_written = rv;
}
void
write_or_die(int fd, const void *buf, size_t size)
{
size_t size_written = 0;
write_or_die_partial(fd, buf, size, size_written);
if (size_written != size) {
THROW_ERROR(runtime_error, "write: only " << size_written << " of " << size << " bytes written");
}
}
void
pwrite_or_die(int fd, const void *buf, size_t size, off_t offset)
{
if (size > (static_cast<size_t>(~0) >> 1)) {
THROW_ERROR(invalid_argument, "pwrite: cannot write " << size << ", more than signed size allows");
}
if (fd < 0) {
THROW_ERROR(invalid_argument, "pwrite: trying to write on a closed file descriptor");
}
int rv = ::pwrite(fd, buf, size, offset);
if (rv != static_cast<int>(size)) {
THROW_ERROR(runtime_error, "pwrite: only " << rv << " of " << size << " bytes written at offset " << offset);
}
}
template<>
void
write_or_die<string>(int fd, const string &text)
{
return write_or_die(fd, text.data(), text.size());
}
void
read_partial_or_die(int fd, void *buf, size_t size, size_t &size_read)
{
if (size > (static_cast<size_t>(~0) >> 1)) {
THROW_ERROR(invalid_argument, "cannot read " << size << ", more than signed size allows");
}
if (fd < 0) {
THROW_ERROR(runtime_error, "read: trying to read on a closed file descriptor");
}
size_read = 0;
while (size) {
int rv = read(fd, buf, size);
if (rv < 0) {
if (errno == EINTR) {
CHATTER_TRACE("resuming after EINTR");
continue;
}
THROW_ERRNO("read: " << size << " bytes");
}
if (rv > static_cast<int>(size)) {
THROW_ERROR(runtime_error, "read: somehow read more bytes (" << rv << ") than requested (" << size << ")");
}
if (rv == 0) break;
size_read += rv;
size -= rv;
// CHATTER("read " << rv << " bytes from fd " << fd);
}
}
string
read_string(int fd, size_t size)
{
string rv(size, '\0');
size_t size_read = 0;
void *rvp = const_cast<char *>(rv.data());
read_partial_or_die(fd, rvp, size, size_read);
rv.resize(size_read);
return rv;
}
void
read_or_die(int fd, void *buf, size_t size)
{
size_t size_read = 0;
read_partial_or_die(fd, buf, size, size_read);
if (size_read != size) {
THROW_ERROR(runtime_error, "read: " << size_read << " of " << size << " bytes");
}
}
void
pread_or_die(int fd, void *buf, size_t size, off_t offset)
{
if (size > (static_cast<size_t>(~0) >> 1)) {
THROW_ERROR(invalid_argument, "cannot read " << size << ", more than signed size allows");
}
if (fd < 0) {
throw runtime_error("read: trying to read on a closed file descriptor");
} else {
while (size) {
int rv = pread(fd, buf, size, offset);
if (rv < 0) {
if (errno == EINTR) {
CHATTER(__func__ << "resuming after EINTR");
continue;
}
THROW_ERRNO("pread: " << size << " bytes");
}
if (rv != static_cast<int>(size)) {
THROW_ERROR(runtime_error, "pread: " << size << " bytes at offset " << offset << " returned " << rv);
}
break;
}
}
}
template<>
void
pread_or_die<string>(int fd, string &text, off_t offset)
{
return pread_or_die(fd, const_cast<char *>(text.data()), text.size(), offset);
}
template<>
void
pread_or_die<vector<char>>(int fd, vector<char> &text, off_t offset)
{
return pread_or_die(fd, text.data(), text.size(), offset);
}
template<>
void
pread_or_die<vector<uint8_t>>(int fd, vector<uint8_t> &text, off_t offset)
{
return pread_or_die(fd, text.data(), text.size(), offset);
}
Stat::Stat()
{
memset_zero<stat>(this);
}
Stat &
Stat::lstat(const string &filename)
{
CHATTER_UNWIND("lstat " << filename);
DIE_IF_MINUS_ONE(::lstat(filename.c_str(), this));
return *this;
}
Stat &
Stat::fstat(int fd)
{
CHATTER_UNWIND("fstat " << fd);
DIE_IF_MINUS_ONE(::fstat(fd, this));
return *this;
}
Stat::Stat(int fd)
{
memset_zero<stat>(this);
fstat(fd);
}
Stat::Stat(const string &filename)
{
memset_zero<stat>(this);
lstat(filename);
}
string
readlink_or_die(const string &path)
{
// Start with a reasonable guess since it will usually work
off_t size = 4096;
while (size < 1048576) {
char buf[size + 1];
int rv;
DIE_IF_MINUS_ONE(rv = readlink(path.c_str(), buf, size + 1));
// No negative values allowed except -1
THROW_CHECK1(runtime_error, rv, rv >= 0);
if (rv <= size) {
buf[rv] = 0;
return buf;
}
// cerr << "Retrying readlink(" << path << ", buf, " << size + 1 << ")" << endl;
// This is from the Linux readlink(2) man page (release 3.44).
// It only works when the filesystem reports st_size accurately for symlinks,
// and at least one doesn't, so we can't rely on it at all.
// size = lstat_or_die(path).st_size;
size *= 2;
}
THROW_ERROR(runtime_error, "readlink: maximum buffer size exceeded");
}
// Turn a FD into a human-recognizable filename OR an error message.
string
name_fd(int fd)
{
try {
ostringstream oss;
oss << "/proc/self/fd/" << fd;
return readlink_or_die(oss.str());
} catch (exception &e) {
return string(e.what());
}
}
bool
assert_no_leaked_fds()
{
struct rlimit rlim;
int rv = getrlimit(RLIMIT_NOFILE, &rlim);
if (rv) {
perror("getrlimit(RLIMIT_NOFILE)");
// Well, that sucked. Guess.
rlim.rlim_cur = 1024;
}
CHATTER("Checking for leaked FDs in range 3.." << rlim.rlim_cur);
int leaked_fds = 0;
for (unsigned i = 3; i < rlim.rlim_cur; ++i) {
struct stat buf;
if (! fstat(i, &buf)) {
CHATTER("WARNING: fd " << i << " open at exit");
++leaked_fds;
}
}
CHATTER(leaked_fds << " leaked FD(s) found");
return leaked_fds == 0;
}
pair<Fd, Fd>
socketpair_or_die(int domain, int type, int protocol)
{
pair<Fd, Fd> rv;
int sv[2];
DIE_IF_MINUS_ONE(socketpair(domain, type, protocol, sv));
rv.first = sv[0];
rv.second = sv[1];
return rv;
}
void
dup2_or_die(int fd_in, int fd_out)
{
DIE_IF_MINUS_ONE(dup2(fd_in, fd_out));
}
string
st_mode_ntoa(mode_t mode)
{
static const bits_ntoa_table table[] = {
NTOA_TABLE_ENTRY_BITS(S_IFMT),
NTOA_TABLE_ENTRY_BITS(S_IFSOCK),
NTOA_TABLE_ENTRY_BITS(S_IFLNK),
NTOA_TABLE_ENTRY_BITS(S_IFMT),
NTOA_TABLE_ENTRY_BITS(S_IFSOCK),
NTOA_TABLE_ENTRY_BITS(S_IFLNK),
NTOA_TABLE_ENTRY_BITS(S_IFREG),
NTOA_TABLE_ENTRY_BITS(S_IFBLK),
NTOA_TABLE_ENTRY_BITS(S_IFDIR),
NTOA_TABLE_ENTRY_BITS(S_IFCHR),
NTOA_TABLE_ENTRY_BITS(S_IFIFO),
NTOA_TABLE_ENTRY_BITS(S_ISUID),
NTOA_TABLE_ENTRY_BITS(S_ISGID),
NTOA_TABLE_ENTRY_BITS(S_ISVTX),
NTOA_TABLE_ENTRY_BITS(S_IRWXU),
NTOA_TABLE_ENTRY_BITS(S_IRUSR),
NTOA_TABLE_ENTRY_BITS(S_IWUSR),
NTOA_TABLE_ENTRY_BITS(S_IXUSR),
NTOA_TABLE_ENTRY_BITS(S_IRWXG),
NTOA_TABLE_ENTRY_BITS(S_IRGRP),
NTOA_TABLE_ENTRY_BITS(S_IWGRP),
NTOA_TABLE_ENTRY_BITS(S_IXGRP),
NTOA_TABLE_ENTRY_BITS(S_IRWXO),
NTOA_TABLE_ENTRY_BITS(S_IROTH),
NTOA_TABLE_ENTRY_BITS(S_IWOTH),
NTOA_TABLE_ENTRY_BITS(S_IXOTH),
NTOA_TABLE_ENTRY_END()
};
return bits_ntoa(mode, table);
}
};

1050
lib/fs.cc Normal file
View File

File diff suppressed because it is too large Load Diff

96
lib/interp.cc Normal file
View File

@@ -0,0 +1,96 @@
#include "crucible/interp.h"
#include "crucible/chatter.h"
namespace crucible {
using namespace std;
int
Proc::exec(const ArgList &args)
{
return m_cmd(args);
}
Proc::Proc(const function<int(const ArgList &)> &f) :
m_cmd(f)
{
}
Command::~Command()
{
}
ArgList::ArgList(const char **argv)
{
while (argv && *argv) {
push_back(*argv++);
}
}
ArgList::ArgList(const vector<string> &&that) :
vector<string>(that)
{
}
Interp::~Interp()
{
}
Interp::Interp(const map<string, shared_ptr<Command> > &cmdlist) :
m_commands(cmdlist)
{
}
void
Interp::add_command(const string &name, const shared_ptr<Command> &command)
{
m_commands[name] = command;
}
int
Interp::exec(const ArgList &args)
{
auto next_arg = args.begin();
++next_arg;
return m_commands.at(args[0])->exec(vector<string>(next_arg, args.end()));
}
ArgParser::~ArgParser()
{
}
ArgParser::ArgParser()
{
}
void
ArgParser::add_opt(string opt, ArgActor actor)
{
m_string_opts[opt] = actor;
}
void
ArgParser::parse_backend(void *t, const ArgList &args)
{
bool quote_args = false;
for (string arg : args) {
if (quote_args) {
cerr << "arg: '" << arg << "'" << endl;
continue;
}
if (arg == "--") {
quote_args = true;
continue;
}
if (arg.compare(0, 2, "--") == 0) {
auto found = m_string_opts.find(arg.substr(2, string::npos));
if (found != m_string_opts.end()) {
found->second.predicate(t, "foo");
}
(void)t;
}
}
}
};

40
lib/ntoa.cc Normal file
View File

@@ -0,0 +1,40 @@
#include "crucible/ntoa.h"
#include <cassert>
#include <sstream>
#include <string>
namespace crucible {
using namespace std;
string bits_ntoa(unsigned long n, const bits_ntoa_table *table)
{
string out;
while (n && table->a) {
// No bits in n outside of mask
assert( ((~table->mask) & table->n) == 0);
if ( (n & table->mask) == table->n) {
if (!out.empty()) {
out += "|";
}
out += table->a;
n &= ~(table->mask);
}
++table;
}
if (n) {
ostringstream oss;
oss << "0x" << hex << n;
if (!out.empty()) {
out += "|";
}
out += oss.str();
}
if (out.empty()) {
out = "0";
}
return out;
}
};

26
lib/path.cc Normal file
View File

@@ -0,0 +1,26 @@
#include "crucible/path.h"
#include "crucible/error.h"
namespace crucible {
using namespace std;
string
basename(string s)
{
size_t left = s.find_last_of("/");
size_t right = s.find_last_not_of("/");
if (left == string::npos) {
return s;
}
return s.substr(left + 1, right);
}
string
join(string dir, string base)
{
// TODO: a lot of sanity checking, maybe canonicalization
return dir + "/" + base;
}
};

121
lib/process.cc Normal file
View File

@@ -0,0 +1,121 @@
#include "crucible/process.h"
#include "crucible/chatter.h"
#include "crucible/error.h"
#include <utility>
// for gettid()
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <unistd.h>
#include <sys/syscall.h>
namespace crucible {
using namespace std;
bool
Process::joinable()
{
return !!m_pid;
}
Process::~Process()
{
if (joinable()) {
// because it's just not the same without the word "zombie"...
CHATTER("ZOMBIE WARNING: joinable Process pid " << m_pid << " abandoned");
}
}
Process::Process() :
m_pid(0)
{
}
Process::Process(Process &&move_from) :
m_pid(0)
{
swap(m_pid, move_from.m_pid);
}
void
Process::do_fork(function<int()> child_func)
{
int rv = fork();
if (rv < 0) {
THROW_ERRNO("fork failed");
}
m_pid = rv;
if (rv == 0) {
// child
catch_all([&]() {
int rv = child_func();
exit(rv);
});
terminate();
}
}
Process::status_type
Process::join()
{
if (m_pid == 0) {
THROW_ERROR(invalid_argument, "Process not created");
}
int status = 0;
pid_t rv = waitpid(m_pid, &status, 0);
if (rv == -1) {
THROW_ERRNO("waitpid failed, pid = " << m_pid);
}
if (rv != m_pid) {
THROW_ERROR(runtime_error, "waitpid failed, wanted pid = " << m_pid << ", got rv = " << rv << ", status = " << status);
}
m_pid = 0;
return status;
}
void
Process::detach()
{
m_pid = 0;
}
Process::native_handle_type
Process::native_handle()
{
return m_pid;
}
Process::id
Process::get_id()
{
return m_pid;
}
void
Process::kill(int sig)
{
if (!m_pid) {
THROW_ERROR(invalid_argument, "Process not created");
}
int rv = ::kill(m_pid, sig);
if (rv) {
THROW_ERRNO("killing process " << m_pid << " with signal " << sig);
}
}
template<>
struct ResourceHandle<Process::id, Process>;
pid_t
gettid()
{
return syscall(SYS_gettid);
}
}

43
lib/string.cc Normal file
View File

@@ -0,0 +1,43 @@
#include "crucible/string.h"
#include "crucible/error.h"
#include <inttypes.h>
namespace crucible {
using namespace std;
string
to_hex(uint64_t i)
{
return astringprintf("0x%" PRIx64, i);
}
uint64_t
from_hex(const string &s)
{
return stoull(s, 0, 0);
}
vector<string>
split(string delim, string s)
{
if (delim.empty()) {
THROW_ERROR(invalid_argument, "delimiter empty when splitting '" << s << "'");
}
vector<string> rv;
size_t n = 0;
while (n < s.length()) {
size_t f = s.find(delim, n);
if (f == string::npos) {
rv.push_back(s.substr(n));
break;
}
if (f > n) {
rv.push_back(s.substr(n, f - n));
}
n = f + delim.length();
}
return rv;
}
};

158
lib/time.cc Normal file
View File

@@ -0,0 +1,158 @@
#include "crucible/time.h"
#include "crucible/error.h"
#include <algorithm>
#include <cmath>
#include <ctime>
#include <thread>
namespace crucible {
double
nanosleep(double secs)
{
if (secs <= 0) return secs;
struct timespec req;
req.tv_sec = time_t(floor(secs));
req.tv_nsec = long((secs - floor(secs)) * 1000000000);
// Just silently ignore weirdo values for now
if (req.tv_sec < 0) return secs;
if (req.tv_sec > 1000000000) return secs;
if (req.tv_nsec < 0) return secs;
if (req.tv_nsec > 1000000000) return secs;
struct timespec rem;
rem.tv_sec = 0;
rem.tv_nsec = 0;
int nanosleep_rv = ::nanosleep(&req, &rem);
if (nanosleep_rv) {
THROW_ERRNO("nanosleep (" << secs << ") { tv_sec = " << req.tv_sec << ", tv_nsec = " << req.tv_nsec << " }");
}
return rem.tv_sec + (double(rem.tv_nsec) / 1000000000.0);
}
Timer::Timer() :
m_start(chrono::high_resolution_clock::now())
{
}
double
Timer::age() const
{
chrono::high_resolution_clock::time_point end = chrono::high_resolution_clock::now();
return chrono::duration<double>(end - m_start).count();
}
double
Timer::report(int precision) const
{
return ceil(age() * precision) / precision;
}
void
Timer::reset()
{
m_start = chrono::high_resolution_clock::now();
}
void
Timer::set(const chrono::high_resolution_clock::time_point &start)
{
m_start = start;
}
void
Timer::set(double delta)
{
m_start += chrono::duration_cast<chrono::high_resolution_clock::duration>(chrono::duration<double>(delta));
}
double
Timer::lap()
{
auto end = chrono::high_resolution_clock::now();
double rv = chrono::duration<double>(end - m_start).count();
m_start = end;
return rv;
}
ostream &
operator<<(ostream &os, const Timer &t)
{
return os << t.report();
}
bool
Timer::operator<(double d) const
{
return age() < d;
}
bool
Timer::operator>(double d) const
{
return age() > d;
}
RateLimiter::RateLimiter(double rate, double burst) :
m_rate(rate),
m_burst(burst)
{
}
RateLimiter::RateLimiter(double rate) :
m_rate(rate),
m_burst(rate)
{
}
void
RateLimiter::update_tokens()
{
double delta = m_timer.lap();
m_tokens += delta * m_rate;
if (m_tokens > m_burst) {
m_tokens = m_burst;
}
}
void
RateLimiter::sleep_for(double cost)
{
borrow(cost);
while (1) {
unique_lock<mutex> lock(m_mutex);
update_tokens();
if (m_tokens >= 0) {
return;
}
double sleep_time(-m_tokens / m_rate);
lock.unlock();
if (sleep_time > 0.0) {
nanosleep(sleep_time);
} else {
return;
}
}
}
bool
RateLimiter::is_ready()
{
unique_lock<mutex> lock(m_mutex);
update_tokens();
return m_tokens >= 0;
}
void
RateLimiter::borrow(double cost)
{
unique_lock<mutex> lock(m_mutex);
m_tokens -= cost;
}
}

16
lib/uuid.cc Normal file
View File

@@ -0,0 +1,16 @@
#include "crucible/uuid.h"
namespace crucible {
using namespace std;
const size_t uuid_unparsed_size = 37; // "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx\0"
string
uuid_unparse(const unsigned char in[16])
{
char out[uuid_unparsed_size];
::uuid_unparse(in, out);
return string(out);
}
}