#include "bin.h" #include "chunk-table.h" #include "commands.h" #include "fs-tree.h" #include "image.h" #include "misc.h" #include "status.h" #include "string-table.h" #include "volume.h" #include #include #include #include #include #include #include #define BUFFER_SIZE 65536 enum { OPT_OUTPATH, OPT_OUTPATH_PATH, OPT_IDENT, OPT_IDENT_VAL, OPT_DIRECTORY, OPT_DIRECTORY_PATH, OPT_TAGGED_DIRECTORY, OPT_TAGGED_DIRECTORY_TAG, OPT_TAGGED_DIRECTORY_PATH, OPT_VERBOSE, }; struct capture_ctx { struct fs_tree ctx_tree; struct ec3_image_ioctx *ctx_image; struct chunk_table ctx_chunks; struct string_table ctx_strings; struct ec3_volume *ctx_volume; }; static enum ec3_status capture_file( struct capture_ctx *ctx, struct ec3_volume *vol, b_directory *dir, const char *filename, const b_path *filepath, ec3_chunk_id out_chunk) { const struct ec3_image_info *image_info = ec3_image_ioctx_get_info(ctx->ctx_image); size_t key = string_table_get(&ctx->ctx_strings, filename); size_t buf_len = image_info->img_cluster_size; char *buf = malloc(buf_len); if (!buf) { return EC3_ERR_NO_MEMORY; } b_file *src = NULL; b_status status = b_file_open( dir, filepath, B_FILE_READ_ONLY | B_FILE_BINARY, &src); if (!B_OK(status)) { free(buf); return ec3_status_from_b_status(status, EC3_ERR_NO_ENTRY); } enum ec3_status s2 = EC3_SUCCESS; chunk_table_begin_chunk(&ctx->ctx_chunks); size_t chunk_size = 0; while (1) { size_t nr_read = 0; status = b_file_read( src, B_OFFSET_CURRENT, buf_len, buf, &nr_read); if (!B_OK(status)) { s2 = ec3_status_from_b_status( status, EC3_ERR_IO_FAILURE); break; } enum ec3_status s2 = chunk_table_put(&ctx->ctx_chunks, buf, nr_read); if (s2 != EC3_SUCCESS) { break; } chunk_size += nr_read; if (nr_read < buf_len) { break; } } s2 = chunk_table_end_chunk(&ctx->ctx_chunks, out_chunk); if (s2 != EC3_SUCCESS) { return s2; } char id_str[128]; ec3_chunk_id_to_string(out_chunk, id_str, sizeof id_str); // printf("wrote %zu byte chunk %s\n", chunk_size, id_str); free(buf); b_file_release(src); return s2; } static int print_fs_tree_node( struct fs_tree *tree, struct fs_tree_node *node, unsigned int depth, int direction, void *arg) { if (direction == ITERATE_POSTORDER) { depth = 0; } for (unsigned int i = 0; i < depth; i++) { fputs(" ", stdout); } printf("%s ", node->n_name); char id_str[128]; switch (node->n_type) { case FS_TREE_FILE: ec3_chunk_id_to_string(node->n_chunk, id_str, sizeof id_str); printf("[%s]", id_str); break; case FS_TREE_DIR: printf("[d]"); break; default: break; } printf("\n"); return 0; } struct capture_directory_structure_args { b_list *args_stack; struct capture_ctx *args_ctx; }; static int capture_directory_structure_callback( struct fs_tree *tree, struct fs_tree_node *node, unsigned int depth, int direction, void *arg) { if (direction != ITERATE_POSTORDER) { return 0; } struct capture_directory_structure_args *args = arg; b_list *stack = args->args_stack; struct capture_ctx *ctx = args->args_ctx; if (node->n_type == FS_TREE_FILE) { b_list_push_back(stack, node); return 0; } size_t nr_children = fs_tree_node_get_nr_children(node); struct ec3_directory_entry dent; enum ec3_status status = EC3_SUCCESS; chunk_table_begin_chunk(&ctx->ctx_chunks); for (size_t i = 0; i < nr_children; i++) { struct fs_tree_node *child = b_list_pop_back(stack); if (!child) { return EC3_ERR_INTERNAL_FAILURE; } dent.d_name = b_i32_htob( string_table_get(&ctx->ctx_strings, child->n_name)); dent.d_vnode = b_i32_htob(child->n_id); status = chunk_table_put(&ctx->ctx_chunks, &dent, sizeof dent); } status = chunk_table_end_chunk(&ctx->ctx_chunks, node->n_chunk); if (status != EC3_SUCCESS) { return status; } char id_str[128]; ec3_chunk_id_to_string(node->n_chunk, id_str, sizeof id_str); printf("[d %s / %zu] wrote %zu byte chunk %s\n", node->n_name, node->n_id, nr_children * sizeof dent, id_str); struct ec3_vnode vnode = { .v_id = node->n_id, .v_mode = EC3_V_DIR, }; memcpy(vnode.v_data, node->n_chunk, sizeof vnode.v_data); ec3_volume_put_vnode(ctx->ctx_volume, &vnode); b_list_push_back(stack, node); return 0; } static enum ec3_status capture_directory_structure(struct capture_ctx *ctx) { b_list *stack = b_list_create(); struct capture_directory_structure_args args = { .args_stack = stack, .args_ctx = ctx, }; fs_tree_iterate( &ctx->ctx_tree, capture_directory_structure_callback, &args); return EC3_SUCCESS; } static enum ec3_status capture_directory( struct capture_ctx *ctx, uint64_t id, const char *cpath) { b_path *path = b_path_create_from_cstr(cpath); b_directory *dir; b_status status = b_directory_open(NULL, path, &dir); b_path_release(path); if (!B_OK(status)) { return EC3_ERR_NO_ENTRY; } struct ec3_tag_ioctx *volu; enum ec3_status s2 = ec3_image_ioctx_create_tag( ctx->ctx_image, EC3_TAG_VOLU, id, EC3_TAG_IO_READ | EC3_TAG_IO_WRITE, &volu); if (s2 != EC3_SUCCESS) { b_directory_release(dir); return s2; } struct ec3_volume *volume; s2 = ec3_volume_create(ctx->ctx_image, volu, &volume); if (s2 != EC3_SUCCESS) { ec3_tag_ioctx_close(volu); b_directory_release(dir); return s2; } ctx->ctx_volume = volume; fs_tree_init(&ctx->ctx_tree); struct ec3_vnode vnode = {0}; vnode.v_id = ctx->ctx_tree.fs_root->n_id; ec3_volume_put_vnode(volume, &vnode); enum ec3_status status2 = EC3_SUCCESS; b_directory_iterator it; b_directory_iterator_begin(dir, &it, B_DIRECTORY_ITERATE_PARENT_LAST); while (b_directory_iterator_is_valid(&it)) { if (!b_directory_path_is_file(dir, it.filepath)) { printf("dir: %s\n", b_path_ptr(it.filepath)); b_directory_iterator_next(&it); continue; } ec3_vnode_from_file_info(&it.info, &vnode); printf("file: %s\n", b_path_ptr(it.filepath)); status2 = capture_file( ctx, volume, dir, it.filename, it.filepath, vnode.v_data); if (status2 != EC3_SUCCESS) { b_err("failed to capture file '%s'", b_path_ptr(it.filepath)); b_i("error code: %s", ec3_status_to_string(status2)); break; } fs_tree_put( &ctx->ctx_tree, b_path_ptr(it.filepath), &vnode.v_id, vnode.v_data); ec3_volume_put_vnode(volume, &vnode); b_directory_iterator_next(&it); } // fs_tree_iterate(&ctx->ctx_tree, print_fs_tree_node, NULL); capture_directory_structure(ctx); fs_tree_fini(&ctx->ctx_tree); ec3_tag_ioctx_close(volu); b_directory_release(dir); return status2; } static int capture( const b_command *self, const b_arglist *opt, const b_array *args) { const char *out_path = NULL; b_arglist_get_string(opt, OPT_OUTPATH, OPT_OUTPATH_PATH, 0, &out_path); enum ec3_status status = EC3_SUCCESS; uint64_t ident = 0; const char *ident_str; b_arglist_get_string(opt, OPT_IDENT, OPT_IDENT_VAL, 0, &ident_str); if (ident_str) { status = ec3_identifier_from_string(ident_str, &ident); } if (status != EC3_SUCCESS) { b_err("'%s' is not a valid container identifier", ident_str); return -1; } struct ec3_image_ioctx *image = NULL; struct ec3_parameters param = { .p_cluster_size = EC3_CLUSTER_16K, .p_compression_func = EC3_COMPRESSION_ZSTD, .p_ident = ident, }; status = ec3_image_ioctx_open( out_path, ¶m, EC3_IMAGE_IO_WRITE | EC3_IMAGE_IO_TRUNCATE, &image); if (status != EC3_SUCCESS) { b_err("cannot initialise EC3 writer"); return -1; } uint64_t ctab_id, cdat_id, stab_id; ec3_identifier_from_string("_CHKTAB0", &ctab_id); ec3_identifier_from_string("_CHKDAT0", &cdat_id); ec3_identifier_from_string("_VOLSTR0", &stab_id); struct ec3_tag_ioctx *ctab, *cdat, *stab; status = ec3_image_ioctx_create_tag( image, EC3_TAG_CTAB, ctab_id, EC3_TAG_IO_READ | EC3_TAG_IO_WRITE, &ctab); if (status != EC3_SUCCESS) { b_err("cannot create chunk table tag"); b_i("error code: %s", ec3_status_to_string(status)); return -1; } status = ec3_image_ioctx_create_tag( image, EC3_TAG_CDAT, cdat_id, EC3_TAG_IO_READ | EC3_TAG_IO_WRITE, &cdat); if (status != EC3_SUCCESS) { b_err("cannot create chunk data tag"); b_i("error code: %s", ec3_status_to_string(status)); return -1; } status = ec3_image_ioctx_create_tag( image, EC3_TAG_STAB, stab_id, EC3_TAG_IO_WRITE | EC3_TAG_IO_SEQUENTIAL, &stab); if (status != EC3_SUCCESS) { b_err("cannot create string table tag"); b_i("error code: %s", ec3_status_to_string(status)); return -1; } struct capture_ctx ctx = {0}; ctx.ctx_image = image; chunk_table_init(ctab, cdat, param.p_cluster_size, &ctx.ctx_chunks); chunk_table_init_empty_table(&ctx.ctx_chunks); string_table_init(&ctx.ctx_strings); uint64_t next_auto_id = 0; b_arglist_iterator it = {0}; b_arglist_foreach_filtered(&it, opt, OPT_DIRECTORY, OPT_DIRECTORY_PATH) { printf("%s\n", it.value->val_str); status = capture_directory( &ctx, next_auto_id, it.value->val_str); next_auto_id++; if (status != EC3_SUCCESS) { b_err("an error occurred while writing to the " "container"); b_i("error: %s", ec3_status_to_string(status)); return -1; } } for (size_t i = 0;; i++) { b_arglist_option *option = NULL; b_status err = b_arglist_get_option( opt, OPT_TAGGED_DIRECTORY, i, &option); if (!option) { break; } b_arglist_value *tag = NULL, *path = NULL; err = b_arglist_option_get_value( option, OPT_TAGGED_DIRECTORY_TAG, 0, &tag); err = b_arglist_option_get_value( option, OPT_TAGGED_DIRECTORY_PATH, 0, &path); printf("%s:%s\n", tag->val_str, path->val_str); uint64_t id = 0; status = ec3_identifier_from_string(tag->val_str, &id); if (status != EC3_SUCCESS) { b_err("'%s' is not a valid tag identifier", id); return -1; } status = capture_directory(&ctx, id, path->val_str); if (status != EC3_SUCCESS) { b_err("an error occurred while writing to the " "container"); return -1; } } b_btree_iterator s_it; b_btree_iterator_begin(&ctx.ctx_strings.s_offset_tree, &s_it); while (b_btree_iterator_is_valid(&s_it)) { struct string_table_entry *entry = b_unbox( struct string_table_entry, s_it.node, e_offset_node); size_t len = strlen(entry->e_str) + 1; size_t nr_written = 0; status = ec3_tag_ioctx_write( stab, entry->e_str, len, &nr_written); b_btree_iterator_next(&s_it); } string_table_finish(&ctx.ctx_strings); chunk_table_finish(&ctx.ctx_chunks); ec3_tag_ioctx_close(stab); ec3_tag_ioctx_close(ctab); ec3_tag_ioctx_close(cdat); ec3_image_ioctx_close(image); return 0; } B_COMMAND(CMD_CAPTURE, CMD_ROOT) { B_COMMAND_NAME("capture"); B_COMMAND_SHORT_NAME('Z'); B_COMMAND_DESC( "capture one or more directories into an ec3 container. each " "directory specified will be stored in a separate volume " "within " "the created container."); B_COMMAND_FLAGS(B_COMMAND_SHOW_HELP_BY_DEFAULT); B_COMMAND_FUNCTION(capture); B_COMMAND_HELP_OPTION(); B_COMMAND_OPTION(OPT_IDENT) { B_OPTION_SHORT_NAME('I'); B_OPTION_LONG_NAME("ident"); B_OPTION_DESC( "the string or number to use as the container " "identifier"); B_OPTION_ARG(OPT_IDENT_VAL) { B_ARG_NAME("value"); B_ARG_NR_VALUES(1); } } B_COMMAND_OPTION(OPT_OUTPATH) { B_OPTION_SHORT_NAME('o'); B_OPTION_LONG_NAME("out"); B_OPTION_DESC("the path to save the new file to."); B_OPTION_ARG(OPT_OUTPATH_PATH) { B_ARG_NAME("path"); B_ARG_NR_VALUES(1); } } B_COMMAND_OPTION(OPT_DIRECTORY) { B_OPTION_SHORT_NAME('d'); B_OPTION_LONG_NAME("directory"); B_OPTION_DESC( "a directory to add to the container. a volume " "will be created " "within the container to store the specified " "directory."); B_OPTION_ARG(OPT_DIRECTORY_PATH) { B_ARG_NAME("path"); B_ARG_NR_VALUES(1); } } B_COMMAND_OPTION(OPT_TAGGED_DIRECTORY) { B_OPTION_SHORT_NAME('D'); B_OPTION_LONG_NAME("tagged-directory"); B_OPTION_DESC( "a file to add to the container, with an associated " "tag. a disk " "image will be created within the container to store " "the specified " "directory. the tag must be either: (a) a 64-bit " "hexadecimal " "number; or (b) a string of no more than 8 " "characters."); B_OPTION_ARG(OPT_TAGGED_DIRECTORY_TAG) { B_ARG_NAME("tag"); B_ARG_NR_VALUES(1); } B_OPTION_ARG(OPT_TAGGED_DIRECTORY_PATH) { B_ARG_NAME("path"); B_ARG_NR_VALUES(1); } } B_COMMAND_OPTION(OPT_VERBOSE) { B_OPTION_SHORT_NAME('v'); B_OPTION_LONG_NAME("verbose"); B_OPTION_DESC( "show detailed output logs. this option can be " "specified multiple " "times to increase the level of output."); } B_COMMAND_USAGE() { B_COMMAND_USAGE_OPT(OPT_OUTPATH); B_COMMAND_USAGE_OPT(OPT_DIRECTORY); } B_COMMAND_USAGE() { B_COMMAND_USAGE_OPT(OPT_OUTPATH); B_COMMAND_USAGE_OPT(OPT_TAGGED_DIRECTORY); } }