From de99b880974ea59f8af53ee618dcb2d793a37c44 Mon Sep 17 00:00:00 2001 From: Max Wash Date: Sat, 15 Feb 2025 12:36:37 +0000 Subject: [PATCH] move cluster table management from ec3_reader/writer to ec3_pipeline --- src/aes256.c | 7 +- src/b-tree.c | 4 + src/bin.h | 6 +- src/cluster.c | 12 ++- src/cluster.h | 6 +- src/commands.h | 1 + src/file.c | 39 --------- src/pipeline.c | 129 ++++++++++++++++++++++++++--- src/pipeline.h | 47 ++++++++--- src/query.c | 34 ++++++++ src/read.c | 218 +++++++++++++++++++++++++++++++++++++++++++++++-- src/read.h | 14 ++++ src/status.h | 1 + src/wrap.c | 2 +- src/write.c | 81 +++++++++--------- src/write.h | 7 +- src/zstd.c | 10 ++- 17 files changed, 491 insertions(+), 127 deletions(-) delete mode 100644 src/file.c diff --git a/src/aes256.c b/src/aes256.c index 9634d51..679f7ad 100644 --- a/src/aes256.c +++ b/src/aes256.c @@ -5,6 +5,7 @@ static enum ec3_status encrypt( const void *src, size_t len, void *dest, + size_t dest_max, size_t *nr_written) { return EC3_SUCCESS; @@ -13,7 +14,9 @@ static enum ec3_status encrypt( static enum ec3_status decrypt( struct ec3_pipeline_stage *stage, const void *src, + size_t len, void *dest, + size_t dest_max, size_t *nr_read) { return EC3_SUCCESS; @@ -22,6 +25,6 @@ static enum ec3_status decrypt( const struct ec3_pipeline_stage_type pipeline_aes256 = { .t_id = EC3_PIPELINE_AES256, .t_flags = EC3_PIPELINE_F_NONE, - .t_data_in = decrypt, - .t_data_out = encrypt, + .t_cluster_in = decrypt, + .t_cluster_out = encrypt, }; diff --git a/src/b-tree.c b/src/b-tree.c index 9c51a41..ad2b3e7 100644 --- a/src/b-tree.c +++ b/src/b-tree.c @@ -537,6 +537,10 @@ int b_tree_get(struct b_tree *tree, b_tree_node_entry *to_get) } } + if (next_id == B_TREE_INVALID_PTR) { + break; + } + err = node_read(tree, next_id, next); if (err != 0) { return err; diff --git a/src/bin.h b/src/bin.h index 3654547..90ccaaa 100644 --- a/src/bin.h +++ b/src/bin.h @@ -118,7 +118,7 @@ struct ec3_tag_table_entry { b_i32 tag_checksum; b_i32 tag_reserved1; b_i64 tag_ident; - b_i64 tag_reserved2; + b_i64 tag_length; }; /* Extents serve two purposes: @@ -206,7 +206,6 @@ struct ec3_vnode_group { b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_4K + 1]; } g_4k; -#if 0 struct { struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_8K]; b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_8K + 1]; @@ -226,7 +225,6 @@ struct ec3_vnode_group { struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_64K]; b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_64K + 1]; } g_64k; -#endif }; }; @@ -247,7 +245,6 @@ struct ec3_vnode_chunk_group { b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_4K + 1]; } g_4k; -#if 0 struct { struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_8K]; b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_8K + 1]; @@ -267,7 +264,6 @@ struct ec3_vnode_chunk_group { struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_64K]; b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_64K + 1]; } g_64k; -#endif }; }; diff --git a/src/cluster.c b/src/cluster.c index c79addc..ae1357e 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -18,7 +18,8 @@ static int node_init(struct ec3_cluster_group *n) static int tree_get_node(struct b_tree *p, unsigned long id, b_tree_node *n) { struct cluster_table *table = (struct cluster_table *)p; - size_t offset = (id * sizeof(struct ec3_cluster_group)); + size_t offset + = table->t_offset + (id * sizeof(struct ec3_cluster_group)); fseek(table->t_storage, offset, SEEK_SET); size_t r = fread(n, @@ -34,7 +35,8 @@ static int tree_put_node( const b_tree_node *n) { struct cluster_table *table = (struct cluster_table *)p; - size_t offset = (id * sizeof(struct ec3_cluster_group)); + size_t offset + = table->t_offset + (id * sizeof(struct ec3_cluster_group)); fseek(table->t_storage, offset, SEEK_SET); size_t r = fwrite( n, @@ -147,11 +149,15 @@ static const struct b_tree_ops cluster_table_ops = { .entry_compare = entry_compare, }; -void cluster_table_init(struct cluster_table *table, FILE *storage) +void cluster_table_init( + struct cluster_table *table, + FILE *storage, + size_t offset) { memset(table, 0x0, sizeof *table); table->t_storage = storage; + table->t_offset = offset; b_tree_init( &table->t_base, diff --git a/src/cluster.h b/src/cluster.h index 0df9a61..e8a4a95 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -9,6 +9,7 @@ struct cluster_table { struct b_tree t_base; size_t t_nr_groups; FILE *t_storage; + size_t t_offset; }; struct cluster { @@ -19,7 +20,10 @@ struct cluster { unsigned int c_flags; }; -extern void cluster_table_init(struct cluster_table *table, FILE *storage); +extern void cluster_table_init( + struct cluster_table *table, + FILE *storage, + size_t offset); extern void cluster_table_finish(struct cluster_table *table); extern int cluster_table_get( diff --git a/src/commands.h b/src/commands.h index b7d05a2..d01bd45 100644 --- a/src/commands.h +++ b/src/commands.h @@ -5,6 +5,7 @@ enum command_id { CMD_ROOT = 0, CMD_CREATE, CMD_WRAP, + CMD_UNWRAP, CMD_CAPTURE, CMD_SHELL, CMD_EXTRACT, diff --git a/src/file.c b/src/file.c deleted file mode 100644 index 113259d..0000000 --- a/src/file.c +++ /dev/null @@ -1,39 +0,0 @@ -#include "pipeline.h" - -#include - -static enum ec3_status write( - struct ec3_pipeline_stage *stage, - const void *src, - size_t len, - void *dest, - size_t *nr_written) -{ - FILE *fp = stage->s_arg; - size_t r = fwrite(src, 1, len, fp); - - *nr_written = r; - - if (r < len) { - return EC3_ERR_IO_FAILURE; - } - - return EC3_SUCCESS; -} - -static enum ec3_status read( - struct ec3_pipeline_stage *stage, - const void *src, - void *dest, - size_t *nr_read) -{ - FILE *fp = stage->s_arg; - return EC3_SUCCESS; -} - -const struct ec3_pipeline_stage_type pipeline_file = { - .t_id = EC3_PIPELINE_FILE, - .t_flags = EC3_PIPELINE_F_NONE, - .t_data_in = read, - .t_data_out = write, -}; diff --git a/src/pipeline.c b/src/pipeline.c index 86530cb..8670f18 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -1,4 +1,5 @@ #include "pipeline.h" + #include "bin.h" #include @@ -6,12 +7,10 @@ extern const struct ec3_pipeline_stage_type pipeline_zstd; extern const struct ec3_pipeline_stage_type pipeline_aes256; -extern const struct ec3_pipeline_stage_type pipeline_file; static const struct ec3_pipeline_stage_type *stage_types[] = { [EC3_PIPELINE_ZSTD] = &pipeline_zstd, [EC3_PIPELINE_AES256] = &pipeline_aes256, - [EC3_PIPELINE_FILE] = &pipeline_file, }; static const size_t nr_stage_types = sizeof stage_types / sizeof stage_types[0]; @@ -58,6 +57,10 @@ extern enum ec3_status ec3_pipeline_create( struct ec3_pipeline_stage_args stages[], size_t nr_stages, size_t cluster_size, + FILE *target, + size_t target_base_offset, + FILE *cluster_table, + size_t cluster_table_base_offset, struct ec3_pipeline **out) { enum ec3_status status = EC3_SUCCESS; @@ -100,6 +103,15 @@ extern enum ec3_status ec3_pipeline_create( b_queue_push_back(&pipeline->p_stages, &stage->s_entry); } + cluster_table_init( + &pipeline->p_cluster_table, + cluster_table, + cluster_table_base_offset); + + pipeline->p_target = target; + pipeline->p_target_base_offset = target_base_offset; + pipeline->p_cluster_size = cluster_size; + *out = pipeline; return status; } @@ -108,11 +120,11 @@ void ec3_pipeline_destroy(struct ec3_pipeline *p) { } -enum ec3_status ec3_pipeline_data_out( +enum ec3_status ec3_pipeline_write_cluster( struct ec3_pipeline *pipeline, void *p, size_t len, - size_t *nr_written) + size_t *bytes_written) { b_queue_entry *cur = b_queue_first(&pipeline->p_stages); enum ec3_status status = EC3_SUCCESS; @@ -121,6 +133,8 @@ enum ec3_status ec3_pipeline_data_out( size_t stage_in_size = len; size_t stage_out_size = 0; + size_t dest_max = pipeline->p_cluster_size; + while (cur) { struct ec3_pipeline_stage *stage = b_unbox(struct ec3_pipeline_stage, cur, s_entry); @@ -132,11 +146,12 @@ enum ec3_status ec3_pipeline_data_out( dest = src; } - status = stage->s_type->t_data_out( + status = stage->s_type->t_cluster_out( stage, src, stage_in_size, dest, + dest_max, &stage_out_size); if (status != EC3_SUCCESS) { @@ -148,20 +163,112 @@ enum ec3_status ec3_pipeline_data_out( cur = b_queue_next(cur); } - if (nr_written) { - *nr_written = stage_out_size; + stage_out_size = fwrite(src, 1, stage_in_size, pipeline->p_target); + + if (bytes_written) { + *bytes_written = stage_out_size; } + struct cluster cluster = { + .c_id = pipeline->p_next_cluster_id++, + .c_base = pipeline->p_data_offset, + .c_len = stage_out_size, + .c_flags = pipeline->p_cluster_flags, + .c_checksum = 0, + }; + + pipeline->p_data_offset += stage_out_size; + + cluster_table_put(&pipeline->p_cluster_table, &cluster); + return EC3_SUCCESS; } -enum ec3_status ec3_pipeline_data_in( +enum ec3_status ec3_pipeline_seek(struct ec3_pipeline *pipeline, size_t pos) +{ + fseek(pipeline->p_target, + pipeline->p_target_base_offset + pos, + SEEK_SET); + return EC3_SUCCESS; +} + +enum ec3_status ec3_pipeline_read_cluster( struct ec3_pipeline *pipeline, void *p, - size_t max, + size_t cluster_id, size_t *nr_read) { - return EC3_ERR_NOT_SUPPORTED; + if (!pipeline->p_read_buf) { + pipeline->p_read_buf = malloc(pipeline->p_cluster_size); + + if (!pipeline->p_read_buf) { + return EC3_ERR_NO_MEMORY; + } + } + + struct cluster cluster = {0}; + enum ec3_status status = cluster_table_get( + &pipeline->p_cluster_table, + cluster_id, + &cluster); + + if (status != EC3_SUCCESS) { + return status; + } + + b_queue_entry *cur = b_queue_first(&pipeline->p_stages); + void *src = pipeline->p_read_buf; + + fseek(pipeline->p_target, + pipeline->p_target_base_offset + cluster.c_base, + SEEK_SET); + + size_t stage_in_size + = fread(pipeline->p_read_buf, + 1, + cluster.c_len, + pipeline->p_target); + size_t stage_out_size = 0; + + if (stage_in_size != cluster.c_len) { + return EC3_ERR_END_OF_FILE; + } + + while (cur) { + struct ec3_pipeline_stage *stage + = b_unbox(struct ec3_pipeline_stage, cur, s_entry); + + void *dest; + if (stage->s_type->t_flags & EC3_PIPELINE_F_BUFFERED) { + dest = stage->s_buf; + } else { + dest = src; + } + + status = stage->s_type->t_cluster_in( + stage, + src, + stage_in_size, + dest, + pipeline->p_cluster_size, + &stage_out_size); + + if (status != EC3_SUCCESS) { + return status; + } + + src = dest; + stage_in_size = stage_out_size; + cur = b_queue_next(cur); + } + + memcpy(p, src, stage_in_size); + + if (nr_read) { + *nr_read = stage_in_size; + } + + return EC3_SUCCESS; } size_t ec3_get_cluster_size(unsigned int v) @@ -189,4 +296,4 @@ enum ec3_pipeline_stage_type_id ec3_get_pipeline_stage_for_encryption_func( default: return EC3_PIPELINE_NONE; } -} \ No newline at end of file +} diff --git a/src/pipeline.h b/src/pipeline.h index a6b27d7..3d81a89 100644 --- a/src/pipeline.h +++ b/src/pipeline.h @@ -1,16 +1,17 @@ #ifndef PIPELINE_H_ #define PIPELINE_H_ +#include "cluster.h" #include "status.h" #include #include +#include enum ec3_pipeline_stage_type_id { EC3_PIPELINE_NONE = 0, EC3_PIPELINE_AES256, EC3_PIPELINE_ZSTD, - EC3_PIPELINE_FILE, }; enum ec3_pipeline_stage_type_flags { @@ -24,16 +25,19 @@ struct ec3_pipeline_stage_type { enum ec3_pipeline_stage_type_id t_id; enum ec3_pipeline_stage_type_flags t_flags; - enum ec3_status (*t_data_in)( - struct ec3_pipeline_stage *, - const void *, - void *, - size_t *); - enum ec3_status (*t_data_out)( + enum ec3_status (*t_cluster_in)( struct ec3_pipeline_stage *, const void *, size_t, void *, + size_t, + size_t *); + enum ec3_status (*t_cluster_out)( + struct ec3_pipeline_stage *, + const void *, + size_t, + void *, + size_t, size_t *); }; @@ -50,6 +54,17 @@ struct ec3_pipeline_stage_args { }; struct ec3_pipeline { + FILE *p_target; + size_t p_target_base_offset; + + size_t p_cluster_size; + unsigned int p_cluster_flags; + struct cluster_table p_cluster_table; + + size_t p_next_cluster_id; + size_t p_data_offset; + + void *p_read_buf; b_queue p_stages; }; @@ -57,22 +72,28 @@ extern enum ec3_status ec3_pipeline_create( struct ec3_pipeline_stage_args stages[], size_t nr_stages, size_t cluster_size, + FILE *target, + size_t target_base_offset, + FILE *cluster_table, + size_t cluster_table_base_offset, struct ec3_pipeline **out); extern void ec3_pipeline_destroy(struct ec3_pipeline *p); -extern enum ec3_status ec3_pipeline_data_out( +extern enum ec3_status ec3_pipeline_write_cluster( struct ec3_pipeline *pipeline, void *p, size_t len, - size_t *nr_written); -extern enum ec3_status ec3_pipeline_data_in( + size_t *bytes_written); +extern enum ec3_status ec3_pipeline_read_cluster( struct ec3_pipeline *pipeline, void *p, - size_t max, + size_t cluster_id, size_t *nr_read); extern size_t ec3_get_cluster_size(unsigned int v); -extern enum ec3_pipeline_stage_id ec3_get_pipeline_stage_for_encryption_func(unsigned int func); -extern enum ec3_pipeline_stage_id ec3_get_pipeline_stage_for_compression_func(unsigned int func); +extern enum ec3_pipeline_stage_type_id +ec3_get_pipeline_stage_for_encryption_func(unsigned int func); +extern enum ec3_pipeline_stage_type_id +ec3_get_pipeline_stage_for_compression_func(unsigned int func); #endif diff --git a/src/query.c b/src/query.c index 5905134..113a871 100644 --- a/src/query.c +++ b/src/query.c @@ -1,3 +1,4 @@ +#include "bin.h" #include "commands.h" #include "misc.h" #include "read.h" @@ -23,6 +24,30 @@ static void tag_type_string(unsigned long in, char out[5]) out[4] = 0; } +static const char *encryption_function_name(unsigned int v) +{ + switch (v) { + case EC3_ENCRYPTION_NONE: + return "NONE"; + case EC3_ENCRYPTION_AES256: + return "AES256"; + default: + return "UNKNOWN"; + } +} + +static const char *compression_function_name(unsigned int v) +{ + switch (v) { + case EC3_COMPRESSION_NONE: + return "NONE"; + case EC3_COMPRESSION_ZSTD: + return "ZSTD"; + default: + return "UNKNOWN"; + } +} + static int query( const b_command *self, const b_arglist *opt, @@ -65,6 +90,12 @@ static int query( printf(" %-20s: %04x\n", "format version", c_info->c_version); printf(" %-20s: %s\n", "identifier", container_id); printf(" %-20s: %u\n", "cluster size", c_info->c_cluster_size); + printf(" %-20s: %s\n", + "compression", + compression_function_name(c_info->c_compression_function)); + printf(" %-20s: %s\n", + "encryption", + encryption_function_name(c_info->c_encryption_function)); printf(" %-20s: 0x%zx\n", "tag table offset", c_info->c_tag_table_offset); @@ -95,6 +126,9 @@ static int query( printf(" (%lx/%s) %s\n", tags[i].tag_type, tag_type, tag_id); printf(" %-8s: %08lx\n", "flags", tags[i].tag_flags); printf(" %-8s: %08lx\n", "checksum", tags[i].tag_checksum); + printf(" %-8s: %llu bytes\n", + "size", + tags[i].tag_total_length); } printf("\nextents:\n"); diff --git a/src/read.c b/src/read.c index 7a4fb82..f518757 100644 --- a/src/read.c +++ b/src/read.c @@ -19,6 +19,13 @@ struct ec3_reader { struct ec3_pipeline *r_pipeline; }; +struct ec3_tag_reader { + struct ec3_reader *tag_parent; + const struct ec3_tag_info *tag_info; + size_t tag_current_cluster; + unsigned char *tag_buf; +}; + static void decode_header(const struct ec3_header *in, struct ec3_reader *out) { out->c_info.c_version = b_i16_btoh(in->h_version); @@ -32,6 +39,8 @@ static void decode_header(const struct ec3_header *in, struct ec3_reader *out) out->c_info.c_nr_tags = b_i32_btoh(in->h_tag_count); out->c_info.c_nr_extents = b_i32_btoh(in->h_tag_count); out->c_info.c_nr_cluster_groups = b_i32_btoh(in->h_cluster_group_count); + out->c_info.c_encryption_function = b_i16_btoh(in->h_encryption); + out->c_info.c_compression_function = b_i16_btoh(in->h_compression); out->c_info.c_id = b_i64_btoh(in->h_app_magic); } @@ -43,6 +52,7 @@ static void decode_tag( out->tag_flags = b_i32_btoh(in->tag_flags); out->tag_checksum = b_i32_btoh(in->tag_checksum); out->tag_ident = b_i64_btoh(in->tag_ident); + out->tag_total_length = b_i64_btoh(in->tag_length); } static void decode_extent( @@ -127,16 +137,15 @@ enum ec3_status ec3_reader_create(FILE *inp, struct ec3_reader **out) unsigned short compression = b_i16_btoh(header.h_compression); unsigned short encryption = b_i16_btoh(header.h_encryption); - struct ec3_pipeline_stage_args stages[3] = {0}; - stages[0].type = EC3_PIPELINE_FILE; + struct ec3_pipeline_stage_args stages[2] = {0}; if (encryption != EC3_ENCRYPTION_NONE) { - stages[1].type = ec3_get_pipeline_stage_for_encryption_func( + stages[0].type = ec3_get_pipeline_stage_for_encryption_func( encryption); } if (compression != EC3_COMPRESSION_NONE) { - stages[2].type = ec3_get_pipeline_stage_for_compression_func( + stages[1].type = ec3_get_pipeline_stage_for_compression_func( compression); } @@ -144,6 +153,10 @@ enum ec3_status ec3_reader_create(FILE *inp, struct ec3_reader **out) stages, sizeof stages / sizeof stages[0], reader->c_info.c_cluster_size, + inp, + sizeof header, + inp, + reader->c_info.c_cluster_table_offset, &reader->r_pipeline); if (status != EC3_SUCCESS) { @@ -215,12 +228,207 @@ const struct ec3_extent_info *ec3_reader_get_extents( return reader->c_extents; } +const struct ec3_tag_info *ec3_reader_get_tag_info( + const struct ec3_reader *reader, + unsigned long long ident) +{ + for (size_t i = 0; i < reader->c_info.c_nr_tags; i++) { + if (reader->c_tags[i].tag_ident == ident) { + return &reader->c_tags[i]; + } + } + + return NULL; +} + enum ec3_status ec3_reader_open_tag( struct ec3_reader *reader, uint64_t tag_ident, struct ec3_tag_reader **out) { - return EC3_ERR_NOT_SUPPORTED; + const struct ec3_tag_info *tag_info = NULL; + for (unsigned int i = 0; i < reader->c_info.c_nr_tags; i++) { + if (reader->c_tags[i].tag_ident == tag_ident) { + tag_info = &reader->c_tags[i]; + break; + } + } + + if (!tag_info) { + return EC3_ERR_NO_ENTRY; + } + + struct ec3_tag_reader *tag_reader = malloc(sizeof *tag_reader); + if (!tag_reader) { + return EC3_ERR_NO_MEMORY; + } + + memset(tag_reader, 0x0, sizeof *tag_reader); + + tag_reader->tag_parent = reader; + tag_reader->tag_info = tag_info; + tag_reader->tag_current_cluster = (size_t)-1; + tag_reader->tag_buf = malloc(reader->c_info.c_cluster_size); + + if (!tag_reader->tag_buf) { + free(tag_reader); + return EC3_ERR_NO_MEMORY; + } + + *out = tag_reader; + return EC3_SUCCESS; +} + +static int logical_cluster_to_physical_cluster( + struct ec3_tag_reader *tag, + size_t cluster_id, + size_t *physical_cluster) +{ + const struct ec3_extent_info *extent_buf = tag->tag_parent->c_extents; + size_t nr_extents = tag->tag_parent->c_info.c_nr_extents; + for (size_t i = 0; i < nr_extents; i++) { + const struct ec3_extent_info *extent = &extent_buf[i]; + + if (extent->ex_owner != tag->tag_info->tag_ident) { + continue; + } + + if (cluster_id < extent->ex_logical_cluster) { + continue; + } + + if (cluster_id + >= extent->ex_logical_cluster + extent->ex_count) { + continue; + } + + *physical_cluster = (cluster_id - extent->ex_logical_cluster) + + extent->ex_physical_cluster; + return 0; + } + + return -1; +} + +static enum ec3_status read_cluster( + struct ec3_tag_reader *tag, + size_t cluster_id, + size_t *nr_read) +{ + size_t physical_cluster; + if (logical_cluster_to_physical_cluster( + tag, + cluster_id, + &physical_cluster) + != 0) { + /* end-of-file */ + *nr_read = 0; + return EC3_SUCCESS; + } + + return ec3_pipeline_read_cluster( + tag->tag_parent->r_pipeline, + tag->tag_buf, + physical_cluster, + nr_read); +} + +enum ec3_status ec3_tag_reader_read( + struct ec3_tag_reader *tag, + size_t first_cluster, + size_t nr_clusters, + void *buf, + size_t *bytes_read) +{ + unsigned char *p = buf; + + enum ec3_status status = EC3_SUCCESS; + size_t last_cluster = first_cluster + nr_clusters - 1; + size_t nr_read_from_cluster = 0; + size_t nr_read = 0; + + for (size_t i = first_cluster; i <= last_cluster; i++) { + status = read_cluster(tag, i, &nr_read_from_cluster); + + if (status != EC3_SUCCESS) { + return status; + } + + memcpy(p + nr_read, tag->tag_buf, nr_read_from_cluster); + nr_read += nr_read_from_cluster; + } + + *bytes_read = nr_read; + return EC3_SUCCESS; +#if 0 + enum ec3_status status = EC3_SUCCESS; + unsigned char *p = buf; + size_t p_offset = 0; + + size_t cluster_size = tag->tag_parent->c_info.c_cluster_size; + const unsigned char *cluster_data = tag->tag_buf; + size_t nr_read_from_cluster = 0; + + size_t cluster = 0; + size_t start = offset; + size_t end = offset + count; + + size_t first_cluster = start / cluster_size; + + if ((start % cluster_size) > 0) { + cluster = start / cluster_size; + status = read_cluster(tag, cluster, &nr_read_from_cluster); + + if (status != EC3_SUCCESS) { + return status; + } + + size_t cluster_offset = start % cluster_size; + size_t copy_sz = nr_read_from_cluster - cluster_offset; + memcpy(p + p_offset, cluster_data + cluster_offset, copy_sz); + p_offset += copy_sz; + start += copy_sz; + } + + if (p_offset == count) { + *nr_read = p_offset; + return EC3_SUCCESS; + } + + for (size_t i = first_cluster; i <= last_cluster; i++) { + status = read_cluster(tag, cluster, &nr_read_from_cluster); + + if (status != EC3_SUCCESS) { + return status; + } + + memcpy(p + p_offset, cluster_data, nr_read_from_cluster); + p_offset += nr_read_from_cluster; + } + + if ((end % cluster_size) > 0) { + cluster = end / cluster_size; + status = read_cluster(tag, cluster, &nr_read_from_cluster); + + if (status != EC3_SUCCESS) { + return status; + } + + size_t cluster_offset = 0; + size_t copy_sz = cluster_offset; + if (copy_sz > nr_read_from_cluster) { + copy_sz = nr_read_from_cluster; + } + + memcpy(p + p_offset, cluster_data + cluster_offset, copy_sz); + p_offset += copy_sz; + start += copy_sz; + } + + *nr_read = p_offset; +#endif + + return EC3_SUCCESS; } enum ec3_status ec3_tag_reader_close(struct ec3_tag_reader *tag) diff --git a/src/read.h b/src/read.h index 40b90d1..e2a7d6d 100644 --- a/src/read.h +++ b/src/read.h @@ -19,6 +19,9 @@ struct ec3_container_info { unsigned int c_nr_extents; unsigned int c_nr_cluster_groups; + unsigned int c_encryption_function; + unsigned int c_compression_function; + uint64_t c_id; }; @@ -27,6 +30,7 @@ struct ec3_tag_info { unsigned long tag_flags; unsigned long tag_checksum; unsigned long long tag_ident; + unsigned long long tag_total_length; }; struct ec3_extent_info { @@ -46,11 +50,21 @@ extern const struct ec3_tag_info *ec3_reader_get_tags( extern const struct ec3_extent_info *ec3_reader_get_extents( const struct ec3_reader *reader); +extern const struct ec3_tag_info *ec3_reader_get_tag_info( + const struct ec3_reader *reader, + unsigned long long ident); + extern enum ec3_status ec3_reader_open_tag( struct ec3_reader *reader, uint64_t tag_ident, struct ec3_tag_reader **out); +extern enum ec3_status ec3_tag_reader_read( + struct ec3_tag_reader *tag, + size_t offset, + size_t count, + void *buf, + size_t *nr_read); extern enum ec3_status ec3_tag_reader_close(struct ec3_tag_reader *tag); #endif diff --git a/src/status.h b/src/status.h index e948cb7..dfff468 100644 --- a/src/status.h +++ b/src/status.h @@ -11,6 +11,7 @@ enum ec3_status { EC3_ERR_INVALID_VALUE, EC3_ERR_NAME_EXISTS, EC3_ERR_IO_FAILURE, + EC3_ERR_END_OF_FILE, }; #endif diff --git a/src/wrap.c b/src/wrap.c index 6605a5f..bc00a84 100644 --- a/src/wrap.c +++ b/src/wrap.c @@ -42,7 +42,7 @@ static enum ec3_status add_file( struct ec3_tag_writer *tag = NULL; enum ec3_status status - = ec3_writer_create_tag(writer, type, id, 0, &tag); + = ec3_writer_create_tag(writer, 0, type, id, 0, &tag); if (status != EC3_SUCCESS) { b_err("cannot initialise EC3 tag writer"); diff --git a/src/write.c b/src/write.c index b59627b..d07bed5 100644 --- a/src/write.c +++ b/src/write.c @@ -25,7 +25,7 @@ struct ec3_writer { size_t w_nr_tags; FILE *w_data; - struct cluster_table w_cluster_table; + FILE *w_cluster_table; FILE *w_extent_table; FILE *w_tag_table; @@ -33,11 +33,13 @@ struct ec3_writer { }; struct ec3_tag_writer { + size_t w_index; struct ec3_writer *w_parent; unsigned long w_type; uint64_t w_ident; unsigned long w_flags; unsigned char *w_buf; + size_t w_total_bytes; size_t w_ptr; size_t w_nr_clusters; b_queue_entry w_entry; @@ -57,14 +59,18 @@ enum ec3_status ec3_writer_create( size_t cluster_size = ec3_get_cluster_size(param->p_cluster_size); - FILE *cluster_table = tmpfile(); - writer->w_data = param->p_outp; - cluster_table_init(&writer->w_cluster_table, cluster_table); writer->w_extent_table = tmpfile(); writer->w_tag_table = tmpfile(); + writer->w_cluster_table = tmpfile(); - struct ec3_pipeline_stage_args stages[3] = {0}; + struct ec3_header header = {0}; + size_t written = fwrite(&header, sizeof header, 1, writer->w_data); + if (written != 1) { + return EC3_ERR_IO_FAILURE; + } + + struct ec3_pipeline_stage_args stages[2] = {0}; if (param->p_compression_func != EC3_COMPRESSION_NONE) { stages[0].type = ec3_get_pipeline_stage_for_compression_func( @@ -76,14 +82,15 @@ enum ec3_status ec3_writer_create( param->p_encryption_func); } - stages[2].type = EC3_PIPELINE_FILE; - stages[2].arg = writer->w_data; - struct ec3_pipeline *pipeline = NULL; enum ec3_status status = ec3_pipeline_create( stages, sizeof stages / sizeof stages[0], cluster_size, + writer->w_data, + 0, + writer->w_cluster_table, + 0, &pipeline); if (status != EC3_SUCCESS) { @@ -92,12 +99,6 @@ enum ec3_status ec3_writer_create( writer->w_pipeline = pipeline; - struct ec3_header header = {0}; - size_t written = fwrite(&header, sizeof header, 1, writer->w_data); - if (written != 1) { - return EC3_ERR_IO_FAILURE; - } - *out = writer; return EC3_SUCCESS; } @@ -167,7 +168,7 @@ void ec3_writer_finish(struct ec3_writer *w) } size_t cluster_table_offset = ftell(w->w_data); - status = copy_file(w->w_cluster_table.t_storage, w->w_data); + status = copy_file(w->w_cluster_table, w->w_data); size_t extent_table_offset = ftell(w->w_data); status = copy_file(w->w_extent_table, w->w_data); @@ -187,8 +188,10 @@ void ec3_writer_finish(struct ec3_writer *w) header.h_tag_count = b_i32_htob(w->w_nr_tags); header.h_extent_count = b_i32_htob(w->w_nr_extents); header.h_app_magic = b_i64_htob(w->w_param.p_ident); + header.h_encryption = b_i16_htob(w->w_param.p_encryption_func); + header.h_compression = b_i16_htob(w->w_param.p_compression_func); header.h_cluster_group_count - = b_i32_htob(w->w_cluster_table.t_nr_groups); + = b_i32_htob(w->w_pipeline->p_cluster_table.t_nr_groups); fwrite(&header, sizeof header, 1, w->w_data); } @@ -215,9 +218,10 @@ static bool is_tag_ident_free(struct ec3_writer *w, uint64_t ident) enum ec3_status ec3_writer_create_tag( struct ec3_writer *w, + enum ec3_tag_writer_flags writer_flags, uint32_t tag_type, uint64_t tag_ident, - unsigned int flags, + unsigned int tag_flags, struct ec3_tag_writer **out_writer) { struct ec3_tag_writer *tag = malloc(sizeof *tag); @@ -230,7 +234,7 @@ enum ec3_status ec3_writer_create_tag( size_t cluster_size = ec3_get_cluster_size(w->w_param.p_cluster_size); tag->w_parent = w; - tag->w_flags = flags; + tag->w_flags = tag_flags; tag->w_type = tag_type; tag->w_ident = tag_ident; tag->w_buf = malloc(cluster_size); @@ -240,19 +244,7 @@ enum ec3_status ec3_writer_create_tag( return EC3_ERR_NO_MEMORY; } - struct ec3_tag_table_entry entry = {0}; - entry.tag_type = b_i32_htob(tag_type); - entry.tag_ident = b_i64_htob(tag_ident); - entry.tag_flags = b_i32_htob(flags); - - size_t written = fwrite(&entry, sizeof entry, 1, w->w_tag_table); - if (written != 1) { - free(tag->w_buf); - free(tag); - return EC3_ERR_IO_FAILURE; - } - - w->w_nr_tags++; + tag->w_index = w->w_nr_tags++; *out_writer = tag; return EC3_SUCCESS; } @@ -267,7 +259,7 @@ static enum ec3_status flush_tag_buffer(struct ec3_tag_writer *w) enum ec3_status status = EC3_SUCCESS; size_t nr_written = 0; - status = ec3_pipeline_data_out( + status = ec3_pipeline_write_cluster( container->w_pipeline, buf, w->w_ptr, @@ -277,17 +269,6 @@ static enum ec3_status flush_tag_buffer(struct ec3_tag_writer *w) return status; } - struct cluster cluster = { - .c_id = container->w_next_cluster_id++, - .c_base = container->w_data_offset, - .c_len = nr_written, - .c_flags = w->w_flags, - .c_checksum = 0, - }; - - cluster_table_put(&container->w_cluster_table, &cluster); - - container->w_data_offset += nr_written; w->w_ptr = 0; w->w_nr_clusters++; @@ -335,6 +316,7 @@ enum ec3_status ec3_tag_writer_write( remaining -= to_write; w->w_ptr += to_write; + w->w_total_bytes += to_write; if (remaining == 0) { status = flush_tag_buffer(w); } @@ -361,5 +343,18 @@ enum ec3_status ec3_tag_writer_finish(struct ec3_tag_writer *w) return status; } + struct ec3_tag_table_entry entry = {0}; + entry.tag_type = b_i32_htob(w->w_type); + entry.tag_ident = b_i64_htob(w->w_ident); + entry.tag_flags = b_i32_htob(w->w_flags); + entry.tag_length = b_i64_htob(w->w_total_bytes); + + size_t written + = fwrite(&entry, sizeof entry, 1, w->w_parent->w_tag_table); + + if (written != 1) { + return EC3_ERR_IO_FAILURE; + } + return status; } diff --git a/src/write.h b/src/write.h index 817a48f..a3ac76f 100644 --- a/src/write.h +++ b/src/write.h @@ -19,6 +19,10 @@ struct ec3_parameters { size_t p_encryption_key_size; }; +enum ec3_tag_writer_flags { + EC3_TAG_WRITER_BUFFERED = 0x01u, +}; + struct ec3_writer; struct ec3_tag_writer; @@ -30,9 +34,10 @@ extern void ec3_writer_write_image(struct ec3_writer *w, FILE *fp); extern enum ec3_status ec3_writer_create_tag( struct ec3_writer *w, + enum ec3_tag_writer_flags writer_flags, uint32_t tag_type, uint64_t tag_ident, - unsigned int flags, + unsigned int tag_flags, struct ec3_tag_writer **out_writer); extern enum ec3_status ec3_tag_writer_write( diff --git a/src/zstd.c b/src/zstd.c index 3a5bbec..95eaf04 100644 --- a/src/zstd.c +++ b/src/zstd.c @@ -7,24 +7,28 @@ static enum ec3_status compress( const void *src, size_t len, void *dest, + size_t dest_max, size_t *nr_written) { - *nr_written = ZSTD_compress(dest, len, src, len, 10); + *nr_written = ZSTD_compress(dest, dest_max, src, len, 10); return EC3_SUCCESS; } static enum ec3_status decompress( struct ec3_pipeline_stage *stage, const void *src, + size_t len, void *dest, + size_t dest_max, size_t *nr_read) { + *nr_read = ZSTD_decompress(dest, dest_max, src, len); return EC3_SUCCESS; } const struct ec3_pipeline_stage_type pipeline_zstd = { .t_id = EC3_PIPELINE_ZSTD, .t_flags = EC3_PIPELINE_F_BUFFERED, - .t_data_in = decompress, - .t_data_out = compress, + .t_cluster_in = decompress, + .t_cluster_out = compress, };