wash/ec3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add cluster i/o pipeline

Browse Source
This commit is contained in:
max 🍓
2025-01-30 18:10:38 +00:00
parent a69595b324
commit dad7c27bf6
18 changed files with 1147 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@@ -7,7 +7,7 @@ set(Bluelib_STATIC 1)
find_package(Bluelib REQUIRED)
find_package(ZSTD REQUIRED)
file(GLOB ec3_sources
file(GLOB_RECURSE ec3_sources
src/*.c
src/*.h)
@@ -19,4 +19,4 @@ target_link_libraries(ec3
Bluelib::Cmd
${ZSTD_LIBRARY})
target_include_directories(ec3 PRIVATE ${ZSTD_INCLUDE_DIR})
target_include_directories(ec3 PRIVATE ${ZSTD_INCLUDE_DIR})

12
doc/format.txt
View File

@@ -299,13 +299,11 @@ version: 1.0
Header Value Cluster Size (bytes) Cluster Size (kilobytes)
--------------------------------------------------------------------
0x00 16,384 16
0x01 32,768 32
0x02 65,536 64
0x03 131,072 128
0x04 262,144 256
0x05 524,288 512
0x06 1,048,576 1,024
0x00 4,096 4
0x01 8,192 8
0x02 16,384 16
0x03 32,768 32
0x04 65,536 64
5.1.4 Tag Table Offset
This specifies the offset in bytes from the beginning of the image file

27
src/aes256.c Normal file
View File

@@ -0,0 +1,27 @@
#include "pipeline.h"
static enum ec3_status encrypt(
struct ec3_pipeline_stage *stage,
const void *src,
size_t len,
void *dest,
size_t *nr_written)
{
return EC3_SUCCESS;
}
static enum ec3_status decrypt(
struct ec3_pipeline_stage *stage,
const void *src,
void *dest,
size_t *nr_read)
{
return EC3_SUCCESS;
}
const struct ec3_pipeline_stage_type pipeline_aes256 = {
.t_id = EC3_PIPELINE_AES256,
.t_flags = EC3_PIPELINE_F_NONE,
.t_data_in = decrypt,
.t_data_out = encrypt,
};

0
src/b-tree.c Normal file
View File

29
src/b-tree.h Normal file
View File

@@ -0,0 +1,29 @@
#ifndef B_TREE_H_
#define B_TREE_H_
#include <stddef.h>
struct b_tree;
typedef void *b_tree_node;
typedef void *b_tree_node_entry;
struct b_tree_ops {
size_t node_size;
size_t entry_size;
int (*tree_get_node)(struct b_tree *, int, void *);
int (*tree_put_node)(struct b_tree *, int, void *);
int (*tree_alloc_node)(struct b_tree *);
int (*node_get_nr_entries)(b_tree_node *);
b_tree_node_entry *(*node_get_entry)(b_tree_node *, int);
int (*node_get_child)(b_tree_node *, int);
int (*entry_compare)(b_tree_node_entry *, b_tree_node_entry *);
};
struct b_tree {
const struct b_tree_ops *tree_ops;
};
#endif

269
src/bin.h
View File

@@ -3,39 +3,110 @@
#include <blue/core/endian.h>
#define EC3_SIGNATURE 0x45433358
#define EC3_SIGNATURE 0x45433358
#define EC3_VERSION_1_0 0x0100
#define EC3_VERSION_1_0 0x0100
#define EC3_CLUSTER_16K 0x00u
#define EC3_CLUSTER_32K 0x01u
#define EC3_CLUSTER_64K 0x02u
#define EC3_CLUSTER_128K 0x03u
#define EC3_CLUSTER_256K 0x04u
#define EC3_CLUSTER_512K 0x05u
#define EC3_CLUSTER_1M 0x06u
#define EC3_CLUSTER_4K 0x00u
#define EC3_CLUSTER_8K 0x01u
#define EC3_CLUSTER_16K 0x02u
#define EC3_CLUSTER_32K 0x03u
#define EC3_CLUSTER_64K 0x04u
#define EC3_TAG_VOLU 0x564F4C55
#define EC3_TAG_CTAB 0x43544142
#define EC3_TAG_XATR 0x58415452
#define EC3_TAG_STAB 0x53544142
#define EC3_TAG_MFST 0x4D465354
#define EC3_TAG_BLOB 0x424C4F42
#define EC3_TAG_EXEC 0x45584543
#define EC3_TAG_CERT 0x43455254
#define EC3_TAG_CSIG 0x43534947
#define EC3_COMPRESSION_NONE 0x00u
#define EC3_COMPRESSION_ZSTD 0x01u
#define EC3_TAG_SIGNED 0x00000001u
#define EC3_TAG_COMPRESSED 0x00000002u
#define EC3_TAG_ENCRYPTED 0x00000004u
#define EC3_ENCRYPTION_NONE 0x00u
#define EC3_ENCRYPTION_AES256 0x01u
#define EC3_TAG_VOLU 0x564F4C55
#define EC3_TAG_CTAB 0x43544142
#define EC3_TAG_XATR 0x58415452
#define EC3_TAG_STAB 0x53544142
#define EC3_TAG_MFST 0x4D465354
#define EC3_TAG_BLOB 0x424C4F42
#define EC3_TAG_EXEC 0x45584543
#define EC3_TAG_CERT 0x43455254
#define EC3_TAG_CSIG 0x43534947
#define EC3_TAG_SIGNED 0x00000001u
#define EC3_TAG_COMPRESSED 0x00000002u
#define EC3_TAG_ENCRYPTED 0x00000004u
/* 32K per cluster group */
#define EC3_CLUSTERS_PER_GROUP 1635
/* 1K per extent group */
#define EC3_EXTENTS_PER_GROUP 36
/* each chunk group occupies one cluster, so the number of chunks per group
* depends on the cluster size */
#define EC3_CHUNKS_PER_GROUP_4K 146
#define EC3_CHUNKS_PER_GROUP_8K 292
#define EC3_CHUNKS_PER_GROUP_16K 584
#define EC3_CHUNKS_PER_GROUP_32K 1170
#define EC3_CHUNKS_PER_GROUP_64K 2340
/* each vnode group occupies one cluster, so the number of vnodes per group
* depends on the cluster size */
#define EC3_VNODES_PER_GROUP_4K 255
#define EC3_VNODES_PER_GROUP_8K 511
#define EC3_VNODES_PER_GROUP_16K 1023
#define EC3_VNODES_PER_GROUP_32K 2047
#define EC3_VNODES_PER_GROUP_64K 4095
/* each vnode/chunk link group occupies one cluster, so the number of links per
* group depends on the cluster size */
#define EC3_VNCH_PER_GROUP_4K 146
#define EC3_VNCH_PER_GROUP_8K 292
#define EC3_VNCH_PER_GROUP_16K 584
#define EC3_VNCH_PER_GROUP_32K 1170
#define EC3_VNCH_PER_GROUP_64K 2340
/* Chunks are identified by a 128-bit (16-byte) hash */
#define EC3_CHUNK_ID_SIZE 16
typedef uint8_t ec3_chunk_id[EC3_CHUNK_ID_SIZE];
struct ec3_header {
b_i32 h_sig;
b_i32 h_magic;
b_i16 h_version;
b_i16 h_cluster_size;
b_i64 h_tag_table_offset;
b_i64 h_tag_count;
b_i64 h_extent_table_offset;
b_i64 h_cluster_table_offset;
b_i32 h_tag_count;
b_i32 h_extent_count;
b_i32 h_cluster_group_count;
b_i16 h_compression;
b_i16 h_encryption;
b_i64 h_app_magic;
uint8_t h_reserved[8];
};
struct ec3_cluster {
/* cluster identifier */
b_i32 c_id;
/* lower 32-bits of the cluster bounds */
b_i32 c_bounds0;
/* upper 32-bits of the cluster bounds */
b_i32 c_bounds1;
/* CRC-16 of the on-disk cluster data */
b_i16 c_checksum;
/* flags that apply to this cluster */
b_i16 c_flags;
};
struct ec3_cluster_group {
/* the number of clusters that this group contains */
b_i16 g_nr_clusters;
uint8_t g_reserved[2];
/* array of clusters contained within the group. */
struct ec3_cluster g_clusters[EC3_CLUSTERS_PER_GROUP];
/* offsets to other cluster groups, relative to the start of the
* cluster group table. the cluster groups form a B-tree. */
b_i32 g_child_offsets[EC3_CLUSTERS_PER_GROUP + 1];
/* pad the group out to an even 32K */
uint8_t g_padding[20];
};
struct ec3_tag_table_entry {
@@ -44,9 +115,157 @@ struct ec3_tag_table_entry {
b_i32 tag_checksum;
b_i32 tag_reserved1;
b_i64 tag_ident;
b_i64 tag_offset;
b_i64 tag_size;
b_i64 tag_reserved2;
};
/* Extents serve two purposes:
* 1. They specify which clusters in an image are allocated to which tags.
* 2. They define the mapping between "logical" clusters and "physical"
* clusters.
*
* For example, logical cluster 0 of a certain tag (i.e. the cluster that
* contains the first 16K of the tag's data) might be mapped to physical
* cluster 17 (i.e. the cluster that has an id of 17).
*/
struct ec3_extent {
/* the id of the tag that this range of clusters belongs to */
b_i64 ex_owner;
/* the id of the first logical cluster that this extent represents. */
b_i32 ex_logical_cluster;
/* the id of the first physical cluster that this extent represents. */
b_i32 ex_physical_cluster;
/* the number of clusters included in this extent. */
b_i32 ex_count;
};
struct ec3_chunk {
ec3_chunk_id c_id;
b_i16 c_length;
b_i16 c_offset1;
b_i32 c_offset0;
};
struct ec3_chunk_group {
/* the number of chunks that this group contains */
b_i16 g_nr_chunks;
uint8_t g_reserved[2];
union {
struct {
struct ec3_chunk g_chunks[EC3_CHUNKS_PER_GROUP_4K];
b_i32 g_child_offsets[EC3_CHUNKS_PER_GROUP_4K + 1];
} g_4k;
#if 0
struct {
struct ec3_chunk g_chunks[EC3_CHUNKS_PER_GROUP_8K];
b_i32 g_child_offsets[EC3_CHUNKS_PER_GROUP_8K + 1];
} g_8k;
struct {
struct ec3_chunk g_chunks[EC3_CHUNKS_PER_GROUP_16K];
b_i32 g_child_offsets[EC3_CHUNKS_PER_GROUP_16K + 1];
} g_16k;
struct {
struct ec3_chunk g_chunks[EC3_CHUNKS_PER_GROUP_32K];
b_i32 g_child_offsets[EC3_CHUNKS_PER_GROUP_32K + 1];
} g_32k;
struct {
struct ec3_chunk g_chunks[EC3_CHUNKS_PER_GROUP_64K];
b_i32 g_child_offsets[EC3_CHUNKS_PER_GROUP_64K + 1];
} g_64k;
#endif
};
};
struct ec3_directory_entry {
b_i32 d_name;
b_i32 d_vnode;
};
struct ec3_vnode {
b_i32 n_id;
b_i32 n_attrib;
/* length of file in chunks */
b_i32 n_length;
};
struct ec3_vnode_group {
/* the number of vnodes that this group contains */
b_i16 g_nr_vnodes;
uint8_t g_reserved[2];
union {
struct {
struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_4K];
b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_4K + 1];
} g_4k;
#if 0
struct {
struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_8K];
b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_8K + 1];
} g_8k;
struct {
struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_16K];
b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_16K + 1];
} g_16k;
struct {
struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_32K];
b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_32K + 1];
} g_32k;
struct {
struct ec3_vnode g_vnodes[EC3_VNODES_PER_GROUP_64K];
b_i32 g_child_offsets[EC3_VNODES_PER_GROUP_64K + 1];
} g_64k;
#endif
};
};
struct ec3_vnode_chunk {
ec3_chunk_id c_chunk_id;
b_i32 c_vnode_id;
b_i32 c_vnode_index;
};
struct ec3_vnode_chunk_group {
/* the number of vnode/chunk links that this group contains */
b_i16 g_nr_entries;
uint8_t g_reserved[2];
union {
struct {
struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_4K];
b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_4K + 1];
} g_4k;
#if 0
struct {
struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_8K];
b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_8K + 1];
} g_8k;
struct {
struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_16K];
b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_16K + 1];
} g_16k;
struct {
struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_32K];
b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_32K + 1];
} g_32k;
struct {
struct ec3_vnode_chunk g_links[EC3_VNCH_PER_GROUP_64K];
b_i32 g_child_offsets[EC3_VNCH_PER_GROUP_64K + 1];
} g_64k;
#endif
};
};
#endif

24
src/compress.h Normal file
View File

@@ -0,0 +1,24 @@
#ifndef COMPRESS_H_
#define COMPRESS_H_
#include <stddef.h>
struct ec3_compression_function {
enum ec3_status (*e_compress)(
const void *in,
size_t in_size,
void *out,
size_t out_max,
size_t *out_size);
enum ec3_status (*e_decompress)(
const void *in,
size_t in_size,
void *out,
size_t out_size);
};
extern const struct ec3_compression_function *ec3_compression_function_get(
unsigned int type);
#endif

44
src/create.c
View File

@@ -1,9 +1,10 @@
#include "bin.h"
#include "commands.h"
#include <blue/cmd.h>
#include <zstd.h>
#include <stdio.h>
#include <stdlib.h>
#include <zstd.h>
enum {
ARG_INPATH,
@@ -18,7 +19,22 @@ static int create(
{
const char *in_path = NULL, *out_path = NULL;
b_arglist_get_string(opt, B_COMMAND_INVALID_ID, ARG_INPATH, 0, &in_path);
printf("cluster group = %zu\n", sizeof(struct ec3_cluster_group));
printf("chunk group = %zu\n", sizeof(struct ec3_chunk_group));
printf("vnode group = %zu\n", sizeof(struct ec3_vnode_group));
printf("vnch group = %zu\n", sizeof(struct ec3_vnode_chunk_group));
printf("vnode = %zu\n", sizeof(struct ec3_vnode));
printf("chunk = %zu\n", sizeof(struct ec3_chunk));
printf("link = %zu\n", sizeof(struct ec3_vnode_chunk));
return 0;
b_arglist_get_string(
opt,
B_COMMAND_INVALID_ID,
ARG_INPATH,
0,
&in_path);
b_arglist_get_string(opt, OPT_OUTPATH, OPT_OUTPATH_PATH, 0, &out_path);
printf("in path: %s\n", in_path);
@@ -61,17 +77,23 @@ static int create(
total += r;
bool last_chunk = r < in_bufsz;
ZSTD_EndDirective mode = last_chunk ? ZSTD_e_end : ZSTD_e_continue;
ZSTD_EndDirective mode
= last_chunk ? ZSTD_e_end : ZSTD_e_continue;
ZSTD_inBuffer input = { in_buf, r, 0 };
ZSTD_inBuffer input = {in_buf, r, 0};
int finished;
do {
ZSTD_outBuffer output = { out_buf, out_bufsz, 0 };
size_t remaining = ZSTD_compressStream2(zstd, &output, &input, mode);
ZSTD_outBuffer output = {out_buf, out_bufsz, 0};
size_t remaining = ZSTD_compressStream2(
zstd,
&output,
&input,
mode);
fwrite(out_buf, 1, output.pos, out);
finished = last_chunk ? (remaining == 0) : (input.pos == input.size);
finished = last_chunk ? (remaining == 0)
: (input.pos == input.size);
} while (!finished);
if (last_chunk) {
@@ -111,12 +133,14 @@ B_COMMAND(CMD_CREATE, CMD_ROOT)
}
}
B_COMMAND_ARG(ARG_INPATH) {
B_COMMAND_ARG(ARG_INPATH)
{
B_ARG_NAME("input file");
B_ARG_NR_VALUES(1);
}
B_COMMAND_USAGE() {
B_COMMAND_USAGE()
{
B_COMMAND_USAGE_ARG(ARG_INPATH);
B_COMMAND_USAGE_OPT(OPT_OUTPATH);
}

19
src/encrypt.h Normal file
View File

@@ -0,0 +1,19 @@
#ifndef ENCRYPT_H_
#define ENCRYPT_H_
#include <stddef.h>
struct ec3_encryption_function {
const size_t e_key_size;
const size_t e_block_size;
enum ec3_status (
*e_encrypt)(const void *in, void *out, const void *key);
enum ec3_status (
*e_decrypt)(const void *in, void *out, const void *key);
};
extern const struct ec3_encryption_function *ec3_encryption_function_get(
unsigned int type);
#endif

39
src/file.c Normal file
View File

@@ -0,0 +1,39 @@
#include "pipeline.h"
#include <stdio.h>
static enum ec3_status write(
struct ec3_pipeline_stage *stage,
const void *src,
size_t len,
void *dest,
size_t *nr_written)
{
FILE *fp = stage->s_arg;
size_t r = fwrite(src, 1, len, fp);
*nr_written = r;
if (r < len) {
return EC3_ERR_IO_FAILURE;
}
return EC3_SUCCESS;
}
static enum ec3_status read(
struct ec3_pipeline_stage *stage,
const void *src,
void *dest,
size_t *nr_read)
{
FILE *fp = stage->s_arg;
return EC3_SUCCESS;
}
const struct ec3_pipeline_stage_type pipeline_file = {
.t_id = EC3_PIPELINE_FILE,
.t_flags = EC3_PIPELINE_F_NONE,
.t_data_in = read,
.t_data_out = write,
};

156
src/pipeline.c Normal file
View File

@@ -0,0 +1,156 @@
#include "pipeline.h"
#include <stdlib.h>
#include <string.h>
extern const struct ec3_pipeline_stage_type pipeline_zstd;
extern const struct ec3_pipeline_stage_type pipeline_aes256;
extern const struct ec3_pipeline_stage_type pipeline_file;
static const struct ec3_pipeline_stage_type *stage_types[] = {
[EC3_PIPELINE_ZSTD] = &pipeline_zstd,
[EC3_PIPELINE_AES256] = &pipeline_aes256,
[EC3_PIPELINE_FILE] = &pipeline_file,
};
static const size_t nr_stage_types = sizeof stage_types / sizeof stage_types[0];
static enum ec3_status create_pipeline_stage(
const struct ec3_pipeline_stage_type *type,
size_t cluster_size,
void *arg,
struct ec3_pipeline_stage **out)
{
struct ec3_pipeline_stage *stage = malloc(sizeof *stage);
if (!stage) {
return EC3_ERR_NO_MEMORY;
}
memset(stage, 0x0, sizeof *stage);
stage->s_type = type;
stage->s_arg = arg;
if (type->t_flags & EC3_PIPELINE_F_BUFFERED) {
stage->s_buf = malloc(cluster_size);
if (!stage) {
free(stage);
return EC3_ERR_NO_MEMORY;
}
memset(stage->s_buf, 0x0, cluster_size);
}
*out = stage;
return EC3_SUCCESS;
}
extern enum ec3_status ec3_pipeline_create(
struct ec3_pipeline_stage_args stages[],
size_t nr_stages,
size_t cluster_size,
struct ec3_pipeline **out)
{
enum ec3_status status = EC3_SUCCESS;
struct ec3_pipeline *pipeline = malloc(sizeof *pipeline);
if (!pipeline) {
return EC3_ERR_NO_MEMORY;
}
memset(pipeline, 0x0, sizeof *pipeline);
for (size_t i = 0; i < nr_stages; i++) {
struct ec3_pipeline_stage_args *args = &stages[i];
if (args->type == EC3_PIPELINE_NONE) {
continue;
}
if (args->type < 0 || args->type >= nr_stage_types) {
return EC3_ERR_NOT_SUPPORTED;
}
const struct ec3_pipeline_stage_type *type
= stage_types[stages[i].type];
if (!type) {
return EC3_ERR_NOT_SUPPORTED;
}
struct ec3_pipeline_stage *stage = NULL;
status = create_pipeline_stage(
type,
cluster_size,
args->arg,
&stage);
if (status != EC3_SUCCESS) {
return status;
}
b_queue_push_back(&pipeline->p_stages, &stage->s_entry);
}
*out = pipeline;
return status;
}
void ec3_pipeline_destroy(struct ec3_pipeline *p)
{
}
enum ec3_status ec3_pipeline_data_out(
struct ec3_pipeline *pipeline,
void *p,
size_t len,
size_t *nr_written)
{
b_queue_entry *cur = b_queue_first(&pipeline->p_stages);
enum ec3_status status = EC3_SUCCESS;
void *src = p;
size_t stage_in_size = len;
size_t stage_out_size = 0;
while (cur) {
struct ec3_pipeline_stage *stage
= b_unbox(struct ec3_pipeline_stage, cur, s_entry);
void *dest;
if (stage->s_type->t_flags & EC3_PIPELINE_F_BUFFERED) {
dest = stage->s_buf;
} else {
dest = src;
}
status = stage->s_type->t_data_out(
stage,
src,
stage_in_size,
dest,
&stage_out_size);
if (status != EC3_SUCCESS) {
return status;
}
src = dest;
stage_in_size = stage_out_size;
cur = b_queue_next(cur);
}
if (nr_written) {
*nr_written = stage_out_size;
}
return EC3_SUCCESS;
}
enum ec3_status ec3_pipeline_data_in(
struct ec3_pipeline *pipeline,
void *p,
size_t max,
size_t *nr_read)
{
return EC3_ERR_NOT_SUPPORTED;
}

74
src/pipeline.h Normal file
View File

@@ -0,0 +1,74 @@
#ifndef PIPELINE_H_
#define PIPELINE_H_
#include "status.h"
#include <blue/core/queue.h>
#include <stddef.h>
enum ec3_pipeline_stage_type_id {
EC3_PIPELINE_NONE = 0,
EC3_PIPELINE_AES256,
EC3_PIPELINE_ZSTD,
EC3_PIPELINE_FILE,
};
enum ec3_pipeline_stage_type_flags {
EC3_PIPELINE_F_NONE = 0x00u,
EC3_PIPELINE_F_BUFFERED = 0x01u,
};
struct ec3_pipeline_stage;
struct ec3_pipeline_stage_type {
enum ec3_pipeline_stage_type_id t_id;
enum ec3_pipeline_stage_type_flags t_flags;
enum ec3_status (*t_data_in)(
struct ec3_pipeline_stage *,
const void *,
void *,
size_t *);
enum ec3_status (*t_data_out)(
struct ec3_pipeline_stage *,
const void *,
size_t,
void *,
size_t *);
};
struct ec3_pipeline_stage {
const struct ec3_pipeline_stage_type *s_type;
void *s_buf;
void *s_arg;
b_queue_entry s_entry;
};
struct ec3_pipeline_stage_args {
enum ec3_pipeline_stage_type_id type;
void *arg;
};
struct ec3_pipeline {
b_queue p_stages;
};
extern enum ec3_status ec3_pipeline_create(
struct ec3_pipeline_stage_args stages[],
size_t nr_stages,
size_t cluster_size,
struct ec3_pipeline **out);
extern void ec3_pipeline_destroy(struct ec3_pipeline *p);
extern enum ec3_status ec3_pipeline_data_out(
struct ec3_pipeline *pipeline,
void *p,
size_t len,
size_t *nr_written);
extern enum ec3_status ec3_pipeline_data_in(
struct ec3_pipeline *pipeline,
void *p,
size_t max,
size_t *nr_read);
#endif

3
src/status.h
View File

@@ -7,6 +7,9 @@ enum ec3_status {
EC3_ERR_NO_ENTRY,
EC3_ERR_NOT_SUPPORTED,
EC3_ERR_BAD_STATE,
EC3_ERR_INVALID_VALUE,
EC3_ERR_NAME_EXISTS,
EC3_ERR_IO_FAILURE,
};
#endif

4
src/tag.h Normal file
View File

@@ -0,0 +1,4 @@
#ifndef TAG_H_
#define TAG_H_
#endif

69
src/wrap.c
View File

@@ -1,14 +1,16 @@
#include "bin.h"
#include "commands.h"
#include "write.h"
#include <blue/cmd.h>
#include <stdlib.h>
#include <blue/term.h>
#include <errno.h>
enum {
OPT_OUTPATH,
OPT_OUTPATH_PATH,
ARG_FILE,
ARG_FILE_PATH,
OPT_TAGGED_FILE,
OPT_TAGGED_FILE_TAG,
@@ -20,6 +22,65 @@ static int wrap(
const b_arglist *opt,
const b_array *args)
{
const char *in_path = NULL, *out_path = NULL;
b_arglist_get_string(opt, B_COMMAND_INVALID_ID, ARG_FILE, 0, &in_path);
b_arglist_get_string(opt, OPT_OUTPATH, OPT_OUTPATH_PATH, 0, &out_path);
printf("in path: %s\n", in_path);
printf("out path: %s\n", out_path);
FILE *inp = fopen(in_path, "rb");
if (!inp) {
b_err("cannot open '%s'", in_path);
b_i("reason: %s", strerror(errno));
return -1;
}
FILE *outp = fopen(out_path, "wb");
if (!outp) {
b_err("cannot open '%s'", out_path);
b_i("reason: %s", strerror(errno));
return -1;
}
struct ec3_writer *writer = NULL;
struct ec3_parameters param = {
.p_outp = outp,
.p_cluster_size = EC3_CLUSTER_16K,
.p_compression_func = EC3_COMPRESSION_ZSTD,
};
enum ec3_status status = ec3_writer_create(&param, &writer);
if (status != EC3_SUCCESS) {
b_err("cannot initialise EC3 writer");
return -1;
}
struct ec3_tag_writer *tag = NULL;
status = ec3_writer_create_tag(writer, EC3_TAG_BLOB, 0, 0, &tag);
if (status != EC3_SUCCESS) {
b_err("cannot initialise EC3 tag writer");
return -1;
}
char buf[4096];
while (1) {
size_t r = fread(buf, 1, sizeof buf, inp);
status = ec3_tag_writer_write(tag, buf, r);
if (r < sizeof buf) {
break;
}
}
ec3_tag_writer_finish(tag);
ec3_writer_finish(writer);
fclose(inp);
fclose(outp);
return 0;
}
@@ -29,8 +90,8 @@ B_COMMAND(CMD_WRAP, CMD_ROOT)
B_COMMAND_SHORT_NAME('W');
B_COMMAND_DESC(
"wrap one or more files into an ec3 container. each file will "
"be "
"stored in a separate blob tag within the created container.");
"be stored in a separate blob tag within the created "
"container.");
B_COMMAND_FLAGS(B_COMMAND_SHOW_HELP_BY_DEFAULT);
B_COMMAND_FUNCTION(wrap);

368
src/write.c Normal file
View File

@@ -0,0 +1,368 @@
#include "write.h"
#include "bin.h"
#include "pipeline.h"
#include <blue/core/queue.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
struct ec3_writer {
b_queue w_tag_writers;
unsigned long w_next_tag_ident;
struct ec3_parameters w_param;
uint64_t w_extent_tag;
size_t w_extent_logical_start;
size_t w_extent_physical_start;
size_t w_extent_nr_clusters;
size_t w_data_offset;
FILE *w_data;
FILE *w_cluster_table;
FILE *w_extent_table;
FILE *w_tag_table;
struct ec3_pipeline *w_pipeline;
};
struct ec3_tag_writer {
struct ec3_writer *w_parent;
unsigned long w_type;
uint64_t w_ident;
unsigned long w_flags;
unsigned char *w_buf;
size_t w_ptr;
size_t w_nr_clusters;
b_queue_entry w_entry;
};
static const size_t cluster_sizes[] = {
[EC3_CLUSTER_4K] = 0x1000,
[EC3_CLUSTER_8K] = 0x2000,
[EC3_CLUSTER_16K] = 0x4000,
[EC3_CLUSTER_32K] = 0x8000,
[EC3_CLUSTER_64K] = 0x10000,
};
static enum ec3_pipeline_stage_type_id pipeline_stage_for_compression_func(
unsigned int func)
{
switch (func) {
case EC3_COMPRESSION_ZSTD:
return EC3_PIPELINE_ZSTD;
default:
return EC3_PIPELINE_NONE;
}
}
static enum ec3_pipeline_stage_type_id pipeline_stage_for_encryption_func(
unsigned int func)
{
switch (func) {
case EC3_ENCRYPTION_AES256:
return EC3_PIPELINE_AES256;
default:
return EC3_PIPELINE_NONE;
}
}
enum ec3_status ec3_writer_create(
const struct ec3_parameters *param,
struct ec3_writer **out)
{
struct ec3_writer *writer = malloc(sizeof *writer);
if (!writer) {
return EC3_ERR_NO_MEMORY;
}
memset(writer, 0x0, sizeof *writer);
memcpy(&writer->w_param, param, sizeof *param);
size_t cluster_size = cluster_sizes[param->p_cluster_size];
writer->w_data = param->p_outp;
writer->w_cluster_table = tmpfile();
writer->w_extent_table = tmpfile();
writer->w_tag_table = tmpfile();
struct ec3_pipeline_stage_args stages[3] = {0};
if (param->p_compression_func != EC3_COMPRESSION_NONE) {
stages[0].type = pipeline_stage_for_compression_func(
param->p_compression_func);
}
if (param->p_encryption_func != EC3_ENCRYPTION_NONE) {
stages[1].type = pipeline_stage_for_encryption_func(
param->p_encryption_func);
}
stages[2].type = EC3_PIPELINE_FILE;
stages[2].arg = writer->w_data;
struct ec3_pipeline *pipeline = NULL;
enum ec3_status status = ec3_pipeline_create(
stages,
sizeof stages / sizeof stages[0],
cluster_size,
&pipeline);
if (status != EC3_SUCCESS) {
return status;
}
writer->w_pipeline = pipeline;
struct ec3_header header = {0};
size_t written = fwrite(&header, sizeof header, 1, writer->w_data);
if (written != 1) {
return EC3_ERR_IO_FAILURE;
}
*out = writer;
return EC3_SUCCESS;
}
static enum ec3_status copy_file(FILE *src, FILE *dest)
{
const size_t buf_len = 16384;
char *buf = malloc(buf_len);
if (!buf) {
return EC3_ERR_IO_FAILURE;
}
fseek(src, 0x0, SEEK_SET);
enum ec3_status status = EC3_SUCCESS;
while (1) {
size_t r = fread(buf, 1, buf_len, src);
if (ferror(src)) {
status = EC3_ERR_IO_FAILURE;
break;
}
size_t w = fwrite(buf, 1, r, dest);
if (w != r) {
status = EC3_ERR_IO_FAILURE;
break;
}
if (r < buf_len) {
break;
}
}
free(buf);
return status;
}
static enum ec3_status flush_extent_entry(struct ec3_writer *w)
{
struct ec3_extent extent = {0};
extent.ex_owner = b_i64_htob(w->w_extent_tag);
extent.ex_physical_cluster = b_i32_htob(w->w_extent_physical_start);
extent.ex_logical_cluster = b_i32_htob(w->w_extent_logical_start);
extent.ex_count = b_i32_htob(w->w_extent_nr_clusters);
size_t written = fwrite(&extent, sizeof extent, 1, w->w_extent_table);
if (written != 1) {
return EC3_ERR_IO_FAILURE;
}
return EC3_SUCCESS;
}
void ec3_writer_finish(struct ec3_writer *w)
{
enum ec3_status status = EC3_SUCCESS;
if (w->w_extent_nr_clusters > 0) {
status = flush_extent_entry(w);
}
if (status != EC3_SUCCESS) {
return;
}
size_t cluster_table_offset = ftell(w->w_data);
status = copy_file(w->w_cluster_table, w->w_data);
size_t extent_table_offset = ftell(w->w_data);
status = copy_file(w->w_extent_table, w->w_data);
size_t tag_table_offset = ftell(w->w_data);
status = copy_file(w->w_tag_table, w->w_data);
fseek(w->w_data, 0x0, SEEK_SET);
struct ec3_header header = {0};
header.h_magic = b_i32_htob(EC3_SIGNATURE);
header.h_version = b_i16_htob(EC3_VERSION_1_0);
header.h_cluster_size = b_i16_htob(w->w_param.p_cluster_size);
header.h_tag_table_offset = b_i64_htob(tag_table_offset);
header.h_extent_table_offset = b_i64_htob(extent_table_offset);
header.h_cluster_table_offset = b_i64_htob(cluster_table_offset);
fwrite(&header, sizeof header, 1, w->w_data);
}
void ec3_writer_write_image(struct ec3_writer *w, FILE *fp)
{
}
static bool is_tag_ident_free(struct ec3_writer *w, uint64_t ident)
{
b_queue_iterator it = {0};
b_queue_foreach(&it, &w->w_tag_writers)
{
struct ec3_tag_writer *tag
= b_unbox(struct ec3_tag_writer, it.entry, w_entry);
if (tag->w_ident == ident) {
return false;
}
}
return true;
}
enum ec3_status ec3_writer_create_tag(
struct ec3_writer *w,
uint32_t tag_type,
uint64_t tag_ident,
unsigned int flags,
struct ec3_tag_writer **out_writer)
{
struct ec3_tag_writer *tag = malloc(sizeof *tag);
if (!tag) {
return EC3_ERR_NO_MEMORY;
}
memset(tag, 0x0, sizeof *tag);
size_t cluster_size = cluster_sizes[w->w_param.p_cluster_size];
tag->w_parent = w;
tag->w_flags = flags;
tag->w_type = tag_type;
tag->w_ident = tag_ident;
tag->w_buf = malloc(cluster_size);
if (!tag->w_buf) {
free(tag);
return EC3_ERR_NO_MEMORY;
}
struct ec3_tag_table_entry entry = {0};
entry.tag_type = b_i32_htob(tag_type);
entry.tag_ident = b_i64_htob(tag_ident);
entry.tag_flags = b_i32_htob(flags);
size_t written = fwrite(&entry, sizeof entry, 1, w->w_tag_table);
if (written != 1) {
free(tag->w_buf);
free(tag);
return EC3_ERR_IO_FAILURE;
}
*out_writer = tag;
return EC3_SUCCESS;
}
static enum ec3_status flush_tag_buffer(struct ec3_tag_writer *w)
{
struct ec3_writer *container = w->w_parent;
unsigned char *buf = w->w_buf;
enum ec3_status status = EC3_SUCCESS;
size_t nr_written = 0;
status = ec3_pipeline_data_out(
container->w_pipeline,
buf,
w->w_ptr,
&nr_written);
if (status != EC3_SUCCESS) {
return status;
}
container->w_data_offset += nr_written;
w->w_ptr = 0;
w->w_nr_clusters++;
if (container->w_extent_tag == w->w_ident) {
container->w_extent_nr_clusters++;
return EC3_SUCCESS;
}
if (container->w_extent_nr_clusters) {
status = flush_extent_entry(container);
}
if (status != EC3_SUCCESS) {
return status;
}
container->w_extent_tag = w->w_ident;
container->w_extent_physical_start += container->w_extent_nr_clusters;
container->w_extent_logical_start = w->w_nr_clusters;
container->w_extent_nr_clusters = 0;
return EC3_SUCCESS;
}
enum ec3_status ec3_tag_writer_write(
struct ec3_tag_writer *w,
const void *p,
size_t len)
{
size_t cluster_size
= cluster_sizes[w->w_parent->w_param.p_cluster_size];
enum ec3_status status = EC3_SUCCESS;
void *buf = w->w_buf;
while (len > 0) {
size_t remaining = cluster_size - w->w_ptr;
size_t to_write = len;
if (len > remaining) {
to_write = remaining;
}
memcpy(buf + w->w_ptr, p, to_write);
remaining -= to_write;
w->w_ptr += to_write;
if (remaining == 0) {
status = flush_tag_buffer(w);
}
if (status != EC3_SUCCESS) {
return status;
}
len -= to_write;
}
return status;
}
enum ec3_status ec3_tag_writer_finish(struct ec3_tag_writer *w)
{
enum ec3_status status = EC3_SUCCESS;
if (w->w_ptr > 0) {
status = flush_tag_buffer(w);
}
if (status != EC3_SUCCESS) {
return status;
}
return status;
}

26
src/write.h
View File

@@ -6,17 +6,39 @@
#include <stdint.h>
#include <stdio.h>
struct ec3_parameters {
FILE *p_outp;
unsigned int p_cluster_size;
unsigned int p_compression_func;
unsigned int p_encryption_func;
uint64_t p_ident;
const void *p_encryption_key;
size_t p_encryption_key_size;
};
struct ec3_writer;
struct ec3_tag_writer;
extern enum ec3_status ec3_writer_create(void);
extern void ec3_writer_destroy(struct ec3_writer *w);
extern enum ec3_status ec3_writer_create(
const struct ec3_parameters *param,
struct ec3_writer **out);
extern void ec3_writer_finish(struct ec3_writer *w);
extern void ec3_writer_write_image(struct ec3_writer *w, FILE *fp);
extern enum ec3_status ec3_writer_create_tag(
struct ec3_writer *w,
uint32_t tag_type,
uint64_t tag_ident,
unsigned int flags,
struct ec3_tag_writer **out_writer);
extern enum ec3_status ec3_tag_writer_write(
struct ec3_tag_writer *w,
const void *p,
size_t len);
extern enum ec3_status ec3_tag_writer_finish(struct ec3_tag_writer *w);
#endif

30
src/zstd.c Normal file
View File

@@ -0,0 +1,30 @@
#include "pipeline.h"
#include <zstd.h>
static enum ec3_status compress(
struct ec3_pipeline_stage *stage,
const void *src,
size_t len,
void *dest,
size_t *nr_written)
{
*nr_written = ZSTD_compress(dest, len, src, len, 10);
return EC3_SUCCESS;
}
static enum ec3_status decompress(
struct ec3_pipeline_stage *stage,
const void *src,
void *dest,
size_t *nr_read)
{
return EC3_SUCCESS;
}
const struct ec3_pipeline_stage_type pipeline_zstd = {
.t_id = EC3_PIPELINE_ZSTD,
.t_flags = EC3_PIPELINE_F_BUFFERED,
.t_data_in = decompress,
.t_data_out = compress,
};