/* Copyright (c) 2024, Dominic Szablewski - https://phoboslab.org SPDX-License-Identifier: MIT Command line tool to create and unpack qop archives */ #define _DEFAULT_SOURCE #include #include #include #include #include #define QOP_IMPLEMENTATION #include "qop.h" #define MAX_PATH_LEN 1024 #define BUFFER_SIZE 4096 #define UNUSED(x) (void)(x) #define STRINGIFY(x) #x #define TOSTRING(x) STRINGIFY(x) #define die(...) \ printf("Abort at " TOSTRING(__FILE__) " line " TOSTRING(__LINE__) ": " __VA_ARGS__); \ printf("\n"); \ exit(1) #define error_if(TEST, ...) \ if (TEST) { \ die(__VA_ARGS__); \ } // ----------------------------------------------------------------------------- // Platform specific file/dir handling typedef struct { char *name; unsigned char is_dir; unsigned char is_file; } pi_dirent; #if defined(_WIN32) #include typedef struct { WIN32_FIND_DATA data; pi_dirent current; HANDLE dir; unsigned char is_first; } pi_dir; pi_dir *pi_dir_open(const char *path) { char find_str[MAX_PATH_LEN]; snprintf(find_str, MAX_PATH_LEN, "%s/*", path); pi_dir *d = malloc(sizeof(pi_dir)); d->is_first = 1; d->dir = FindFirstFile(find_str, &d->data); if (d->dir == INVALID_HANDLE_VALUE) { free(d); return NULL; } return d; } pi_dirent *pi_dir_next(pi_dir *d) { if (!d->is_first) { if (FindNextFile(d->dir, &d->data) == 0) { return NULL; } } d->is_first = 0; d->current.name = d->data.cFileName; d->current.is_dir = d->data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY; d->current.is_file = !d->current.is_dir; return &d->current; } void pi_dir_close(pi_dir *d) { FindClose(d->dir); free(d); } int pi_mkdir(char *path, int mode) { UNUSED(mode); return CreateDirectory(path, NULL) ? 0 : -1; } #else #include typedef struct { DIR *dir; struct dirent *data; pi_dirent current; } pi_dir; pi_dir *pi_dir_open(const char *path) { DIR *dir = opendir(path); if (!dir) { return NULL; } pi_dir *d = malloc(sizeof(pi_dir)); d->dir = dir; return d; } pi_dirent *pi_dir_next(pi_dir *d) { d->data = readdir(d->dir); if (!d->data) { return NULL; } d->current.name = d->data->d_name; d->current.is_dir = d->data->d_type & DT_DIR; d->current.is_file = d->data->d_type == DT_REG; return &d->current; } void pi_dir_close(pi_dir *d) { closedir(d->dir); free(d); } int pi_mkdir(char *path, int mode) { return mkdir(path, mode); } #endif // ----------------------------------------------------------------------------- // Unpack int create_path(const char *path, const mode_t mode) { char tmp[MAX_PATH_LEN]; char *p = NULL; struct stat sb; size_t len; // copy path len = strnlen(path, MAX_PATH_LEN); if (len == 0 || len == MAX_PATH_LEN) { return -1; } memcpy(tmp, path, len); tmp[len] = '\0'; // remove file part char *last_slash = strrchr(tmp, '/'); if (last_slash == NULL) { return 0; } *last_slash = '\0'; // check if path exists and is a directory if (stat(tmp, &sb) == 0) { if (S_ISDIR(sb.st_mode)) { return 0; } } // recursive mkdir for (p = tmp + 1; *p; p++) { if (*p == '/') { *p = 0; if (stat(tmp, &sb) != 0) { if (pi_mkdir(tmp, mode) < 0) { return -1; } } else if (!S_ISDIR(sb.st_mode)) { return -1; } *p = '/'; } } if (stat(tmp, &sb) != 0) { if (pi_mkdir(tmp, mode) < 0) { return -1; } } else if (!S_ISDIR(sb.st_mode)) { return -1; } return 0; } unsigned int copy_out(FILE *src, unsigned int offset, unsigned int size, const char *dest_path) { FILE *dest = fopen(dest_path, "wb"); error_if(!dest, "Could not open file %s for writing", dest_path); char buffer[BUFFER_SIZE]; size_t bytes_read, bytes_written; unsigned int bytes_total = 0; unsigned int read_size = size < BUFFER_SIZE ? size : BUFFER_SIZE; fseek(src, offset, SEEK_SET); while (read_size > 0 && (bytes_read = fread(buffer, 1, read_size, src)) > 0) { bytes_written = fwrite(buffer, 1, bytes_read, dest); error_if(bytes_written != bytes_read, "Write error"); bytes_total += bytes_written; if (bytes_total >= size) { break; } if (size - bytes_total < read_size) { read_size = size - bytes_total; } } error_if(ferror(src), "read error for file %s", dest_path); fclose(dest); return bytes_total; } void unpack(const char *archive_path, int list_only) { qop_desc qop; int archive_size = qop_open(archive_path, &qop); error_if(archive_size == 0, "Could not open archive %s", archive_path); // Read the archive index int index_len = qop_read_index(&qop, malloc(qop.hashmap_size)); error_if(index_len == 0, "Could not read index from archive %s", archive_path); // Extract all files for (unsigned int i = 0; i < qop.hashmap_len; i++) { qop_file *file = &qop.hashmap[i]; if (file->size == 0) { continue; } error_if(file->path_len >= MAX_PATH_LEN, "Path for file %016llx exceeds %d", file->hash, MAX_PATH_LEN); char path[MAX_PATH_LEN]; qop_read_path(&qop, file, path); // Integrity check // error_if(!qop_find(&qop, path), "could not find %s", path); printf("%6d %016llx %10d %s\n", i, file->hash, file->size, path); if (!list_only) { error_if(create_path(path, 0755) != 0, "Could not create path %s", path); copy_out(qop.fh, qop.files_offset + file->offset + file->path_len, file->size, path); } } free(qop.hashmap); qop_close(&qop); } // ----------------------------------------------------------------------------- // Pack typedef struct { qop_file *files; int len; int capacity; int size; } pack_state; void write_16(unsigned int v, FILE *fh) { unsigned char b[sizeof(unsigned short)]; b[0] = 0xff & (v ); b[1] = 0xff & (v >> 8); int written = fwrite(b, sizeof(unsigned short), 1, fh); error_if(!written, "Write error"); } void write_32(unsigned int v, FILE *fh) { unsigned char b[sizeof(unsigned int)]; b[0] = 0xff & (v ); b[1] = 0xff & (v >> 8); b[2] = 0xff & (v >> 16); b[3] = 0xff & (v >> 24); int written = fwrite(b, sizeof(unsigned int), 1, fh); error_if(!written, "Write error"); } void write_64(qop_uint64_t v, FILE *fh) { unsigned char b[sizeof(qop_uint64_t)]; b[0] = 0xff & (v ); b[1] = 0xff & (v >> 8); b[2] = 0xff & (v >> 16); b[3] = 0xff & (v >> 24); b[4] = 0xff & (v >> 32); b[5] = 0xff & (v >> 40); b[6] = 0xff & (v >> 48); b[7] = 0xff & (v >> 56); int written = fwrite(b, sizeof(qop_uint64_t), 1, fh); error_if(!written, "Write error"); } unsigned int copy_into(const char *src_path, FILE *dest) { FILE *src = fopen(src_path, "rb"); error_if(!src, "Could not open file %s for reading", src_path); char buffer[BUFFER_SIZE]; size_t bytes_read, bytes_written; unsigned int bytes_total = 0; while ((bytes_read = fread(buffer, 1, BUFFER_SIZE, src)) > 0) { bytes_written = fwrite(buffer, 1, bytes_read, dest); error_if(bytes_written != bytes_read, "Write error"); bytes_total += bytes_written; } error_if(ferror(src), "read error for file %s", src_path); fclose(src); return bytes_total; } void add_file(const char *path, FILE *dest, pack_state *state) { if (state->len >= state->capacity) { state->capacity *= 2; state->files = realloc(state->files, state->capacity * sizeof(qop_file)); } qop_uint64_t hash = qop_hash(path); // Write the path into the archive int path_len = strlen(path) + 1; int path_written = fwrite(path, sizeof(char), path_len, dest); error_if(path_written != path_len, "Write error"); // Copy the file into the archive unsigned int size = copy_into(path, dest); printf("%6d %016llx %10d %s\n", state->len, hash, size, path); // Collect file info for the index state->files[state->len] = (qop_file){ .hash = hash, .offset = state->size, .size = size, .path_len = path_len, .flags = QOP_FLAG_NONE }; state->size += size + path_len; state->len++; } void add_dir(const char *path, FILE *dest, pack_state *state) { pi_dir *dir = pi_dir_open(path); error_if(!dir, "Could not open directory %s for reading", path); pi_dirent *entry; while ((entry = pi_dir_next(dir))) { if ( entry->is_dir && strcmp(entry->name, ".") != 0 && strcmp(entry->name, "..") != 0 ) { char subpath[MAX_PATH_LEN]; snprintf(subpath, MAX_PATH_LEN, "%s/%s", path, entry->name); add_dir(subpath, dest, state); } else if (entry->is_file) { char subpath[MAX_PATH_LEN]; snprintf(subpath, MAX_PATH_LEN, "%s/%s", path, entry->name); add_file(subpath, dest, state); } } pi_dir_close(dir); } void pack(const char *read_dir, char **sources, int sources_len, const char *archive_path) { FILE *dest = fopen(archive_path, "wb"); error_if(!dest, "Could not open file %s for writing", archive_path); pack_state state = { .files = malloc(sizeof(qop_file) * 1024), .len = 0, .capacity = 1024, .size = 0 }; if (read_dir) { error_if(chdir(read_dir) != 0, "Could not change to directory %s", read_dir); } // Add files/directories for (int i = 0; i < sources_len; i++) { struct stat s; error_if(stat(sources[i], &s) != 0, "Could not stat file %s", sources[i]); if (S_ISDIR(s.st_mode)) { add_dir(sources[i], dest, &state); } else if (S_ISREG(s.st_mode)) { add_file(sources[i], dest, &state); } else { die("Path %s is neither a directory nor a regular file", sources[i]); } } // Write index and header unsigned int total_size = state.size + QOP_HEADER_SIZE; for (int i = 0; i < state.len; i++) { write_64(state.files[i].hash, dest); write_32(state.files[i].offset, dest); write_32(state.files[i].size, dest); write_16(state.files[i].path_len, dest); write_16(state.files[i].flags, dest); total_size += 20; } write_32(state.len, dest); write_32(total_size, dest); write_32(QOP_MAGIC, dest); free(state.files); fclose(dest); printf("files: %d, size: %d bytes\n", state.len, total_size); } void exit_usage(void) { puts( "Usage: qopconv [OPTION...] FILE...\n" "\n" "Examples:\n" " qopconv dir1 archive.qop # Create archive.qop from dir1/\n" " qopconv foo bar archive.qop # Create archive.qop from files foo and bar\n" " qoponvv -u archive.qop # Unpack archive.qop in current directory\n" " qopconv -l archive.qop # List files in archive.qop\n" " qopconv -d dir1 dir2 archive.qop # Use dir1 prefix for reading, create\n" " archive.qop from files in dir1/dir2/\n" "\n" "Options (mutually exclusive):\n" " -u ... unpack archive\n" " -l ... list contents of archive\n" " -d ....... change read dir when creating archives\n" ); exit(1); } int main(int argc, char **argv) { if (argc < 3) { exit_usage(); } // Unpack if (strcmp(argv[1], "-u") == 0) { unpack(argv[2], 0); } else if (strcmp(argv[1], "-l") == 0) { unpack(argv[2], 1); } else { int files_start = 1; char *read_dir = NULL; if (strcmp(argv[1], "-d") == 0) { read_dir = argv[2]; files_start = 3; } if (argc < 2 + files_start) { exit_usage(); } pack(read_dir, argv + files_start, argc - 1 - files_start, argv[argc-1]); } return 0; }