From e760535dba215681cef7a3a4d844366f87b27b55 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 16 Sep 2021 17:34:46 +0200 Subject: [PATCH 1/2] lz4: Start of implementation. --- core/compress/common.odin | 25 ++- core/compress/lz4/frame.odin | 346 +++++++++++++++++++++++++++++++++++ core/compress/lz4/lz4.odin | 75 ++++++++ 3 files changed, 444 insertions(+), 2 deletions(-) create mode 100644 core/compress/lz4/frame.odin create mode 100644 core/compress/lz4/lz4.odin diff --git a/core/compress/common.odin b/core/compress/common.odin index 683679566..445da29db 100644 --- a/core/compress/common.odin +++ b/core/compress/common.odin @@ -56,9 +56,11 @@ Error :: union { as it may involve an I/O error, a Deflate error, etc. */ image.Error, + io.Error, } General_Error :: enum { + None = 0, File_Not_Found, Cannot_Open_File, File_Too_Short, @@ -69,7 +71,6 @@ General_Error :: enum { Incompatible_Options, Unimplemented, - /* Memory errors */ @@ -98,7 +99,28 @@ GZIP_Error :: enum { You can tweak this setting using `-define:COMPRESS_OUTPUT_ALLOCATE_MAX=size_in_bytes` */ Output_Exceeds_COMPRESS_OUTPUT_ALLOCATE_MAX, +} +LZ4_Error :: enum { + Generic, + Max_Block_Size_Invalid, + Block_Mode_Invalid, + Content_Checksum_Invalid, + Compression_Level_Invalid, + Header_Version_Wrong, + Block_Checksum_Invalid, + Reserved_Flag_Set, + Allocation_Failed, + Source_Size_Too_Large, + Output_Buffer_Too_Small, + Frame_Header_Incomplete, + Frame_Type_Unknown, + Frame_Type_Wrong, + Frame_Size_Wrong, + Source_Input_Wrong, + Decompression_Failed, + Header_Checksum_Invalid, + Frame_Decoding_Already_Started, } ZIP_Error :: enum { @@ -125,7 +147,6 @@ Deflate_Error :: enum { BType_3, } - // General I/O context for ZLIB, LZW, etc. Context_Memory_Input :: struct #packed { input_data: []u8, diff --git a/core/compress/lz4/frame.odin b/core/compress/lz4/frame.odin new file mode 100644 index 000000000..1b5907906 --- /dev/null +++ b/core/compress/lz4/frame.odin @@ -0,0 +1,346 @@ +package lz4 + +/* + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + List of contributors: + Jeroen van Rijn: Initial implementation, optimization. + + Thanks to Yann Collet for creating the LZ4 compression format. + This implementation is written in accordance with the + (LZ4 block and frame specifications)[https://github.com/lz4/lz4/tree/dev/doc] +*/ + +import "core:hash/xxhash" + +Magic :: enum u32le { + Frame_Magic_Number = 0x184D2204, + Frame_Magic_Skippable_Start = 0x184D2A50, + Frame_Magic_Skippable_Max = 0x184D2A5F, + Frame_Magic_Skippable_Mask = 0xFFFFFFF0, + Block_Uncompressed_Flag = 0x80000000, +} +Block_Size_ID_Default :: Frame_Block_Size_ID.max_64_kiB + +/* + LZ4_Error :: enum { + Generic, + Max_Block_Size_Invalid, + Block_Mode_Invalid, + Content_Checksum_Invalid, + Compression_Level_Invalid, + Header_Version_Wrong, + Block_Checksum_Invalid, + Reserved_Flag_Set, + Allocation_Failed, + Source_Size_Too_Large, + Output_Buffer_Too_Small, + Frame_Header_Incomplete, + Frame_Type_Unknown, + Frame_Type_Wrong, + Frame_Size_Wrong, + Source_Input_Wrong, + Decompression_Failed, + Header_Checksum_Invalid, + Frame_Decoding_Already_Started, + } +*/ + +/*-************************************ +* Structures and local types +**************************************/ + +MEMORY_USAGE :: 14 +HASH_TABLE_SIZE :: 1 << MEMORY_USAGE +STREAM_SIZE :: (1 << MEMORY_USAGE) + 32 /* static size, for inter-version compatibility */ +STREAM_SIZE_RAWPTR :: STREAM_SIZE / size_of(rawptr) +HASH_LOG :: MEMORY_USAGE - 2 +HASH_SIZE_U32 :: 1 << HASH_LOG + +HC_DICTIONARY_LOG_SIZE :: 16 +HC_MAX_D :: 1 << HC_DICTIONARY_LOG_SIZE +HC_MAX_D_MASK :: HC_MAX_D - 1 +HC_HASH_LOG :: 15 +HC_HASH_TABLE_SIZE :: 1 << HC_HASH_LOG +HC_HASH_MASK :: HC_HASH_TABLE_SIZE - 1 + +LZ4_stream_t_internal :: struct { + hash_table: [HASH_SIZE_U32]u32, + current_offset: u32, + table_type: u32, + dictionary: []u8, // const LZ4_byte* dictionary; + dictionary_context: ^LZ4_stream_t_internal, + dictionary_size: u32, +} + +LZ4_streamDecode_t_internal :: struct { + external_dict: []u8, + external_dict_size: uint, + prefix_end: []u8, + prefix_size: uint, +} + +/* + LZ4_stream_t : + * Do not use below internal definitions directly ! + * Declare or allocate an LZ4_stream_t instead. + * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. + * The structure definition can be convenient for static allocation + * (on stack, or as part of larger structure). + * Init this structure with LZ4_initStream() before first use. + * note : only use this definition in association with static linking ! + * this definition is not API/ABI safe, and may change in future versions. + */ +LZ4_Stream :: struct #raw_union { + table: [STREAM_SIZE_RAWPTR]rawptr, + internal_donotuse: LZ4_stream_t_internal, +} + +/* + LZ4_streamDecode_t: information structure to track an LZ4 stream during decompression. + init this structure using LZ4_setStreamDecode() before first use. + note : only use in association with static linking ! + this definition is not API/ABI safe, + and may change in a future version ! +*/ +when size_of(rawptr) == 16 { + LZ4_STREAM_DECODE_SIZE_U64 :: 4 + 2 // AS-400 +} else { + LZ4_STREAM_DECODE_SIZE_U64 :: 4 + 0 +} + +LZ4_STREAM_DECODE_SIZE :: (LZ4_STREAM_DECODE_SIZE_U64 * size_of(u64)) + +LZ4_stream_decode :: struct #raw_union { + table: [LZ4_STREAM_DECODE_SIZE_U64]u64, + internal_donotuse: LZ4_streamDecode_t_internal, +} /* previously typedef'd to LZ4_streamDecode_t */ + + +HC_CCtx_internal :: struct { + hash_table: [HC_HASH_TABLE_SIZE]u32, + chain_table: [HC_MAX_D]u16, + end: []u8, /* next block here to continue on current prefix */ + base: []u8, /* All index relative to this position */ + dict_base: []u8, /* alternate base for extDict */ + dict_limit: u32, /* below that point, need extDict */ + low_limit: u32, /* below that point, no more dict */ + next_to_update: u32, /* index from which to continue dictionary update */ + compression_level: u16, + favor_decompression_speed: i8, /* favor decompression speed if this flag set, otherwise, favor compression ratio */ + dirty: i8, /* stream has to be fully reset if this flag is set */ + dict_context: ^HC_CCtx_internal, +} + +/* + Do not use these definitions directly ! + Declare or allocate an LZ4_streamHC_t instead. +*/ +STREAM_HC_SIZE :: 262200 /* static size, for inter-version compatibility */ +STREAM_HC_SIZE_RAWPTR :: STREAM_HC_SIZE / size_of(rawptr) + + +LZ4_Stream_HC :: struct #raw_union { + table: [STREAM_HC_SIZE_RAWPTR]rawptr, + internal_donotuse: HC_CCtx_internal, +} /* previously typedef'd to LZ4_streamHC_t */ + +LZ4_Compression_Dictionary :: struct { + dictionary_content: []u8, + fast_context: ^LZ4_Stream, + high_compression_context: ^LZ4_Stream_HC, +} + +LZ4_Compression_Context :: struct { + prefs: Preferences, + version: u32, + c_stage: u32, + c_dict: ^LZ4_Compression_Dictionary, + max_block_size: uint, + max_buffer_size: uint, + temp_buffer: []u8, /* internal buffer, for streaming */ + temp_in: []u8, /* starting position of data compress within internal buffer (>= tmpBuff) */ + // size_t tmpInSize; /* amount of data to compress after tmpIn */ + total_in_size: u64, + xxh: xxhash.XXH32_state, + lz4_ctx_ptr: rawptr, + lz4_context_alloc: u16, /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + lz4_context_state: u16, /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ +} + +Last_Block_Status :: enum { + not_done, + from_temp_buffer, + from_source_buffer, +} + +/*-*************************************************** +* Frame Decompression +*****************************************************/ + +Decompression_Stage :: enum { + get_frame_header = 0, + store_frame_header, + init, + get_block_header, + store_block_header, + copy_direct, + get_block_checksum, + get_C_block, + store_C_block, + flush_out, + get_suffix, + store_suffix, + get_S_frame_size, + store_S_frame_size, + skip_skippable, +} + +/* +struct LZ4F_dctx_s { + LZ4F_frameInfo_t frameInfo; + U32 version; + dStage_t dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + XXH32_state_t blockChecksum; + BYTE header[LZ4F_HEADER_SIZE_MAX]; +}; /* typedef'd to LZ4F_dctx in lz4frame.h */ + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); + +*/ + +/* + The larger the block size, the (slightly) better the compression ratio, though there are diminishing returns. + Larger blocks also increase memory usage on both compression and decompression sides. +*/ +Frame_Block_Size_ID :: enum u32 { + default = 0, + max_64_kiB = 4, + max_256_kiB = 5, + max_1_MiB = 6, + max_4_MiB = 7, + obsolete_max_64_kiB, + obsolete_max_256_kiB, + obsolete_max_1_MiB, + obsolete_max_4_MiB, +} + +/* + Linked blocks sharply reduce inefficiencies when using small blocks, they compress better. + However, some LZ4 decoders are only compatible with independent blocks. +*/ +Block_Mode :: enum u32 { + block_linked = 0, + block_independent, + obsolete_block_linked, + obsolete_block_independent, +} + +Content_Checksum :: enum u32 { + no_content_checksum = 0, + content_checksum_enabled, + obsolete_no_content_checksum, + obsolete_content_checksum_enabled, +} + +Frame_Type :: enum u32 { + frame = 0, + skippable_frame, + obsolete_skippable_frame, +} + +Block_Checksum :: enum u32 { + no_block_checksum = 0, + block_checksum_enabled, +} + +/* + LZ4F_frameInfo_t : makes it possible to set or read frame parameters. + Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO, setting all parameters to default. + It's then possible to update selectively some parameters. +*/ +Frame_Info :: struct { + block_size: Frame_Block_Size_ID, /* max64KB, max256KB, max1MB, max4MB; 0 == default */ + block_mode: Block_Mode, /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */ + content_checksum: Content_Checksum, /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */ + frame_type: Frame_Type, /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ + content_size: u64, /* Size of uncompressed content ; 0 == unknown */ + dict_id: u32, /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */ + block_checksum_flag: Block_Checksum, /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */ +} +#assert(size_of(Frame_Info) == 32) + +INIT_FRAMEINFO :: Frame_Info{ + .default, .block_linked, .no_content_checksum, .frame, 0, 0, .no_block_checksum, +} /* v1.8.3+ */ + +/* + LZ4F_preferences_t: makes it possible to supply advanced compression instructions to streaming interface. + Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES, setting all parameters to default. + All reserved fields must be set to zero. +*/ +Preferences :: struct { + frame_info: Frame_Info, + compression_level: i32, /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ + auto_flush: i32, /* 1: always flush; reduces usage of internal buffers */ + favor_decompression_speed: i32, /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */ + reserved: [3]u32, /* must be zero for forward compatibility */ +} +#assert(size_of(Preferences) == 56) + +INIT_PREFERENCES :: Preferences{ + INIT_FRAMEINFO, 0, 0, 0, { 0, 0, 0 }, +} /* v1.8.3+ */ + + +/*-*********************************** +* Advanced compression options +*************************************/ +Compress_Options :: struct { + stable_source: b32, /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ + reserved: [3]u32, +} + +/*--- Resource Management ---*/ +Frame_Version :: 100 /* This number can be used to check for an incompatible API breaking change */ + +/*---- Compression ----*/ + +Frame_Header_Size_Min :: 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ +Frame_Header_Size_Max :: 19 + +/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ +Block_Header_Size :: 4 + +/* Size in bytes of a block checksum footer in little-endian format. */ +Block_Checksum_Size :: 4 + +/* Size in bytes of the content checksum. */ +Content_Checksum_Size :: 4 + +/*-********************************* +* Decompression options +***********************************/ +Uncompress_Options :: struct { + stable_output: b32, /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */ + reserved: [3]u32, /* must be set to zero for forward compatibility */ +} + +/*-*********************************** +* Streaming decompression options +*************************************/ +MIN_SIZE_TO_KNOW_HEADER_LENGTH :: 5 \ No newline at end of file diff --git a/core/compress/lz4/lz4.odin b/core/compress/lz4/lz4.odin new file mode 100644 index 000000000..0c75e9f96 --- /dev/null +++ b/core/compress/lz4/lz4.odin @@ -0,0 +1,75 @@ +package lz4 + +/* + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + List of contributors: + Jeroen van Rijn: Initial implementation, optimization. + + Thanks to Yann Collet for creating the LZ4 compression format. + This implementation is written in accordance with the + (LZ4 block and frame specifications)[https://github.com/lz4/lz4/tree/dev/doc] +*/ + +import "core:compress" + +// import "core:mem" +// import "core:hash" +import "core:bytes" +import "core:fmt" + +Error :: compress.Error + +@(optimization_mode="speed") +uncompress_from_context :: proc(using ctx: ^compress.Context_Memory_Input, expected_output_size := -1, allocator := context.allocator) -> (err: Error) #no_bounds_check { + /* + ctx.output must be a bytes.Buffer for now. We'll add a separate implementation that writes to a stream. + */ + expected_output_size := expected_output_size + + /* + Always set up a minimum allocation size. + */ + expected_output_size = max(max(expected_output_size, compress.COMPRESS_OUTPUT_ALLOCATE_MIN), 512) + + if expected_output_size > 0 && expected_output_size <= compress.COMPRESS_OUTPUT_ALLOCATE_MAX { + /* + Try to pre-allocate the output buffer. + */ + reserve(&ctx.output.buf, expected_output_size) + resize (&ctx.output.buf, expected_output_size) + } + + if len(ctx.output.buf) != expected_output_size { + return .Resize_Failed + } + + ctx.num_bits = 0 + ctx.code_buffer = 0 + + + parse_header(ctx) or_return + + return nil +} + +uncompress_from_byte_array :: proc(input: []u8, buf: ^bytes.Buffer, expected_output_size := -1) -> (err: Error) { + ctx := compress.Context_Memory_Input{} + + ctx.input_data = input + ctx.output = buf + + return uncompress_from_context(ctx=&ctx, expected_output_size=expected_output_size) +} + +uncompress :: proc{uncompress_from_context, uncompress_from_byte_array} + + +parse_header :: proc(z: ^$C) -> (err: Error) { + magic := compress.read_data(z, Magic) or_return + + fmt.printf("magic: %v\n", magic) + + return nil +} \ No newline at end of file From 64ce55944a54c18894b4f6ff93388d3153399022 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 19 Sep 2021 22:10:19 +0200 Subject: [PATCH 2/2] fix mem.new_clone --- core/c/frontend/tokenizer/token.odin | 4 ++-- core/mem/alloc.odin | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/c/frontend/tokenizer/token.odin b/core/c/frontend/tokenizer/token.odin index c4760ea42..1376a651f 100644 --- a/core/c/frontend/tokenizer/token.odin +++ b/core/c/frontend/tokenizer/token.odin @@ -83,13 +83,13 @@ Token :: struct { Is_Keyword_Proc :: #type proc(tok: ^Token) -> bool copy_token :: proc(tok: ^Token) -> ^Token { - t := new_clone(tok^) + t, _ := new_clone(tok^) t.next = nil return t } new_eof :: proc(tok: ^Token) -> ^Token { - t := new_clone(tok^) + t, _ := new_clone(tok^) t.kind = .EOF t.lit = "" return t diff --git a/core/mem/alloc.odin b/core/mem/alloc.odin index 796d5c20a..bdd2899a9 100644 --- a/core/mem/alloc.odin +++ b/core/mem/alloc.odin @@ -215,13 +215,14 @@ new_aligned :: proc($T: typeid, alignment: int, allocator := context.allocator, t = (^T)(raw_data(data)) return } -new_clone :: proc(data: $T, allocator := context.allocator, loc := #caller_location) -> ^T { - data := alloc_bytes(size_of(T), alignment, allocator, loc) or_return - t = (^T)(raw_data(data)) +new_clone :: proc(data: $T, allocator := context.allocator, loc := #caller_location) -> (t: ^T, err: Allocator_Error) { + backing := alloc_bytes(size_of(T), align_of(T), allocator, loc) or_return + t = (^T)(raw_data(backing)) if t != nil { t^ = data + return t, nil } - return + return nil, .Out_Of_Memory } DEFAULT_RESERVE_CAPACITY :: 16