From a6d5f2ea23d200f82705b4b32f469ebb59b6cb9c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 24 Apr 2026 06:35:16 -0700 Subject: [PATCH] pkg/simdutf: no_libc mode --- pkg/simdutf/build.zig | 26 +++ pkg/simdutf/no_libc.zig | 47 ++++ pkg/simdutf/vendor/simdutf.cpp | 137 +++++------ pkg/simdutf/vendor/simdutf.h | 401 ++++++++++++++++++++++++++------- 4 files changed, 465 insertions(+), 146 deletions(-) create mode 100644 pkg/simdutf/no_libc.zig diff --git a/pkg/simdutf/build.zig b/pkg/simdutf/build.zig index 0859edc26..c1455dc32 100644 --- a/pkg/simdutf/build.zig +++ b/pkg/simdutf/build.zig @@ -4,6 +4,7 @@ pub fn build(b: *std.Build) !void { const optimize = b.standardOptimizeOption(.{}); const target = b.standardTargetOptions(.{}); const no_libcxx = b.option(bool, "no_libcxx", "Set SIMDUTF_NO_LIBCXX to avoid libc++ dependency") orelse false; + const no_libc = b.option(bool, "no_libc", "Set SIMDUTF_NO_LIBC and provide Zig stdlib replacements") orelse false; const lib = b.addLibrary(.{ .name = "simdutf", @@ -64,6 +65,31 @@ pub fn build(b: *std.Build) !void { lib.root_module.addCMacro("SIMDUTF_NO_LIBCXX", "1"); } + if (no_libc) { + try flags.appendSlice(b.allocator, &.{ + "-DSIMDUTF_NO_LIBC=1", + "-DSIMDUTF_LIBC_MEMCPY=simdutf_memcpy", + "-DSIMDUTF_LIBC_MEMMOVE=simdutf_memmove", + "-DSIMDUTF_LIBC_MEMSET=simdutf_memset", + "-DSIMDUTF_LIBC_MEMCMP=simdutf_memcmp", + "-DSIMDUTF_LIBC_STRLEN=simdutf_strlen", + "-DSIMDUTF_LIBC_GETENV=simdutf_getenv", + }); + + lib.root_module.addCMacro("SIMDUTF_NO_LIBC", "1"); + + const no_libc_obj = b.addObject(.{ + .name = "simdutf_no_libc", + .root_module = b.createModule(.{ + .root_source_file = b.path("no_libc.zig"), + .target = target, + .optimize = optimize, + .link_libc = false, + }), + }); + lib.addObject(no_libc_obj); + } + if (target.result.abi == .msvc) { // On MSVC we skip linkLibCpp (see above), so the C++ standard is // not set implicitly. simdutf requires C++17, so set it explicitly. diff --git a/pkg/simdutf/no_libc.zig b/pkg/simdutf/no_libc.zig new file mode 100644 index 000000000..4194712bd --- /dev/null +++ b/pkg/simdutf/no_libc.zig @@ -0,0 +1,47 @@ +const std = @import("std"); + +export fn simdutf_memcpy(noalias dest: ?[*]u8, noalias src: ?[*]const u8, n: usize) ?[*]u8 { + const d = dest orelse return dest; + const s = src orelse return dest; + @memcpy(d[0..n], s[0..n]); + return dest; +} + +export fn simdutf_memmove(dest: ?[*]u8, src: ?[*]const u8, n: usize) ?[*]u8 { + const d = dest orelse return dest; + const s = src orelse return dest; + const dst_slice = d[0..n]; + const src_slice = s[0..n]; + if (@intFromPtr(d) <= @intFromPtr(s)) { + @memcpy(dst_slice, src_slice); + } else { + std.mem.copyBackwards(u8, dst_slice, src_slice); + } + return dest; +} + +export fn simdutf_memset(dest: ?[*]u8, c: c_int, n: usize) ?[*]u8 { + const d = dest orelse return dest; + @memset(d[0..n], @as(u8, @intCast(c & 0xff))); + return dest; +} + +export fn simdutf_memcmp(lhs: ?[*]const u8, rhs: ?[*]const u8, n: usize) c_int { + const l = lhs orelse return 0; + const r = rhs orelse return 0; + const order = std.mem.order(u8, l[0..n], r[0..n]); + return switch (order) { + .lt => -1, + .eq => 0, + .gt => 1, + }; +} + +export fn simdutf_strlen(s: ?[*:0]const u8) usize { + const str = s orelse return 0; + return std.mem.len(str); +} + +export fn simdutf_getenv(_: ?[*:0]const u8) ?[*:0]const u8 { + return null; +} diff --git a/pkg/simdutf/vendor/simdutf.cpp b/pkg/simdutf/vendor/simdutf.cpp index 82f9c26d8..c6940b543 100644 --- a/pkg/simdutf/vendor/simdutf.cpp +++ b/pkg/simdutf/vendor/simdutf.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2026-04-21 21:46:47 -0400. Do not edit! */ +/* auto-generated on 2026-04-23 21:18:04 -0700. Do not edit! */ /* begin file src/simdutf.cpp */ #include "simdutf.h" @@ -7,19 +7,23 @@ namespace simdutf { std::string_view to_string(encoding_type bom) { switch (bom) { case UTF16_LE: - return "UTF16 little-endian"; + return std::string_view("UTF16 little-endian", + sizeof("UTF16 little-endian") - 1); case UTF16_BE: - return "UTF16 big-endian"; + return std::string_view("UTF16 big-endian", + sizeof("UTF16 big-endian") - 1); case UTF32_LE: - return "UTF32 little-endian"; + return std::string_view("UTF32 little-endian", + sizeof("UTF32 little-endian") - 1); case UTF32_BE: - return "UTF32 big-endian"; + return std::string_view("UTF32 big-endian", + sizeof("UTF32 big-endian") - 1); case UTF8: - return "UTF8"; + return std::string_view("UTF8", sizeof("UTF8") - 1); case unspecified: - return "unknown"; + return std::string_view("unknown", sizeof("unknown") - 1); default: - return "error"; + return std::string_view("error", sizeof("error") - 1); } } @@ -11308,7 +11312,8 @@ detect_best_supported_implementation_on_first_use::set_best() const noexcept { SIMDUTF_PUSH_DISABLE_WARNINGS SIMDUTF_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: // manually verified this is safe - char *force_implementation_name = getenv("SIMDUTF_FORCE_IMPLEMENTATION"); + const char *force_implementation_name = + simdutf::internal::getenv("SIMDUTF_FORCE_IMPLEMENTATION"); SIMDUTF_POP_DISABLE_WARNINGS if (force_implementation_name) { @@ -12500,7 +12505,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, if (offset + 64 > line_length) { size_t location_end = line_length - offset; size_t to_move = 64 - location_end; - std::memmove(out + location_end + 1, out + location_end, to_move); + simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move); out[location_end] = '\n'; offset = to_move; out += 64 + 1; @@ -12550,7 +12555,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, if (offset + 32 > line_length) { size_t location_end = line_length - offset; size_t to_move = 32 - location_end; - std::memmove(out + location_end + 1, out + location_end, to_move); + simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move); out[location_end] = '\n'; offset = to_move; out += 32 + 1; @@ -13010,7 +13015,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, base64_decode_block(dst, buffer + i * 64); dst += 48; } - std::memcpy(buffer, buffer + (block_size - 1) * 64, + simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } @@ -13048,7 +13053,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 4); + simdutf::internal::memcpy(dst, &triple, 4); dst += 3; buffer_start += 4; @@ -13062,7 +13067,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -13956,11 +13961,11 @@ simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if (len == idx) { return 0; - } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, + } // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers + simdutf::internal::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it is more efficient // to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); + simdutf::internal::memcpy(dst, buf + idx, len - idx); return len - idx; } @@ -17091,7 +17096,7 @@ valid_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { int valid_count2; __m512i vec2 = expand_and_identify(lane2, lane3, valid_count2); uint32_t tmp1; - ::memcpy(&tmp1, ptr + 64, sizeof(tmp1)); + simdutf::internal::memcpy(&tmp1, ptr + 64, sizeof(tmp1)); const __m512i lane4 = _mm512_set1_epi32(tmp1); int valid_count3; __m512i vec3 = expand_and_identify(lane3, lane4, valid_count3); @@ -17213,7 +17218,7 @@ validating_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) { int valid_count2; __m512i vec2 = expand_and_identify(lane2, lane3, valid_count2); uint32_t tmp1; - ::memcpy(&tmp1, ptr + 64, sizeof(tmp1)); + simdutf::internal::memcpy(&tmp1, ptr + 64, sizeof(tmp1)); const __m512i lane4 = _mm512_set1_epi32(tmp1); int valid_count3; __m512i vec3 = expand_and_identify(lane3, lane4, valid_count3); @@ -17345,7 +17350,7 @@ validating_utf8_to_fixed_length_with_constant_checks(const char *str, int valid_count2; __m512i vec2 = expand_and_identify(lane2, lane3, valid_count2); uint32_t tmp1; - ::memcpy(&tmp1, ptr + 64, sizeof(tmp1)); + simdutf::internal::memcpy(&tmp1, ptr + 64, sizeof(tmp1)); const __m512i lane4 = _mm512_set1_epi32(tmp1); int valid_count3; __m512i vec3 = expand_and_identify(lane3, lane4, valid_count3); @@ -18856,7 +18861,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, base64_decode_block(dst, buffer + i * 64); dst += 48; } - std::memcpy(buffer, buffer + (block_size - 1) * 64, + simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } @@ -21541,7 +21546,7 @@ avx2_encode_base64_impl(char *dst, const char *src, size_t srclen, if (offset + 32 > line_length) { size_t location_end = line_length - offset; size_t to_move = 32 - location_end; - std::memmove(out + location_end + 1, out + location_end, to_move); + simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move); out[location_end] = '\n'; offset = to_move; out += 32 + 1; @@ -21554,7 +21559,7 @@ avx2_encode_base64_impl(char *dst, const char *src, size_t srclen, alignas(32) uint8_t buffer[32]; _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer), lookup_pshufb_improved(indices)); - std::memcpy(out, buffer, 32); + simdutf::internal::memcpy(out, buffer, 32); size_t out_pos = 0; size_t local_offset = offset; for (size_t j = 0; j < 32;) { @@ -21664,7 +21669,7 @@ simdutf_really_inline void base64_decode_block_safe(char *out, alignas(32) char buffer[32]; // We enforce safety with a buffer. base64_decode( buffer, _mm256_loadu_si256(reinterpret_cast(src + 32))); - std::memcpy(out + 24, buffer, 24); + simdutf::internal::memcpy(out + 24, buffer, 24); } // --- decoding - base64 class -------------------------------- @@ -21715,7 +21720,7 @@ public: base64_decode(out, chunks[0]); alignas(32) char buffer[32]; // We enforce safety with a buffer. base64_decode(buffer, chunks[1]); - std::memcpy(out + 24, buffer, 24); + simdutf::internal::memcpy(out + 24, buffer, 24); } template @@ -22072,11 +22077,11 @@ simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if (len == idx) { return 0; - } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, + } // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers + simdutf::internal::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it is more efficient // to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); + simdutf::internal::memcpy(dst, buf + idx, len - idx); return len - idx; } @@ -23669,7 +23674,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, base64_decode_block(dst, buffer + (block_size - 2) * 64); } dst += 48; - std::memcpy(buffer, buffer + (block_size - 1) * 64, + simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } @@ -23713,7 +23718,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -23727,7 +23732,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -25607,11 +25612,11 @@ size_t convert_masked_utf8_to_latin1(const char *input, #if defined(__clang__) __attribute__((aligned(16))) char buf[16]; latin1_packed.store(buf); - memcpy(latin1_output, buf, 6); + simdutf::internal::memcpy(latin1_output, buf, 6); #else // writing 8 bytes even though we only care about the first 6 bytes. const auto tmp = vec_u64_t(latin1_packed.value); - memcpy(latin1_output, &tmp[0], 8); + simdutf::internal::memcpy(latin1_output, &tmp[0], 8); #endif latin1_output += 6; // We wrote 6 bytes. return consumed; @@ -26661,7 +26666,7 @@ static simdutf_really_inline void base64_decode_block_safe(char *out, char buffer[16]; base64_decode(buffer, vector_u8::load(src + 3 * 16)); - std::memcpy(out + 36, buffer, 12); + simdutf::internal::memcpy(out + 36, buffer, 12); } // ---base64 decoding::block64 class -------------------------- @@ -26822,7 +26827,7 @@ public: base64_decode(out + 12 * 2, b.chunks[2]); char buffer[16]; base64_decode(buffer, b.chunks[3]); - std::memcpy(out + 12 * 3, buffer, 12); + simdutf::internal::memcpy(out + 12 * 3, buffer, 12); } }; /* end file src/ppc64/ppc64_base64.cpp */ @@ -26891,11 +26896,11 @@ simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if (len == idx) { return 0; - } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, + } // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers + simdutf::internal::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it is more efficient // to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); + simdutf::internal::memcpy(dst, buf + idx, len - idx); return len - idx; } @@ -28485,7 +28490,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, base64_decode_block(dst, buffer + (block_size - 2) * 64); } dst += 48; - std::memcpy(buffer, buffer + (block_size - 1) * 64, + simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } @@ -28529,7 +28534,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -28543,7 +28548,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -31461,7 +31466,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 16), t1); _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 32), t2); _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 48), t3); - std::memcpy(out, buffer, 64); + simdutf::internal::memcpy(out, buffer, 64); size_t out_pos = 0; size_t local_offset = offset; for (size_t j = 0; j < 64;) { @@ -31529,7 +31534,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen, if (offset + 16 > line_length) { size_t location_end = line_length - offset; size_t to_move = 16 - location_end; - std::memmove(out + location_end + 1, out + location_end, to_move); + simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move); out[location_end] = '\n'; offset = to_move; out += 16 + 1; @@ -31641,7 +31646,7 @@ static inline void base64_decode_block_safe(char *out, const char *src) { char buffer[16]; base64_decode(buffer, _mm_loadu_si128(reinterpret_cast(src + 48))); - std::memcpy(out + 36, buffer, 12); + simdutf::internal::memcpy(out + 36, buffer, 12); } // --- decoding - base64 class -------------------------------- @@ -31909,7 +31914,7 @@ public: base64_decode(out + 24, chunks[2]); char buffer[16]; base64_decode(buffer, chunks[3]); - std::memcpy(out + 36, buffer, 12); + simdutf::internal::memcpy(out + 36, buffer, 12); } }; /* end file src/westmere/sse_base64.cpp */ @@ -31978,11 +31983,11 @@ simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if (len == idx) { return 0; - } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, + } // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers + simdutf::internal::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it is more efficient // to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); + simdutf::internal::memcpy(dst, buf + idx, len - idx); return len - idx; } @@ -33570,7 +33575,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, base64_decode_block(dst, buffer + (block_size - 2) * 64); } dst += 48; - std::memcpy(buffer, buffer + (block_size - 1) * 64, + simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } @@ -33614,7 +33619,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -33628,7 +33633,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen, #if !SIMDUTF_IS_BIG_ENDIAN triple = scalar::u32_swap_bytes(triple); #endif - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -36050,7 +36055,7 @@ static inline void base64_decode_block_safe(char *out, const char *src) { alignas(32) char buffer[32]; base64_decode(buffer, __lasx_xvld(reinterpret_cast(src), 32)); - std::memcpy(out + 24, buffer, 24); + simdutf::internal::memcpy(out + 24, buffer, 24); } static inline void base64_decode_block(char *out, block64 *b) { @@ -36061,7 +36066,7 @@ static inline void base64_decode_block_safe(char *out, block64 *b) { base64_decode(out, b->chunks[0]); alignas(32) char buffer[32]; base64_decode(buffer, b->chunks[1]); - std::memcpy(out + 24, buffer, 24); + simdutf::internal::memcpy(out + 24, buffer, 24); } template ::get_remainder(uint8_t *dst) const { if (len == idx) { return 0; - } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, + } // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers + simdutf::internal::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it is more efficient // to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); + simdutf::internal::memcpy(dst, buf + idx, len - idx); return len - idx; } @@ -38908,7 +38913,7 @@ size_t convert_masked_utf8_to_latin1(const char *input, uint64_t buffer[2]; // __lsx_vst(latin1_packed, reinterpret_cast(latin1_output), 0); __lsx_vst(latin1_packed, reinterpret_cast(buffer), 0); - std::memcpy(latin1_output, buffer, 6); + simdutf::internal::memcpy(latin1_output, buffer, 6); latin1_output += 6; // We wrote 6 bytes. return consumed; } @@ -40011,7 +40016,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, base64_decode_block(dst, buffer + i * 64); dst += 48; } - std::memcpy(buffer, buffer + (block_size - 1) * 64, + simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64, 64); // 64 might be too much bufferptr -= (block_size - 1) * 64; } @@ -40048,7 +40053,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, << 8; // lsx is little-endian triple = scalar::u32_swap_bytes(triple); - std::memcpy(dst, &triple, 4); + simdutf::internal::memcpy(dst, &triple, 4); dst += 3; buffer_start += 4; @@ -40061,7 +40066,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen, << 8; // lsx is little-endian triple = scalar::u32_swap_bytes(triple); - std::memcpy(dst, &triple, 3); + simdutf::internal::memcpy(dst, &triple, 3); dst += 3; buffer_start += 4; @@ -40247,11 +40252,11 @@ simdutf_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if (len == idx) { return 0; - } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, + } // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers + simdutf::internal::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it is more efficient // to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); + simdutf::internal::memcpy(dst, buf + idx, len - idx); return len - idx; } diff --git a/pkg/simdutf/vendor/simdutf.h b/pkg/simdutf/vendor/simdutf.h index ff4bf6a83..db7fe353b 100644 --- a/pkg/simdutf/vendor/simdutf.h +++ b/pkg/simdutf/vendor/simdutf.h @@ -1,4 +1,4 @@ -/* auto-generated on 2026-04-21 21:46:47 -0400. Do not edit! */ +/* auto-generated on 2026-04-23 21:18:04 -0700. Do not edit! */ /* begin file include/simdutf.h */ #ifndef SIMDUTF_H #define SIMDUTF_H @@ -556,6 +556,188 @@ #define simdutf_constexpr23 #endif +/* begin file include/simdutf/libc.h */ +#ifndef SIMDUTF_LIBC_H +#define SIMDUTF_LIBC_H + +#include +#include +#include + +#ifndef SIMDUTF_NO_LIBC + #define SIMDUTF_NO_LIBC 0 +#endif + +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_MEMCPY +extern "C" void *SIMDUTF_LIBC_MEMCPY(void *, const void *, size_t) noexcept; + #endif + #ifdef SIMDUTF_LIBC_MEMMOVE +extern "C" void *SIMDUTF_LIBC_MEMMOVE(void *, const void *, size_t) noexcept; + #endif + #ifdef SIMDUTF_LIBC_MEMSET +extern "C" void *SIMDUTF_LIBC_MEMSET(void *, int, size_t) noexcept; + #endif + #ifdef SIMDUTF_LIBC_MEMCMP +extern "C" int SIMDUTF_LIBC_MEMCMP(const void *, const void *, size_t) noexcept; + #endif + #ifdef SIMDUTF_LIBC_STRLEN +extern "C" size_t SIMDUTF_LIBC_STRLEN(const char *) noexcept; + #endif + #ifdef SIMDUTF_LIBC_GETENV +extern "C" const char *SIMDUTF_LIBC_GETENV(const char *) noexcept; + #endif +#endif + +namespace simdutf { +namespace internal { + +simdutf_really_inline void *builtin_memcpy(void *dst, const void *src, + size_t count) noexcept { + auto *output = reinterpret_cast(dst); + const auto *input = reinterpret_cast(src); + for (size_t i = 0; i < count; i++) { + output[i] = input[i]; + } + return dst; +} + +simdutf_really_inline void *builtin_memmove(void *dst, const void *src, + size_t count) noexcept { + auto *output = reinterpret_cast(dst); + const auto *input = reinterpret_cast(src); + if (output == input || count == 0) { + return dst; + } + if (output < input || output >= input + count) { + for (size_t i = 0; i < count; i++) { + output[i] = input[i]; + } + return dst; + } + for (size_t i = count; i != 0; i--) { + output[i - 1] = input[i - 1]; + } + return dst; +} + +simdutf_really_inline void *builtin_memset(void *dst, int value, + size_t count) noexcept { + auto *output = reinterpret_cast(dst); + const auto byte = static_cast(value); + for (size_t i = 0; i < count; i++) { + output[i] = byte; + } + return dst; +} + +simdutf_really_inline int builtin_memcmp(const void *lhs, const void *rhs, + size_t count) noexcept { + const auto *left = reinterpret_cast(lhs); + const auto *right = reinterpret_cast(rhs); + for (size_t i = 0; i < count; i++) { + if (left[i] != right[i]) { + return int(left[i]) - int(right[i]); + } + } + return 0; +} + +simdutf_really_inline size_t builtin_strlen(const char *input) noexcept { + const auto *bytes = reinterpret_cast(input); + size_t length = 0; + while (bytes[length] != '\0') { + length++; + } + return length; +} + +simdutf_really_inline void *memcpy(void *dst, const void *src, + size_t count) noexcept { +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_MEMCPY + return SIMDUTF_LIBC_MEMCPY(dst, src, count); + #else + return builtin_memcpy(dst, src, count); + #endif +#else + return std::memcpy(dst, src, count); +#endif +} + +simdutf_really_inline void *memmove(void *dst, const void *src, + size_t count) noexcept { +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_MEMMOVE + return SIMDUTF_LIBC_MEMMOVE(dst, src, count); + #else + return builtin_memmove(dst, src, count); + #endif +#else + return std::memmove(dst, src, count); +#endif +} + +simdutf_really_inline void *memset(void *dst, int value, + size_t count) noexcept { +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_MEMSET + return SIMDUTF_LIBC_MEMSET(dst, value, count); + #else + return builtin_memset(dst, value, count); + #endif +#else + return std::memset(dst, value, count); +#endif +} + +simdutf_really_inline int memcmp(const void *lhs, const void *rhs, + size_t count) noexcept { +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_MEMCMP + return SIMDUTF_LIBC_MEMCMP(lhs, rhs, count); + #else + return builtin_memcmp(lhs, rhs, count); + #endif +#else + return std::memcmp(lhs, rhs, count); +#endif +} + +simdutf_really_inline size_t strlen(const char *input) noexcept { +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_STRLEN + return SIMDUTF_LIBC_STRLEN(input); + #else + return builtin_strlen(input); + #endif +#else + return std::strlen(input); +#endif +} + +simdutf_really_inline const char *getenv(const char *name) noexcept { +#if SIMDUTF_NO_LIBC + #ifdef SIMDUTF_LIBC_GETENV + return SIMDUTF_LIBC_GETENV(name); + #else + (void)name; + return nullptr; + #endif +#else + SIMDUTF_PUSH_DISABLE_WARNINGS + SIMDUTF_DISABLE_DEPRECATED_WARNING + return std::getenv(name); + SIMDUTF_POP_DISABLE_WARNINGS +#endif +} + +} // namespace internal +} // namespace simdutf + +#endif // SIMDUTF_LIBC_H +/* end file include/simdutf/libc.h */ + #ifndef SIMDUTF_DLLIMPORTEXPORT #if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio /** @@ -842,29 +1024,33 @@ enum error_code { inline std::string_view error_to_string(error_code code) noexcept { switch (code) { case SUCCESS: - return "SUCCESS"; + return std::string_view("SUCCESS", sizeof("SUCCESS") - 1); case HEADER_BITS: - return "HEADER_BITS"; + return std::string_view("HEADER_BITS", sizeof("HEADER_BITS") - 1); case TOO_SHORT: - return "TOO_SHORT"; + return std::string_view("TOO_SHORT", sizeof("TOO_SHORT") - 1); case TOO_LONG: - return "TOO_LONG"; + return std::string_view("TOO_LONG", sizeof("TOO_LONG") - 1); case OVERLONG: - return "OVERLONG"; + return std::string_view("OVERLONG", sizeof("OVERLONG") - 1); case TOO_LARGE: - return "TOO_LARGE"; + return std::string_view("TOO_LARGE", sizeof("TOO_LARGE") - 1); case SURROGATE: - return "SURROGATE"; + return std::string_view("SURROGATE", sizeof("SURROGATE") - 1); case INVALID_BASE64_CHARACTER: - return "INVALID_BASE64_CHARACTER"; + return std::string_view("INVALID_BASE64_CHARACTER", + sizeof("INVALID_BASE64_CHARACTER") - 1); case BASE64_INPUT_REMAINDER: - return "BASE64_INPUT_REMAINDER"; + return std::string_view("BASE64_INPUT_REMAINDER", + sizeof("BASE64_INPUT_REMAINDER") - 1); case BASE64_EXTRA_BITS: - return "BASE64_EXTRA_BITS"; + return std::string_view("BASE64_EXTRA_BITS", + sizeof("BASE64_EXTRA_BITS") - 1); case OUTPUT_BUFFER_TOO_SMALL: - return "OUTPUT_BUFFER_TOO_SMALL"; + return std::string_view("OUTPUT_BUFFER_TOO_SMALL", + sizeof("OUTPUT_BUFFER_TOO_SMALL") - 1); default: - return "OTHER"; + return std::string_view("OTHER", sizeof("OTHER") - 1); } } @@ -1604,9 +1790,9 @@ simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, { for (; pos + 16 <= len; pos += 16) { uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) != 0) { return false; @@ -1637,9 +1823,9 @@ validate_with_errors(InputPtr data, size_t len) noexcept { // process in blocks of 16 bytes when possible for (; pos + 16 <= len; pos += 16) { uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) != 0) { for (; pos < len; pos++) { @@ -1710,7 +1896,7 @@ inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { auto *src_aligned = reinterpret_cast(const_cast(src)); const auto dst_value = std::atomic_ref(*src_aligned).load(std::memory_order_relaxed); - std::memcpy(dst, &dst_value, sizeof(uint64_t)); + simdutf::internal::memcpy(dst, &dst_value, sizeof(uint64_t)); src += alignment; dst += alignment; len -= alignment; @@ -1756,7 +1942,7 @@ inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { while (len >= alignment) { auto *dst_aligned = reinterpret_cast(dst); uint64_t src_val; - std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src + simdutf::internal::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src std::atomic_ref(*dst_aligned) .store(src_val, std::memory_order_relaxed); dst += alignment; @@ -1914,9 +2100,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 1000 // 1000, so it makes sense to concatenate everything @@ -1966,16 +2152,16 @@ inline size_t convert_safe(const char *buf, size_t len, char *utf8_output, utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes, // check that they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 1000 // 1000, so it makes sense to concatenate everything if ((v & 0x8080808080808080) == 0) { // if NONE of these are set, e.g. all of them are zero, then // everything is ASCII - ::memcpy(utf8_output + utf8_pos, buf + pos, 16); + simdutf::internal::memcpy(utf8_output + utf8_pos, buf + pos, 16); utf8_pos += 16; pos += 16; } else { @@ -2050,18 +2236,18 @@ utf8_length_from_latin1(InputPtr input, size_t length) noexcept { }; for (; i + 32 <= length; i += 32) { uint64_t v; - memcpy(&v, input + i, 8); + simdutf::internal::memcpy(&v, input + i, 8); answer += pop(v); - memcpy(&v, input + i + 8, sizeof(v)); + simdutf::internal::memcpy(&v, input + i + 8, sizeof(v)); answer += pop(v); - memcpy(&v, input + i + 16, sizeof(v)); + simdutf::internal::memcpy(&v, input + i + 16, sizeof(v)); answer += pop(v); - memcpy(&v, input + i + 24, sizeof(v)); + simdutf::internal::memcpy(&v, input + i + 24, sizeof(v)); answer += pop(v); } for (; i + 8 <= length; i += 8) { uint64_t v; - memcpy(&v, input + i, sizeof(v)); + simdutf::internal::memcpy(&v, input + i, sizeof(v)); answer += pop(v); } } // !consteval scope @@ -2365,10 +2551,10 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that // they are Latin1 uint64_t v1, v2, v3, v4; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); - ::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); - ::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); - ::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + 4, sizeof(uint64_t)); + simdutf::internal::memcpy(&v3, data + pos + 8, sizeof(uint64_t)); + simdutf::internal::memcpy(&v4, data + pos + 12, sizeof(uint64_t)); if constexpr (!match_system(big_endian)) { v1 = (v1 >> 8) | (v1 << (64 - 8)); @@ -2617,7 +2803,7 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len, if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if constexpr (!match_system(big_endian)) { v = (v >> 8) | (v << (64 - 8)); } @@ -2707,7 +2893,7 @@ simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len, if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if constexpr (!match_system(big_endian)) v = (v >> 8) | (v << (64 - 8)); if ((v & 0xFF80FF80FF80FF80) == 0) { @@ -2814,7 +3000,7 @@ simdutf_constexpr23 size_t convert_with_replacement(const char16_t *data, if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if constexpr (!match_system(big_endian)) { v = (v >> 8) | (v << (64 - 8)); } @@ -2914,7 +3100,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if constexpr (!match_system(big_endian)) { v = (v >> 8) | (v << (64 - 8)); } @@ -3105,7 +3291,7 @@ inline simdutf_constexpr23 result convert_with_errors(const char32_t *data, if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that // they are Latin1 uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0xFFFFFF00FFFFFF00) == 0) { *latin1_output++ = char(data[pos]); *latin1_output++ = char(data[pos + 1]); @@ -3165,7 +3351,7 @@ simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len, if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that they are Latin1 uint64_t v; - std::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0xFFFFFF00FFFFFF00) == 0) { *latin1_output++ = char(data[pos]); *latin1_output++ = char(data[pos + 1]); @@ -3361,7 +3547,7 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len, if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0xFFFFFF80FFFFFF80) == 0) { *utf8_output++ = char(data[pos]); *utf8_output++ = char(data[pos + 1]); @@ -3425,7 +3611,7 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0xFFFFFF80FFFFFF80) == 0) { *utf8_output++ = char(data[pos]); *utf8_output++ = char(data[pos + 1]); @@ -3505,7 +3691,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0xFFFFFF80FFFFFF80) == 0) { *utf8_output++ = char(data[pos]); *utf8_output++ = char(data[pos + 1]); @@ -3581,9 +3767,9 @@ simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data, if (next_pos <= len) { // if it is safe to read 16 more bytes, check // that they are ascii uint64_t v1{}; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2{}; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) == 0) { pos = next_pos; @@ -3682,9 +3868,9 @@ validate_with_errors(BytePtr data, size_t len) noexcept { if (next_pos <= len) { // if it is safe to read 16 more bytes, check that they are ascii uint64_t v1; - std::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) == 0) { pos = next_pos; @@ -3909,9 +4095,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 // 1000 1000 .... etc if ((v & 0x8080808080808080) == @@ -3984,9 +4170,9 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 // 1000 1000...etc if ((v & 0x8080808080808080) == @@ -4136,9 +4322,9 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 1000 // 1000, so it makes sense to concatenate everything @@ -4223,9 +4409,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 16; @@ -4350,9 +4536,9 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 16; @@ -4566,7 +4752,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 8; while (pos < final_pos) { @@ -4675,9 +4861,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 16; @@ -4781,9 +4967,9 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len, if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that // they are ascii uint64_t v1; - ::memcpy(&v1, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t)); uint64_t v2; - ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); + simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); uint64_t v{v1 | v2}; if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 16; @@ -4976,7 +5162,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len, if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that // they are ascii uint64_t v; - ::memcpy(&v, data + pos, sizeof(uint64_t)); + simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t)); if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 8; while (pos < final_pos) { @@ -7884,37 +8070,45 @@ namespace simdutf { inline std::string_view to_string(base64_options options) { switch (options) { case base64_default: - return "base64_default"; + return std::string_view("base64_default", sizeof("base64_default") - 1); case base64_url: - return "base64_url"; + return std::string_view("base64_url", sizeof("base64_url") - 1); case base64_reverse_padding: - return "base64_reverse_padding"; + return std::string_view("base64_reverse_padding", + sizeof("base64_reverse_padding") - 1); case base64_url_with_padding: - return "base64_url_with_padding"; + return std::string_view("base64_url_with_padding", + sizeof("base64_url_with_padding") - 1); case base64_default_accept_garbage: - return "base64_default_accept_garbage"; + return std::string_view("base64_default_accept_garbage", + sizeof("base64_default_accept_garbage") - 1); case base64_url_accept_garbage: - return "base64_url_accept_garbage"; + return std::string_view("base64_url_accept_garbage", + sizeof("base64_url_accept_garbage") - 1); case base64_default_or_url: - return "base64_default_or_url"; + return std::string_view("base64_default_or_url", + sizeof("base64_default_or_url") - 1); case base64_default_or_url_accept_garbage: - return "base64_default_or_url_accept_garbage"; + return std::string_view("base64_default_or_url_accept_garbage", + sizeof("base64_default_or_url_accept_garbage") - + 1); } - return ""; + return std::string_view("", sizeof("") - 1); } inline std::string_view to_string(last_chunk_handling_options options) { switch (options) { case loose: - return "loose"; + return std::string_view("loose", sizeof("loose") - 1); case strict: - return "strict"; + return std::string_view("strict", sizeof("strict") - 1); case stop_before_partial: - return "stop_before_partial"; + return std::string_view("stop_before_partial", + sizeof("stop_before_partial") - 1); case only_full_chunks: - return "only_full_chunks"; + return std::string_view("only_full_chunks", sizeof("only_full_chunks") - 1); } - return ""; + return std::string_view("", sizeof("") - 1); } /** @@ -8789,7 +8983,9 @@ public: * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ - virtual std::string_view name() const noexcept { return _name; } + virtual std::string_view name() const noexcept { + return std::string_view(_name, _name_length); + } /** * The description of this implementation. @@ -8800,7 +8996,9 @@ public: * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ - virtual std::string_view description() const noexcept { return _description; } + virtual std::string_view description() const noexcept { + return std::string_view(_description, _description_length); + } /** * The instruction sets this implementation is compiled against @@ -9594,10 +9792,21 @@ public: protected: /** @private Construct an implementation with the given name and description. * For subclasses. */ + template + simdutf_really_inline implementation( + const char (&name)[NameLength], + const char (&description)[DescriptionLength], + uint32_t required_instruction_sets) + : _name(name), _name_length(NameLength - 1), _description(description), + _description_length(DescriptionLength - 1), + _required_instruction_sets(required_instruction_sets) {} + simdutf_really_inline implementation(const char *name, const char *description, uint32_t required_instruction_sets) - : _name(name), _description(description), + : _name(name), _name_length(simdutf::internal::strlen(name)), + _description(description), + _description_length(simdutf::internal::strlen(description)), _required_instruction_sets(required_instruction_sets) {} protected: @@ -9608,11 +9817,13 @@ private: * The name of this implementation. */ const char *_name; + const size_t _name_length; /** * The description of this implementation. */ const char *_description; + const size_t _description_length; /** * Instruction sets required for this implementation. @@ -9623,6 +9834,19 @@ private: /** @private */ namespace internal { +simdutf_really_inline bool string_view_equal(std::string_view lhs, + std::string_view rhs) noexcept { + if (lhs.size() != rhs.size()) { + return false; + } + for (size_t i = 0; i < lhs.size(); i++) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + /** * The list of available implementations compiled into simdutf. */ @@ -9652,7 +9876,24 @@ public: */ const implementation *operator[](std::string_view name) const noexcept { for (const implementation *impl : *this) { - if (impl->name() == name) { + if (string_view_equal(impl->name(), name)) { + return impl; + } + } + return nullptr; + } + + const implementation *operator[](const char *name) const noexcept { + if (name == nullptr) { + return nullptr; + } + for (const implementation *impl : *this) { + const std::string_view impl_name = impl->name(); + size_t i = 0; + while (i < impl_name.size() && name[i] != '\0' && impl_name[i] == name[i]) { + i++; + } + if (i == impl_name.size() && name[i] == '\0') { return impl; } }