pkg/simdutf: no_libc mode

This commit is contained in:
Mitchell Hashimoto
2026-04-24 06:35:16 -07:00
parent 48ccec182a
commit a6d5f2ea23
4 changed files with 465 additions and 146 deletions

View File

@@ -4,6 +4,7 @@ pub fn build(b: *std.Build) !void {
const optimize = b.standardOptimizeOption(.{});
const target = b.standardTargetOptions(.{});
const no_libcxx = b.option(bool, "no_libcxx", "Set SIMDUTF_NO_LIBCXX to avoid libc++ dependency") orelse false;
const no_libc = b.option(bool, "no_libc", "Set SIMDUTF_NO_LIBC and provide Zig stdlib replacements") orelse false;
const lib = b.addLibrary(.{
.name = "simdutf",
@@ -64,6 +65,31 @@ pub fn build(b: *std.Build) !void {
lib.root_module.addCMacro("SIMDUTF_NO_LIBCXX", "1");
}
if (no_libc) {
try flags.appendSlice(b.allocator, &.{
"-DSIMDUTF_NO_LIBC=1",
"-DSIMDUTF_LIBC_MEMCPY=simdutf_memcpy",
"-DSIMDUTF_LIBC_MEMMOVE=simdutf_memmove",
"-DSIMDUTF_LIBC_MEMSET=simdutf_memset",
"-DSIMDUTF_LIBC_MEMCMP=simdutf_memcmp",
"-DSIMDUTF_LIBC_STRLEN=simdutf_strlen",
"-DSIMDUTF_LIBC_GETENV=simdutf_getenv",
});
lib.root_module.addCMacro("SIMDUTF_NO_LIBC", "1");
const no_libc_obj = b.addObject(.{
.name = "simdutf_no_libc",
.root_module = b.createModule(.{
.root_source_file = b.path("no_libc.zig"),
.target = target,
.optimize = optimize,
.link_libc = false,
}),
});
lib.addObject(no_libc_obj);
}
if (target.result.abi == .msvc) {
// On MSVC we skip linkLibCpp (see above), so the C++ standard is
// not set implicitly. simdutf requires C++17, so set it explicitly.

47
pkg/simdutf/no_libc.zig Normal file
View File

@@ -0,0 +1,47 @@
const std = @import("std");
export fn simdutf_memcpy(noalias dest: ?[*]u8, noalias src: ?[*]const u8, n: usize) ?[*]u8 {
const d = dest orelse return dest;
const s = src orelse return dest;
@memcpy(d[0..n], s[0..n]);
return dest;
}
export fn simdutf_memmove(dest: ?[*]u8, src: ?[*]const u8, n: usize) ?[*]u8 {
const d = dest orelse return dest;
const s = src orelse return dest;
const dst_slice = d[0..n];
const src_slice = s[0..n];
if (@intFromPtr(d) <= @intFromPtr(s)) {
@memcpy(dst_slice, src_slice);
} else {
std.mem.copyBackwards(u8, dst_slice, src_slice);
}
return dest;
}
export fn simdutf_memset(dest: ?[*]u8, c: c_int, n: usize) ?[*]u8 {
const d = dest orelse return dest;
@memset(d[0..n], @as(u8, @intCast(c & 0xff)));
return dest;
}
export fn simdutf_memcmp(lhs: ?[*]const u8, rhs: ?[*]const u8, n: usize) c_int {
const l = lhs orelse return 0;
const r = rhs orelse return 0;
const order = std.mem.order(u8, l[0..n], r[0..n]);
return switch (order) {
.lt => -1,
.eq => 0,
.gt => 1,
};
}
export fn simdutf_strlen(s: ?[*:0]const u8) usize {
const str = s orelse return 0;
return std.mem.len(str);
}
export fn simdutf_getenv(_: ?[*:0]const u8) ?[*:0]const u8 {
return null;
}

View File

@@ -1,4 +1,4 @@
/* auto-generated on 2026-04-21 21:46:47 -0400. Do not edit! */
/* auto-generated on 2026-04-23 21:18:04 -0700. Do not edit! */
/* begin file src/simdutf.cpp */
#include "simdutf.h"
@@ -7,19 +7,23 @@ namespace simdutf {
std::string_view to_string(encoding_type bom) {
switch (bom) {
case UTF16_LE:
return "UTF16 little-endian";
return std::string_view("UTF16 little-endian",
sizeof("UTF16 little-endian") - 1);
case UTF16_BE:
return "UTF16 big-endian";
return std::string_view("UTF16 big-endian",
sizeof("UTF16 big-endian") - 1);
case UTF32_LE:
return "UTF32 little-endian";
return std::string_view("UTF32 little-endian",
sizeof("UTF32 little-endian") - 1);
case UTF32_BE:
return "UTF32 big-endian";
return std::string_view("UTF32 big-endian",
sizeof("UTF32 big-endian") - 1);
case UTF8:
return "UTF8";
return std::string_view("UTF8", sizeof("UTF8") - 1);
case unspecified:
return "unknown";
return std::string_view("unknown", sizeof("unknown") - 1);
default:
return "error";
return std::string_view("error", sizeof("error") - 1);
}
}
@@ -11308,7 +11312,8 @@ detect_best_supported_implementation_on_first_use::set_best() const noexcept {
SIMDUTF_PUSH_DISABLE_WARNINGS
SIMDUTF_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC:
// manually verified this is safe
char *force_implementation_name = getenv("SIMDUTF_FORCE_IMPLEMENTATION");
const char *force_implementation_name =
simdutf::internal::getenv("SIMDUTF_FORCE_IMPLEMENTATION");
SIMDUTF_POP_DISABLE_WARNINGS
if (force_implementation_name) {
@@ -12500,7 +12505,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen,
if (offset + 64 > line_length) {
size_t location_end = line_length - offset;
size_t to_move = 64 - location_end;
std::memmove(out + location_end + 1, out + location_end, to_move);
simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move);
out[location_end] = '\n';
offset = to_move;
out += 64 + 1;
@@ -12550,7 +12555,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen,
if (offset + 32 > line_length) {
size_t location_end = line_length - offset;
size_t to_move = 32 - location_end;
std::memmove(out + location_end + 1, out + location_end, to_move);
simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move);
out[location_end] = '\n';
offset = to_move;
out += 32 + 1;
@@ -13010,7 +13015,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen,
base64_decode_block(dst, buffer + i * 64);
dst += 48;
}
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -13048,7 +13053,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 4);
simdutf::internal::memcpy(dst, &triple, 4);
dst += 3;
buffer_start += 4;
@@ -13062,7 +13067,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -13956,11 +13961,11 @@ simdutf_really_inline size_t
buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
if (len == idx) {
return 0;
} // memcpy(dst, null, 0) will trigger an error with some sanitizers
std::memset(dst, 0x20,
} // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers
simdutf::internal::memset(dst, 0x20,
STEP_SIZE); // std::memset STEP_SIZE because it is more efficient
// to write out 8 or 16 bytes at once.
std::memcpy(dst, buf + idx, len - idx);
simdutf::internal::memcpy(dst, buf + idx, len - idx);
return len - idx;
}
@@ -17091,7 +17096,7 @@ valid_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) {
int valid_count2;
__m512i vec2 = expand_and_identify(lane2, lane3, valid_count2);
uint32_t tmp1;
::memcpy(&tmp1, ptr + 64, sizeof(tmp1));
simdutf::internal::memcpy(&tmp1, ptr + 64, sizeof(tmp1));
const __m512i lane4 = _mm512_set1_epi32(tmp1);
int valid_count3;
__m512i vec3 = expand_and_identify(lane3, lane4, valid_count3);
@@ -17213,7 +17218,7 @@ validating_utf8_to_fixed_length(const char *str, size_t len, OUTPUT *dwords) {
int valid_count2;
__m512i vec2 = expand_and_identify(lane2, lane3, valid_count2);
uint32_t tmp1;
::memcpy(&tmp1, ptr + 64, sizeof(tmp1));
simdutf::internal::memcpy(&tmp1, ptr + 64, sizeof(tmp1));
const __m512i lane4 = _mm512_set1_epi32(tmp1);
int valid_count3;
__m512i vec3 = expand_and_identify(lane3, lane4, valid_count3);
@@ -17345,7 +17350,7 @@ validating_utf8_to_fixed_length_with_constant_checks(const char *str,
int valid_count2;
__m512i vec2 = expand_and_identify(lane2, lane3, valid_count2);
uint32_t tmp1;
::memcpy(&tmp1, ptr + 64, sizeof(tmp1));
simdutf::internal::memcpy(&tmp1, ptr + 64, sizeof(tmp1));
const __m512i lane4 = _mm512_set1_epi32(tmp1);
int valid_count3;
__m512i vec3 = expand_and_identify(lane3, lane4, valid_count3);
@@ -18856,7 +18861,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
base64_decode_block(dst, buffer + i * 64);
dst += 48;
}
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -21541,7 +21546,7 @@ avx2_encode_base64_impl(char *dst, const char *src, size_t srclen,
if (offset + 32 > line_length) {
size_t location_end = line_length - offset;
size_t to_move = 32 - location_end;
std::memmove(out + location_end + 1, out + location_end, to_move);
simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move);
out[location_end] = '\n';
offset = to_move;
out += 32 + 1;
@@ -21554,7 +21559,7 @@ avx2_encode_base64_impl(char *dst, const char *src, size_t srclen,
alignas(32) uint8_t buffer[32];
_mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer),
lookup_pshufb_improved<isbase64url>(indices));
std::memcpy(out, buffer, 32);
simdutf::internal::memcpy(out, buffer, 32);
size_t out_pos = 0;
size_t local_offset = offset;
for (size_t j = 0; j < 32;) {
@@ -21664,7 +21669,7 @@ simdutf_really_inline void base64_decode_block_safe(char *out,
alignas(32) char buffer[32]; // We enforce safety with a buffer.
base64_decode(
buffer, _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src + 32)));
std::memcpy(out + 24, buffer, 24);
simdutf::internal::memcpy(out + 24, buffer, 24);
}
// --- decoding - base64 class --------------------------------
@@ -21715,7 +21720,7 @@ public:
base64_decode(out, chunks[0]);
alignas(32) char buffer[32]; // We enforce safety with a buffer.
base64_decode(buffer, chunks[1]);
std::memcpy(out + 24, buffer, 24);
simdutf::internal::memcpy(out + 24, buffer, 24);
}
template <bool base64_url, bool ignore_garbage, bool default_or_url>
@@ -22072,11 +22077,11 @@ simdutf_really_inline size_t
buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
if (len == idx) {
return 0;
} // memcpy(dst, null, 0) will trigger an error with some sanitizers
std::memset(dst, 0x20,
} // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers
simdutf::internal::memset(dst, 0x20,
STEP_SIZE); // std::memset STEP_SIZE because it is more efficient
// to write out 8 or 16 bytes at once.
std::memcpy(dst, buf + idx, len - idx);
simdutf::internal::memcpy(dst, buf + idx, len - idx);
return len - idx;
}
@@ -23669,7 +23674,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
base64_decode_block(dst, buffer + (block_size - 2) * 64);
}
dst += 48;
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -23713,7 +23718,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -23727,7 +23732,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -25607,11 +25612,11 @@ size_t convert_masked_utf8_to_latin1(const char *input,
#if defined(__clang__)
__attribute__((aligned(16))) char buf[16];
latin1_packed.store(buf);
memcpy(latin1_output, buf, 6);
simdutf::internal::memcpy(latin1_output, buf, 6);
#else
// writing 8 bytes even though we only care about the first 6 bytes.
const auto tmp = vec_u64_t(latin1_packed.value);
memcpy(latin1_output, &tmp[0], 8);
simdutf::internal::memcpy(latin1_output, &tmp[0], 8);
#endif
latin1_output += 6; // We wrote 6 bytes.
return consumed;
@@ -26661,7 +26666,7 @@ static simdutf_really_inline void base64_decode_block_safe(char *out,
char buffer[16];
base64_decode(buffer, vector_u8::load(src + 3 * 16));
std::memcpy(out + 36, buffer, 12);
simdutf::internal::memcpy(out + 36, buffer, 12);
}
// ---base64 decoding::block64 class --------------------------
@@ -26822,7 +26827,7 @@ public:
base64_decode(out + 12 * 2, b.chunks[2]);
char buffer[16];
base64_decode(buffer, b.chunks[3]);
std::memcpy(out + 12 * 3, buffer, 12);
simdutf::internal::memcpy(out + 12 * 3, buffer, 12);
}
};
/* end file src/ppc64/ppc64_base64.cpp */
@@ -26891,11 +26896,11 @@ simdutf_really_inline size_t
buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
if (len == idx) {
return 0;
} // memcpy(dst, null, 0) will trigger an error with some sanitizers
std::memset(dst, 0x20,
} // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers
simdutf::internal::memset(dst, 0x20,
STEP_SIZE); // std::memset STEP_SIZE because it is more efficient
// to write out 8 or 16 bytes at once.
std::memcpy(dst, buf + idx, len - idx);
simdutf::internal::memcpy(dst, buf + idx, len - idx);
return len - idx;
}
@@ -28485,7 +28490,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
base64_decode_block(dst, buffer + (block_size - 2) * 64);
}
dst += 48;
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -28529,7 +28534,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -28543,7 +28548,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -31461,7 +31466,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen,
_mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 16), t1);
_mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 32), t2);
_mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 48), t3);
std::memcpy(out, buffer, 64);
simdutf::internal::memcpy(out, buffer, 64);
size_t out_pos = 0;
size_t local_offset = offset;
for (size_t j = 0; j < 64;) {
@@ -31529,7 +31534,7 @@ size_t encode_base64_impl(char *dst, const char *src, size_t srclen,
if (offset + 16 > line_length) {
size_t location_end = line_length - offset;
size_t to_move = 16 - location_end;
std::memmove(out + location_end + 1, out + location_end, to_move);
simdutf::internal::memmove(out + location_end + 1, out + location_end, to_move);
out[location_end] = '\n';
offset = to_move;
out += 16 + 1;
@@ -31641,7 +31646,7 @@ static inline void base64_decode_block_safe(char *out, const char *src) {
char buffer[16];
base64_decode(buffer,
_mm_loadu_si128(reinterpret_cast<const __m128i *>(src + 48)));
std::memcpy(out + 36, buffer, 12);
simdutf::internal::memcpy(out + 36, buffer, 12);
}
// --- decoding - base64 class --------------------------------
@@ -31909,7 +31914,7 @@ public:
base64_decode(out + 24, chunks[2]);
char buffer[16];
base64_decode(buffer, chunks[3]);
std::memcpy(out + 36, buffer, 12);
simdutf::internal::memcpy(out + 36, buffer, 12);
}
};
/* end file src/westmere/sse_base64.cpp */
@@ -31978,11 +31983,11 @@ simdutf_really_inline size_t
buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
if (len == idx) {
return 0;
} // memcpy(dst, null, 0) will trigger an error with some sanitizers
std::memset(dst, 0x20,
} // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers
simdutf::internal::memset(dst, 0x20,
STEP_SIZE); // std::memset STEP_SIZE because it is more efficient
// to write out 8 or 16 bytes at once.
std::memcpy(dst, buf + idx, len - idx);
simdutf::internal::memcpy(dst, buf + idx, len - idx);
return len - idx;
}
@@ -33570,7 +33575,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
base64_decode_block(dst, buffer + (block_size - 2) * 64);
}
dst += 48;
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -33614,7 +33619,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -33628,7 +33633,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
#if !SIMDUTF_IS_BIG_ENDIAN
triple = scalar::u32_swap_bytes(triple);
#endif
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -36050,7 +36055,7 @@ static inline void base64_decode_block_safe(char *out, const char *src) {
alignas(32) char buffer[32];
base64_decode(buffer,
__lasx_xvld(reinterpret_cast<const __m256i *>(src), 32));
std::memcpy(out + 24, buffer, 24);
simdutf::internal::memcpy(out + 24, buffer, 24);
}
static inline void base64_decode_block(char *out, block64 *b) {
@@ -36061,7 +36066,7 @@ static inline void base64_decode_block_safe(char *out, block64 *b) {
base64_decode(out, b->chunks[0]);
alignas(32) char buffer[32];
base64_decode(buffer, b->chunks[1]);
std::memcpy(out + 24, buffer, 24);
simdutf::internal::memcpy(out + 24, buffer, 24);
}
template <bool base64_url, bool ignore_garbage, bool default_or_url,
@@ -36142,7 +36147,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
base64_decode_block(dst, buffer + (block_size - 2) * 64);
}
dst += 48;
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -36185,7 +36190,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
<< 8;
// lasx is little-endian
triple = scalar::u32_swap_bytes(triple);
std::memcpy(dst, &triple, 4);
simdutf::internal::memcpy(dst, &triple, 4);
dst += 3;
buffer_start += 4;
@@ -36198,7 +36203,7 @@ compress_decode_base64(char *dst, const chartype *src, size_t srclen,
<< 8;
// lasx is little-endian
triple = scalar::u32_swap_bytes(triple);
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -36388,11 +36393,11 @@ simdutf_really_inline size_t
buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
if (len == idx) {
return 0;
} // memcpy(dst, null, 0) will trigger an error with some sanitizers
std::memset(dst, 0x20,
} // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers
simdutf::internal::memset(dst, 0x20,
STEP_SIZE); // std::memset STEP_SIZE because it is more efficient
// to write out 8 or 16 bytes at once.
std::memcpy(dst, buf + idx, len - idx);
simdutf::internal::memcpy(dst, buf + idx, len - idx);
return len - idx;
}
@@ -38908,7 +38913,7 @@ size_t convert_masked_utf8_to_latin1(const char *input,
uint64_t buffer[2];
// __lsx_vst(latin1_packed, reinterpret_cast<uint8_t *>(latin1_output), 0);
__lsx_vst(latin1_packed, reinterpret_cast<uint8_t *>(buffer), 0);
std::memcpy(latin1_output, buffer, 6);
simdutf::internal::memcpy(latin1_output, buffer, 6);
latin1_output += 6; // We wrote 6 bytes.
return consumed;
}
@@ -40011,7 +40016,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen,
base64_decode_block(dst, buffer + i * 64);
dst += 48;
}
std::memcpy(buffer, buffer + (block_size - 1) * 64,
simdutf::internal::memcpy(buffer, buffer + (block_size - 1) * 64,
64); // 64 might be too much
bufferptr -= (block_size - 1) * 64;
}
@@ -40048,7 +40053,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen,
<< 8;
// lsx is little-endian
triple = scalar::u32_swap_bytes(triple);
std::memcpy(dst, &triple, 4);
simdutf::internal::memcpy(dst, &triple, 4);
dst += 3;
buffer_start += 4;
@@ -40061,7 +40066,7 @@ compress_decode_base64(char *dst, const char_type *src, size_t srclen,
<< 8;
// lsx is little-endian
triple = scalar::u32_swap_bytes(triple);
std::memcpy(dst, &triple, 3);
simdutf::internal::memcpy(dst, &triple, 3);
dst += 3;
buffer_start += 4;
@@ -40247,11 +40252,11 @@ simdutf_really_inline size_t
buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
if (len == idx) {
return 0;
} // memcpy(dst, null, 0) will trigger an error with some sanitizers
std::memset(dst, 0x20,
} // simdutf::internal::memcpy(dst, null, 0) will trigger an error with some sanitizers
simdutf::internal::memset(dst, 0x20,
STEP_SIZE); // std::memset STEP_SIZE because it is more efficient
// to write out 8 or 16 bytes at once.
std::memcpy(dst, buf + idx, len - idx);
simdutf::internal::memcpy(dst, buf + idx, len - idx);
return len - idx;
}

View File

@@ -1,4 +1,4 @@
/* auto-generated on 2026-04-21 21:46:47 -0400. Do not edit! */
/* auto-generated on 2026-04-23 21:18:04 -0700. Do not edit! */
/* begin file include/simdutf.h */
#ifndef SIMDUTF_H
#define SIMDUTF_H
@@ -556,6 +556,188 @@
#define simdutf_constexpr23
#endif
/* begin file include/simdutf/libc.h */
#ifndef SIMDUTF_LIBC_H
#define SIMDUTF_LIBC_H
#include <cstddef>
#include <cstdlib>
#include <cstring>
#ifndef SIMDUTF_NO_LIBC
#define SIMDUTF_NO_LIBC 0
#endif
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_MEMCPY
extern "C" void *SIMDUTF_LIBC_MEMCPY(void *, const void *, size_t) noexcept;
#endif
#ifdef SIMDUTF_LIBC_MEMMOVE
extern "C" void *SIMDUTF_LIBC_MEMMOVE(void *, const void *, size_t) noexcept;
#endif
#ifdef SIMDUTF_LIBC_MEMSET
extern "C" void *SIMDUTF_LIBC_MEMSET(void *, int, size_t) noexcept;
#endif
#ifdef SIMDUTF_LIBC_MEMCMP
extern "C" int SIMDUTF_LIBC_MEMCMP(const void *, const void *, size_t) noexcept;
#endif
#ifdef SIMDUTF_LIBC_STRLEN
extern "C" size_t SIMDUTF_LIBC_STRLEN(const char *) noexcept;
#endif
#ifdef SIMDUTF_LIBC_GETENV
extern "C" const char *SIMDUTF_LIBC_GETENV(const char *) noexcept;
#endif
#endif
namespace simdutf {
namespace internal {
simdutf_really_inline void *builtin_memcpy(void *dst, const void *src,
size_t count) noexcept {
auto *output = reinterpret_cast<unsigned char *>(dst);
const auto *input = reinterpret_cast<const unsigned char *>(src);
for (size_t i = 0; i < count; i++) {
output[i] = input[i];
}
return dst;
}
simdutf_really_inline void *builtin_memmove(void *dst, const void *src,
size_t count) noexcept {
auto *output = reinterpret_cast<unsigned char *>(dst);
const auto *input = reinterpret_cast<const unsigned char *>(src);
if (output == input || count == 0) {
return dst;
}
if (output < input || output >= input + count) {
for (size_t i = 0; i < count; i++) {
output[i] = input[i];
}
return dst;
}
for (size_t i = count; i != 0; i--) {
output[i - 1] = input[i - 1];
}
return dst;
}
simdutf_really_inline void *builtin_memset(void *dst, int value,
size_t count) noexcept {
auto *output = reinterpret_cast<unsigned char *>(dst);
const auto byte = static_cast<unsigned char>(value);
for (size_t i = 0; i < count; i++) {
output[i] = byte;
}
return dst;
}
simdutf_really_inline int builtin_memcmp(const void *lhs, const void *rhs,
size_t count) noexcept {
const auto *left = reinterpret_cast<volatile const unsigned char *>(lhs);
const auto *right = reinterpret_cast<volatile const unsigned char *>(rhs);
for (size_t i = 0; i < count; i++) {
if (left[i] != right[i]) {
return int(left[i]) - int(right[i]);
}
}
return 0;
}
simdutf_really_inline size_t builtin_strlen(const char *input) noexcept {
const auto *bytes = reinterpret_cast<volatile const unsigned char *>(input);
size_t length = 0;
while (bytes[length] != '\0') {
length++;
}
return length;
}
simdutf_really_inline void *memcpy(void *dst, const void *src,
size_t count) noexcept {
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_MEMCPY
return SIMDUTF_LIBC_MEMCPY(dst, src, count);
#else
return builtin_memcpy(dst, src, count);
#endif
#else
return std::memcpy(dst, src, count);
#endif
}
simdutf_really_inline void *memmove(void *dst, const void *src,
size_t count) noexcept {
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_MEMMOVE
return SIMDUTF_LIBC_MEMMOVE(dst, src, count);
#else
return builtin_memmove(dst, src, count);
#endif
#else
return std::memmove(dst, src, count);
#endif
}
simdutf_really_inline void *memset(void *dst, int value,
size_t count) noexcept {
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_MEMSET
return SIMDUTF_LIBC_MEMSET(dst, value, count);
#else
return builtin_memset(dst, value, count);
#endif
#else
return std::memset(dst, value, count);
#endif
}
simdutf_really_inline int memcmp(const void *lhs, const void *rhs,
size_t count) noexcept {
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_MEMCMP
return SIMDUTF_LIBC_MEMCMP(lhs, rhs, count);
#else
return builtin_memcmp(lhs, rhs, count);
#endif
#else
return std::memcmp(lhs, rhs, count);
#endif
}
simdutf_really_inline size_t strlen(const char *input) noexcept {
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_STRLEN
return SIMDUTF_LIBC_STRLEN(input);
#else
return builtin_strlen(input);
#endif
#else
return std::strlen(input);
#endif
}
simdutf_really_inline const char *getenv(const char *name) noexcept {
#if SIMDUTF_NO_LIBC
#ifdef SIMDUTF_LIBC_GETENV
return SIMDUTF_LIBC_GETENV(name);
#else
(void)name;
return nullptr;
#endif
#else
SIMDUTF_PUSH_DISABLE_WARNINGS
SIMDUTF_DISABLE_DEPRECATED_WARNING
return std::getenv(name);
SIMDUTF_POP_DISABLE_WARNINGS
#endif
}
} // namespace internal
} // namespace simdutf
#endif // SIMDUTF_LIBC_H
/* end file include/simdutf/libc.h */
#ifndef SIMDUTF_DLLIMPORTEXPORT
#if defined(SIMDUTF_VISUAL_STUDIO) // Visual Studio
/**
@@ -842,29 +1024,33 @@ enum error_code {
inline std::string_view error_to_string(error_code code) noexcept {
switch (code) {
case SUCCESS:
return "SUCCESS";
return std::string_view("SUCCESS", sizeof("SUCCESS") - 1);
case HEADER_BITS:
return "HEADER_BITS";
return std::string_view("HEADER_BITS", sizeof("HEADER_BITS") - 1);
case TOO_SHORT:
return "TOO_SHORT";
return std::string_view("TOO_SHORT", sizeof("TOO_SHORT") - 1);
case TOO_LONG:
return "TOO_LONG";
return std::string_view("TOO_LONG", sizeof("TOO_LONG") - 1);
case OVERLONG:
return "OVERLONG";
return std::string_view("OVERLONG", sizeof("OVERLONG") - 1);
case TOO_LARGE:
return "TOO_LARGE";
return std::string_view("TOO_LARGE", sizeof("TOO_LARGE") - 1);
case SURROGATE:
return "SURROGATE";
return std::string_view("SURROGATE", sizeof("SURROGATE") - 1);
case INVALID_BASE64_CHARACTER:
return "INVALID_BASE64_CHARACTER";
return std::string_view("INVALID_BASE64_CHARACTER",
sizeof("INVALID_BASE64_CHARACTER") - 1);
case BASE64_INPUT_REMAINDER:
return "BASE64_INPUT_REMAINDER";
return std::string_view("BASE64_INPUT_REMAINDER",
sizeof("BASE64_INPUT_REMAINDER") - 1);
case BASE64_EXTRA_BITS:
return "BASE64_EXTRA_BITS";
return std::string_view("BASE64_EXTRA_BITS",
sizeof("BASE64_EXTRA_BITS") - 1);
case OUTPUT_BUFFER_TOO_SMALL:
return "OUTPUT_BUFFER_TOO_SMALL";
return std::string_view("OUTPUT_BUFFER_TOO_SMALL",
sizeof("OUTPUT_BUFFER_TOO_SMALL") - 1);
default:
return "OTHER";
return std::string_view("OTHER", sizeof("OTHER") - 1);
}
}
@@ -1604,9 +1790,9 @@ simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data,
{
for (; pos + 16 <= len; pos += 16) {
uint64_t v1;
std::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) != 0) {
return false;
@@ -1637,9 +1823,9 @@ validate_with_errors(InputPtr data, size_t len) noexcept {
// process in blocks of 16 bytes when possible
for (; pos + 16 <= len; pos += 16) {
uint64_t v1;
std::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) != 0) {
for (; pos < len; pos++) {
@@ -1710,7 +1896,7 @@ inline void memcpy_atomic_read(char *dst, const char *src, size_t len) {
auto *src_aligned = reinterpret_cast<uint64_t *>(const_cast<char *>(src));
const auto dst_value =
std::atomic_ref<uint64_t>(*src_aligned).load(std::memory_order_relaxed);
std::memcpy(dst, &dst_value, sizeof(uint64_t));
simdutf::internal::memcpy(dst, &dst_value, sizeof(uint64_t));
src += alignment;
dst += alignment;
len -= alignment;
@@ -1756,7 +1942,7 @@ inline void memcpy_atomic_write(char *dst, const char *src, size_t len) {
while (len >= alignment) {
auto *dst_aligned = reinterpret_cast<uint64_t *>(dst);
uint64_t src_val;
std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src
simdutf::internal::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src
std::atomic_ref<uint64_t>(*dst_aligned)
.store(src_val, std::memory_order_relaxed);
dst += alignment;
@@ -1914,9 +2100,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 |
v2}; // We are only interested in these bits: 1000 1000 1000
// 1000, so it makes sense to concatenate everything
@@ -1966,16 +2152,16 @@ inline size_t convert_safe(const char *buf, size_t len, char *utf8_output,
utf8_pos + 16 <= utf8_len) { // if it is safe to read 16 more bytes,
// check that they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 |
v2}; // We are only interested in these bits: 1000 1000 1000
// 1000, so it makes sense to concatenate everything
if ((v & 0x8080808080808080) ==
0) { // if NONE of these are set, e.g. all of them are zero, then
// everything is ASCII
::memcpy(utf8_output + utf8_pos, buf + pos, 16);
simdutf::internal::memcpy(utf8_output + utf8_pos, buf + pos, 16);
utf8_pos += 16;
pos += 16;
} else {
@@ -2050,18 +2236,18 @@ utf8_length_from_latin1(InputPtr input, size_t length) noexcept {
};
for (; i + 32 <= length; i += 32) {
uint64_t v;
memcpy(&v, input + i, 8);
simdutf::internal::memcpy(&v, input + i, 8);
answer += pop(v);
memcpy(&v, input + i + 8, sizeof(v));
simdutf::internal::memcpy(&v, input + i + 8, sizeof(v));
answer += pop(v);
memcpy(&v, input + i + 16, sizeof(v));
simdutf::internal::memcpy(&v, input + i + 16, sizeof(v));
answer += pop(v);
memcpy(&v, input + i + 24, sizeof(v));
simdutf::internal::memcpy(&v, input + i + 24, sizeof(v));
answer += pop(v);
}
for (; i + 8 <= length; i += 8) {
uint64_t v;
memcpy(&v, input + i, sizeof(v));
simdutf::internal::memcpy(&v, input + i, sizeof(v));
answer += pop(v);
}
} // !consteval scope
@@ -2365,10 +2551,10 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 32 more bytes, check that
// they are Latin1
uint64_t v1, v2, v3, v4;
::memcpy(&v1, data + pos, sizeof(uint64_t));
::memcpy(&v2, data + pos + 4, sizeof(uint64_t));
::memcpy(&v3, data + pos + 8, sizeof(uint64_t));
::memcpy(&v4, data + pos + 12, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + 4, sizeof(uint64_t));
simdutf::internal::memcpy(&v3, data + pos + 8, sizeof(uint64_t));
simdutf::internal::memcpy(&v4, data + pos + 12, sizeof(uint64_t));
if constexpr (!match_system(big_endian)) {
v1 = (v1 >> 8) | (v1 << (64 - 8));
@@ -2617,7 +2803,7 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if constexpr (!match_system(big_endian)) {
v = (v >> 8) | (v << (64 - 8));
}
@@ -2707,7 +2893,7 @@ simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len,
if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if constexpr (!match_system(big_endian))
v = (v >> 8) | (v << (64 - 8));
if ((v & 0xFF80FF80FF80FF80) == 0) {
@@ -2814,7 +3000,7 @@ simdutf_constexpr23 size_t convert_with_replacement(const char16_t *data,
if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if constexpr (!match_system(big_endian)) {
v = (v >> 8) | (v << (64 - 8));
}
@@ -2914,7 +3100,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if constexpr (!match_system(big_endian)) {
v = (v >> 8) | (v << (64 - 8));
}
@@ -3105,7 +3291,7 @@ inline simdutf_constexpr23 result convert_with_errors(const char32_t *data,
if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
// they are Latin1
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0xFFFFFF00FFFFFF00) == 0) {
*latin1_output++ = char(data[pos]);
*latin1_output++ = char(data[pos + 1]);
@@ -3165,7 +3351,7 @@ simdutf_constexpr23 size_t convert_valid(ReadPtr data, size_t len,
if (pos + 2 <= len) {
// if it is safe to read 8 more bytes, check that they are Latin1
uint64_t v;
std::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0xFFFFFF00FFFFFF00) == 0) {
*latin1_output++ = char(data[pos]);
*latin1_output++ = char(data[pos + 1]);
@@ -3361,7 +3547,7 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0xFFFFFF80FFFFFF80) == 0) {
*utf8_output++ = char(data[pos]);
*utf8_output++ = char(data[pos + 1]);
@@ -3425,7 +3611,7 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0xFFFFFF80FFFFFF80) == 0) {
*utf8_output++ = char(data[pos]);
*utf8_output++ = char(data[pos + 1]);
@@ -3505,7 +3691,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0xFFFFFF80FFFFFF80) == 0) {
*utf8_output++ = char(data[pos]);
*utf8_output++ = char(data[pos + 1]);
@@ -3581,9 +3767,9 @@ simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data,
if (next_pos <= len) { // if it is safe to read 16 more bytes, check
// that they are ascii
uint64_t v1{};
std::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2{};
std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) == 0) {
pos = next_pos;
@@ -3682,9 +3868,9 @@ validate_with_errors(BytePtr data, size_t len) noexcept {
if (next_pos <=
len) { // if it is safe to read 16 more bytes, check that they are ascii
uint64_t v1;
std::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) == 0) {
pos = next_pos;
@@ -3909,9 +4095,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000
// 1000 1000 .... etc
if ((v & 0x8080808080808080) ==
@@ -3984,9 +4170,9 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000
// 1000 1000...etc
if ((v & 0x8080808080808080) ==
@@ -4136,9 +4322,9 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 |
v2}; // We are only interested in these bits: 1000 1000 1000
// 1000, so it makes sense to concatenate everything
@@ -4223,9 +4409,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) == 0) {
size_t final_pos = pos + 16;
@@ -4350,9 +4536,9 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) == 0) {
size_t final_pos = pos + 16;
@@ -4566,7 +4752,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0x8080808080808080) == 0) {
size_t final_pos = pos + 8;
while (pos < final_pos) {
@@ -4675,9 +4861,9 @@ simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) == 0) {
size_t final_pos = pos + 16;
@@ -4781,9 +4967,9 @@ simdutf_constexpr23 result convert_with_errors(InputPtr data, size_t len,
if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that
// they are ascii
uint64_t v1;
::memcpy(&v1, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v1, data + pos, sizeof(uint64_t));
uint64_t v2;
::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
simdutf::internal::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
uint64_t v{v1 | v2};
if ((v & 0x8080808080808080) == 0) {
size_t final_pos = pos + 16;
@@ -4976,7 +5162,7 @@ simdutf_constexpr23 size_t convert_valid(InputPtr data, size_t len,
if (pos + 8 <= len) { // if it is safe to read 8 more bytes, check that
// they are ascii
uint64_t v;
::memcpy(&v, data + pos, sizeof(uint64_t));
simdutf::internal::memcpy(&v, data + pos, sizeof(uint64_t));
if ((v & 0x8080808080808080) == 0) {
size_t final_pos = pos + 8;
while (pos < final_pos) {
@@ -7884,37 +8070,45 @@ namespace simdutf {
inline std::string_view to_string(base64_options options) {
switch (options) {
case base64_default:
return "base64_default";
return std::string_view("base64_default", sizeof("base64_default") - 1);
case base64_url:
return "base64_url";
return std::string_view("base64_url", sizeof("base64_url") - 1);
case base64_reverse_padding:
return "base64_reverse_padding";
return std::string_view("base64_reverse_padding",
sizeof("base64_reverse_padding") - 1);
case base64_url_with_padding:
return "base64_url_with_padding";
return std::string_view("base64_url_with_padding",
sizeof("base64_url_with_padding") - 1);
case base64_default_accept_garbage:
return "base64_default_accept_garbage";
return std::string_view("base64_default_accept_garbage",
sizeof("base64_default_accept_garbage") - 1);
case base64_url_accept_garbage:
return "base64_url_accept_garbage";
return std::string_view("base64_url_accept_garbage",
sizeof("base64_url_accept_garbage") - 1);
case base64_default_or_url:
return "base64_default_or_url";
return std::string_view("base64_default_or_url",
sizeof("base64_default_or_url") - 1);
case base64_default_or_url_accept_garbage:
return "base64_default_or_url_accept_garbage";
return std::string_view("base64_default_or_url_accept_garbage",
sizeof("base64_default_or_url_accept_garbage") -
1);
}
return "<unknown>";
return std::string_view("<unknown>", sizeof("<unknown>") - 1);
}
inline std::string_view to_string(last_chunk_handling_options options) {
switch (options) {
case loose:
return "loose";
return std::string_view("loose", sizeof("loose") - 1);
case strict:
return "strict";
return std::string_view("strict", sizeof("strict") - 1);
case stop_before_partial:
return "stop_before_partial";
return std::string_view("stop_before_partial",
sizeof("stop_before_partial") - 1);
case only_full_chunks:
return "only_full_chunks";
return std::string_view("only_full_chunks", sizeof("only_full_chunks") - 1);
}
return "<unknown>";
return std::string_view("<unknown>", sizeof("<unknown>") - 1);
}
/**
@@ -8789,7 +8983,9 @@ public:
*
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
*/
virtual std::string_view name() const noexcept { return _name; }
virtual std::string_view name() const noexcept {
return std::string_view(_name, _name_length);
}
/**
* The description of this implementation.
@@ -8800,7 +8996,9 @@ public:
*
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
*/
virtual std::string_view description() const noexcept { return _description; }
virtual std::string_view description() const noexcept {
return std::string_view(_description, _description_length);
}
/**
* The instruction sets this implementation is compiled against
@@ -9594,10 +9792,21 @@ public:
protected:
/** @private Construct an implementation with the given name and description.
* For subclasses. */
template <size_t NameLength, size_t DescriptionLength>
simdutf_really_inline implementation(
const char (&name)[NameLength],
const char (&description)[DescriptionLength],
uint32_t required_instruction_sets)
: _name(name), _name_length(NameLength - 1), _description(description),
_description_length(DescriptionLength - 1),
_required_instruction_sets(required_instruction_sets) {}
simdutf_really_inline implementation(const char *name,
const char *description,
uint32_t required_instruction_sets)
: _name(name), _description(description),
: _name(name), _name_length(simdutf::internal::strlen(name)),
_description(description),
_description_length(simdutf::internal::strlen(description)),
_required_instruction_sets(required_instruction_sets) {}
protected:
@@ -9608,11 +9817,13 @@ private:
* The name of this implementation.
*/
const char *_name;
const size_t _name_length;
/**
* The description of this implementation.
*/
const char *_description;
const size_t _description_length;
/**
* Instruction sets required for this implementation.
@@ -9623,6 +9834,19 @@ private:
/** @private */
namespace internal {
simdutf_really_inline bool string_view_equal(std::string_view lhs,
std::string_view rhs) noexcept {
if (lhs.size() != rhs.size()) {
return false;
}
for (size_t i = 0; i < lhs.size(); i++) {
if (lhs[i] != rhs[i]) {
return false;
}
}
return true;
}
/**
* The list of available implementations compiled into simdutf.
*/
@@ -9652,7 +9876,24 @@ public:
*/
const implementation *operator[](std::string_view name) const noexcept {
for (const implementation *impl : *this) {
if (impl->name() == name) {
if (string_view_equal(impl->name(), name)) {
return impl;
}
}
return nullptr;
}
const implementation *operator[](const char *name) const noexcept {
if (name == nullptr) {
return nullptr;
}
for (const implementation *impl : *this) {
const std::string_view impl_name = impl->name();
size_t i = 0;
while (i < impl_name.size() && name[i] != '\0' && impl_name[i] == name[i]) {
i++;
}
if (i == impl_name.size() && name[i] == '\0') {
return impl;
}
}