gb_global BlockingMutex string_buffer_mutex = {}; // NOTE(bill): Used for UTF-8 strings struct String { u8 * text; isize len; u8 const &operator[](isize i) const { GB_ASSERT_MSG(0 <= i && i < len, "[%td]", i); return text[i]; } }; struct String_Iterator { String const &str; isize pos; }; // NOTE(bill): used for printf style arguments #define LIT(x) ((int)(x).len), (x).text #if defined(GB_COMPILER_MSVC) && _MSC_VER < 1700 #define STR_LIT(c_str) make_string(cast(u8 *)c_str, gb_size_of(c_str)-1) #else #define STR_LIT(c_str) String{cast(u8 *)c_str, gb_size_of(c_str)-1} #endif #define str_lit(c_str) STR_LIT(c_str) // NOTE(bill): String16 is only used for Windows due to its file directories struct String16 { u16 * text; isize len; u16 const &operator[](isize i) const { GB_ASSERT_MSG(0 <= i && i < len, "[%td]", i); return text[i]; } }; gb_internal gb_inline String make_string(u8 const *text, isize len) { String s; s.text = cast(u8 *)text; if (len < 0) { len = gb_strlen(cast(char const *)text); } s.len = len; return s; } gb_internal gb_inline String16 make_string16(u16 const *text, isize len) { String16 s; s.text = cast(u16 *)text; s.len = len; return s; } gb_internal isize string16_len(u16 const *s) { if (s == nullptr) { return 0; } u16 const *p = s; while (*p) { p++; } return p - s; } gb_internal gb_inline String make_string_c(char const *text) { return make_string(cast(u8 *)cast(void *)text, gb_strlen(text)); } gb_internal gb_inline String16 make_string16_c(u16 const *text) { return make_string16(text, string16_len(text)); } gb_internal String substring(String const &s, isize lo, isize hi) { isize max = s.len; GB_ASSERT_MSG(lo <= hi && hi <= max, "%td..%td..%td", lo, hi, max); return make_string(s.text+lo, hi-lo); } gb_internal String16 substring(String16 const &s, isize lo, isize hi) { isize max = s.len; GB_ASSERT_MSG(lo <= hi && hi <= max, "%td..%td..%td", lo, hi, max); return make_string16(s.text+lo, hi-lo); } gb_internal char *alloc_cstring(gbAllocator a, String s) { char *c_str = gb_alloc_array(a, char, s.len+1); gb_memmove(c_str, s.text, s.len); c_str[s.len] = '\0'; return c_str; } gb_internal wchar_t *alloc_wstring(gbAllocator a, String16 s) { wchar_t *c_str = gb_alloc_array(a, wchar_t, s.len+1); gb_memmove(c_str, s.text, s.len*2); c_str[s.len] = '\0'; return c_str; } gb_internal gb_inline bool str_eq_ignore_case(String const &a, String const &b) { if (a.len == b.len) { for (isize i = 0; i < a.len; i++) { char x = cast(char)a[i]; char y = cast(char)b[i]; if (gb_char_to_lower(x) != gb_char_to_lower(y)) { return false; } } return true; } return false; } template gb_internal gb_inline bool str_eq_ignore_case(String const &a, char const (&b_)[N]) { if (a.len != N-1) { return false; } String b = {cast(u8 *)b_, N-1}; return str_eq_ignore_case(a, b); } gb_internal void string_to_lower(String *s) { for (isize i = 0; i < s->len; i++) { s->text[i] = gb_char_to_lower(s->text[i]); } } gb_internal int string_compare(String const &a, String const &b) { if (a.text == b.text) { return cast(int)(a.len - b.len); } if (a.text == nullptr) { return -1; } if (b.text == nullptr) { return +1; } uintptr n = gb_min(a.len, b.len); int res = memcmp(a.text, b.text, n); if (res == 0) { res = cast(int)(a.len - b.len); } return res; } gb_internal int string16_compare(String16 const &a, String16 const &b) { if (a.text == b.text) { return cast(int)(a.len - b.len); } if (a.text == nullptr) { return -1; } if (b.text == nullptr) { return +1; } uintptr n = gb_min(a.len, b.len); int res = memcmp(a.text, b.text, n*gb_size_of(u16)); if (res == 0) { res = cast(int)(a.len - b.len); } return res; } gb_internal isize string_index_byte(String const &s, u8 x) { for (isize i = 0; i < s.len; i++) { if (s.text[i] == x) { return i; } } return -1; } gb_internal gb_inline bool str_eq(String const &a, String const &b) { if (a.len != b.len) return false; if (a.len == 0) return true; if (a.text == b.text) return true; return memcmp(a.text, b.text, a.len) == 0; } gb_internal gb_inline bool str_ne(String const &a, String const &b) { return !str_eq(a, b); } gb_internal gb_inline bool str_lt(String const &a, String const &b) { return string_compare(a, b) < 0; } gb_internal gb_inline bool str_gt(String const &a, String const &b) { return string_compare(a, b) > 0; } gb_internal gb_inline bool str_le(String const &a, String const &b) { return string_compare(a, b) <= 0; } gb_internal gb_inline bool str_ge(String const &a, String const &b) { return string_compare(a, b) >= 0; } gb_internal gb_inline bool operator == (String const &a, String const &b) { return str_eq(a, b); } gb_internal gb_inline bool operator != (String const &a, String const &b) { return str_ne(a, b); } gb_internal gb_inline bool operator < (String const &a, String const &b) { return str_lt(a, b); } gb_internal gb_inline bool operator > (String const &a, String const &b) { return str_gt(a, b); } gb_internal gb_inline bool operator <= (String const &a, String const &b) { return str_le(a, b); } gb_internal gb_inline bool operator >= (String const &a, String const &b) { return str_ge(a, b); } template gb_internal bool operator == (String const &a, char const (&b)[N]) { return str_eq(a, make_string(cast(u8 *)b, N-1)); } template gb_internal bool operator != (String const &a, char const (&b)[N]) { return str_ne(a, make_string(cast(u8 *)b, N-1)); } template gb_internal bool operator < (String const &a, char const (&b)[N]) { return str_lt(a, make_string(cast(u8 *)b, N-1)); } template gb_internal bool operator > (String const &a, char const (&b)[N]) { return str_gt(a, make_string(cast(u8 *)b, N-1)); } template gb_internal bool operator <= (String const &a, char const (&b)[N]) { return str_le(a, make_string(cast(u8 *)b, N-1)); } template gb_internal bool operator >= (String const &a, char const (&b)[N]) { return str_ge(a, make_string(cast(u8 *)b, N-1)); } template <> bool operator == (String const &a, char const (&b)[1]) { return a.len == 0; } template <> bool operator != (String const &a, char const (&b)[1]) { return a.len != 0; } gb_internal gb_inline bool str_eq(String16 const &a, String16 const &b) { if (a.len != b.len) return false; if (a.len == 0) return true; return memcmp(a.text, b.text, a.len * gb_size_of(u16)) == 0; } gb_internal gb_inline bool str_ne(String16 const &a, String16 const &b) { return !str_eq(a, b); } gb_internal gb_inline bool str_lt(String16 const &a, String16 const &b) { return string16_compare(a, b) < 0; } gb_internal gb_inline bool str_gt(String16 const &a, String16 const &b) { return string16_compare(a, b) > 0; } gb_internal gb_inline bool str_le(String16 const &a, String16 const &b) { return string16_compare(a, b) <= 0; } gb_internal gb_inline bool str_ge(String16 const &a, String16 const &b) { return string16_compare(a, b) >= 0; } gb_internal gb_inline bool operator == (String16 const &a, String16 const &b) { return str_eq(a, b); } gb_internal gb_inline bool operator != (String16 const &a, String16 const &b) { return str_ne(a, b); } gb_internal gb_inline bool operator < (String16 const &a, String16 const &b) { return str_lt(a, b); } gb_internal gb_inline bool operator > (String16 const &a, String16 const &b) { return str_gt(a, b); } gb_internal gb_inline bool operator <= (String16 const &a, String16 const &b) { return str_le(a, b); } gb_internal gb_inline bool operator >= (String16 const &a, String16 const &b) { return str_ge(a, b); } gb_internal gb_inline bool string_starts_with(String const &s, String const &prefix) { if (prefix.len > s.len) { return false; } return substring(s, 0, prefix.len) == prefix; } gb_internal gb_inline bool string_ends_with(String const &s, String const &suffix) { if (suffix.len > s.len) { return false; } return substring(s, s.len-suffix.len, s.len) == suffix; } gb_internal gb_inline bool string_starts_with(String const &s, u8 prefix) { if (1 > s.len) { return false; } return s[0] == prefix; } gb_internal gb_inline bool string_ends_with(String const &s, u8 suffix) { if (1 > s.len) { return false; } return s[s.len-1] == suffix; } gb_internal gb_inline String string_trim_starts_with(String const &s, String const &prefix) { if (string_starts_with(s, prefix)) { return substring(s, prefix.len, s.len); } return s; } gb_internal String string_split_iterator(String_Iterator *it, const char sep) { isize start = it->pos; isize end = it->str.len; if (start == end) { return str_lit(""); } isize i = start; for (; i < it->str.len; i++) { if (it->str[i] == sep) { String res = substring(it->str, start, i); it->pos += res.len + 1; return res; } } it->pos = end; return substring(it->str, start, end); } gb_internal gb_inline bool is_separator(u8 const &ch) { return (ch == '/' || ch == '\\'); } gb_internal gb_inline isize string_extension_position(String const &str) { isize dot_pos = -1; isize i = str.len; while (i --> 0) { if (is_separator(str[i])) break; if (str[i] == '.') { dot_pos = i; break; } } return dot_pos; } gb_internal String path_extension(String const &str, bool include_dot = true) { isize pos = string_extension_position(str); if (pos < 0) { return make_string(nullptr, 0); } return substring(str, include_dot ? pos : pos + 1, str.len); } gb_internal String path_remove_extension(String const &str) { isize pos = string_extension_position(str); if (pos < 0) { return str; } return substring(str, 0, pos); } gb_internal String string_trim_whitespace(String str) { while (str.len > 0 && rune_is_whitespace(str[str.len-1])) { str.len--; } while (str.len > 0 && str[str.len-1] == 0) { str.len--; } while (str.len > 0 && rune_is_whitespace(str[0])) { str.text++; str.len--; } return str; } gb_internal String string_trim_trailing_whitespace(String str) { while (str.len > 0) { u8 c = str[str.len-1]; if (rune_is_whitespace(c) || c == 0) { str.len -= 1; } else { break; } } return str; } gb_internal String split_lines_first_line_from_array(Array const &array, gbAllocator allocator) { String_Iterator it = {{array.data, array.count}, 0}; String line = string_split_iterator(&it, '\n'); line = string_trim_trailing_whitespace(line); return line; } gb_internal Array split_lines_from_array(Array const &array, gbAllocator allocator) { Array lines = {}; lines.allocator = allocator; String_Iterator it = {{array.data, array.count}, 0}; for (;;) { String line = string_split_iterator(&it, '\n'); if (line.len == 0) { break; } line = string_trim_trailing_whitespace(line); array_add(&lines, line); } return lines; } enum : u32 { PRIME_RABIN_KARP = 16777619u }; gb_internal u32 hash_str_rabin_karp(String const &s, u32 *pow_) { u32 hash = 0; u32 pow = 1; for (isize i = 0; i < s.len; i++) { hash = hash*PRIME_RABIN_KARP + cast(u32)s.text[i]; } u32 sq = PRIME_RABIN_KARP; for (isize i = s.len; i > 0; i >>= 1) { if ((i & 1) != 0) { pow *= sq; } sq *= sq; } if (pow_) *pow_ = pow; return hash; } gb_internal isize string_index(String const &s, String const &substr) { isize n = substr.len; if (n == 0) { return 0; } else if (n == 1) { return string_index_byte(s, substr[0]); } else if (n == s.len) { if (s == substr) { return 0; } return -1; } else if (n > s.len) { return -1; } u32 pow = 1; u32 hash = hash_str_rabin_karp(s, &pow); u32 h = 0; for (isize i = 0; i < n; i++) { h = h*PRIME_RABIN_KARP + cast(u32)s.text[i]; } if (h == hash && substring(s, 0, n) == substr) { return 0; } for (isize i = n; i < s.len; /**/) { h *= PRIME_RABIN_KARP; h += cast(u32)s.text[i]; h -= pow * u32(s.text[i-n]); i += 1; if (h == hash && substring(s, i-n, i) == substr) { return i - n; } } return -1; } struct StringPartition { String head; String match; String tail; }; gb_internal StringPartition string_partition(String const &str, String const &sep) { StringPartition res = {}; isize i = string_index(str, sep); if (i < 0) { res.head = str; return res; } res.head = substring(str, 0, i); res.match = substring(str, i, i+sep.len); res.tail = substring(str, i+sep.len, str.len); return res; } gb_internal bool string_contains_char(String const &s, u8 c) { isize i; for (i = 0; i < s.len; i++) { if (s[i] == c) return true; } return false; } gb_internal bool string_contains_string(String const &haystack, String const &needle) { if (needle.len == 0) return true; if (needle.len > haystack.len) return false; for (isize i = 0; i <= haystack.len - needle.len; i++) { bool found = true; for (isize j = 0; j < needle.len; j++) { if (haystack[i + j] != needle[j]) { found = false; break; } } if (found) { return true; } } return false; } gb_internal String filename_from_path(String s) { isize i = string_extension_position(s); if (i >= 0) { return substring(s, 0, i); } return make_string(nullptr, 0); } gb_internal String filename_without_directory(String s) { isize j = 0; for (j = s.len-1; j >= 0; j--) { if (is_separator(s[j])) { break; } } return substring(s, gb_max(j+1, 0), s.len); } gb_internal String clone_string(gbAllocator a, String const &x) { u8 *data = gb_alloc_array(a, u8, x.len+1); gb_memmove(data, x.text, x.len); data[x.len] = 0; return make_string(data, x.len); } gb_internal String concatenate_strings(gbAllocator a, String const &x, String const &y) { isize len = x.len+y.len; u8 *data = gb_alloc_array(a, u8, len+1); gb_memmove(data, x.text, x.len); gb_memmove(data+x.len, y.text, y.len); data[len] = 0; return make_string(data, len); } gb_internal String concatenate3_strings(gbAllocator a, String const &x, String const &y, String const &z) { isize len = x.len+y.len+z.len; u8 *data = gb_alloc_array(a, u8, len+1); gb_memmove(data, x.text, x.len); gb_memmove(data+x.len, y.text, y.len); gb_memmove(data+x.len+y.len, z.text, z.len); data[len] = 0; return make_string(data, len); } gb_internal String concatenate4_strings(gbAllocator a, String const &x, String const &y, String const &z, String const &w) { isize len = x.len+y.len+z.len+w.len; u8 *data = gb_alloc_array(a, u8, len+1); gb_memmove(data, x.text, x.len); gb_memmove(data+x.len, y.text, y.len); gb_memmove(data+x.len+y.len, z.text, z.len); gb_memmove(data+x.len+y.len+z.len, w.text, w.len); data[len] = 0; return make_string(data, len); } #if defined(GB_SYSTEM_WINDOWS) gb_internal String escape_char(gbAllocator a, String s, char cte) { isize buf_len = s.len; isize cte_count = 0; for (isize j = 0; j < s.len; j++) { if (s.text[j] == cte) { cte_count++; } } u8 *buf = gb_alloc_array(a, u8, buf_len+cte_count); isize i = 0; for (isize j = 0; j < s.len; j++) { u8 c = s.text[j]; if (c == cte) { buf[i++] = '\\'; buf[i++] = c; } else { buf[i++] = c; } } return make_string(buf, i); } #endif gb_internal String string_join_and_quote(gbAllocator a, Array strings) { if (!strings.count) { return make_string(nullptr, 0); } isize str_len = 0; for (isize i = 0; i < strings.count; i++) { str_len += strings[i].len; } gbString s = gb_string_make_reserve(a, str_len+strings.count); // +strings.count for spaces after args. for (isize i = 0; i < strings.count; i++) { if (i > 0) { s = gb_string_append_fmt(s, " "); } #if defined(GB_SYSTEM_WINDOWS) s = gb_string_append_fmt(s, "\"%.*s\" ", LIT(escape_char(a, strings[i], '\\'))); #else s = gb_string_append_fmt(s, "\"%.*s\" ", LIT(strings[i])); #endif } return make_string(cast(u8 *) s, gb_string_length(s)); } gb_internal String copy_string(gbAllocator a, String const &s) { u8 *data = gb_alloc_array(a, u8, s.len+1); gb_memmove(data, s.text, s.len); data[s.len] = 0; return make_string(data, s.len); } gb_internal String normalize_path(gbAllocator a, String const &path, String const &sep) { String s; if (sep.len < 1) { return path; } if (path.len < 1) { s = STR_LIT(""); } else if (is_separator(path[path.len-1])) { s = copy_string(a, path); } else { s = concatenate_strings(a, path, sep); } isize i; for (i = 0; i < s.len; i++) { if (is_separator(s.text[i])) { s.text[i] = sep.text[0]; } } return s; } #if defined(GB_SYSTEM_WINDOWS) gb_internal int convert_multibyte_to_widechar(char const *multibyte_input, int input_length, wchar_t *output, int output_size) { return MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, multibyte_input, input_length, output, output_size); } gb_internal int convert_widechar_to_multibyte(wchar_t const *widechar_input, int input_length, char *output, int output_size) { return WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, widechar_input, input_length, output, output_size, nullptr, nullptr); } #elif defined(GB_SYSTEM_UNIX) || defined(GB_SYSTEM_OSX) #include gb_internal void utf16_encode_surrogate_pair(Rune r, u16 *r1, u16 *r2) { static Rune const _surr_self = 0x10000; static Rune const _surr1 = 0xd800; static Rune const _surr2 = 0xdc00; Rune r_ = r - _surr_self; *r1 = _surr1 + ((r_ >> 10) & 0x3ff); *r2 = _surr2 + (r_ & 0x3ff); } gb_internal isize utf16_decode(u16 const *s, isize n, Rune *r) { static Rune const _surr1 = 0xd800; static Rune const _surr2 = 0xdc00; static Rune const _surr3 = 0xe000; static Rune const _surr_self = 0x10000; if (n < 1) { *r = GB_RUNE_INVALID; return 0; } u16 c = s[0]; if (c < 0xd800 || c > 0xdfff) { *r = cast(Rune)c; return 1; } if (c >= 0xdc00) { *r = GB_RUNE_INVALID; return 1; } if (n < 2) { *r = GB_RUNE_INVALID; return 1; } u16 c2 = s[1]; if (c2 < 0xdc00 || c2 > 0xdfff) { *r = GB_RUNE_INVALID; return 1; } *r = (((c-_surr1)<<10) | (c2 - _surr2)) + _surr_self; return 2; } gb_internal int convert_multibyte_to_widechar(char const *multibyte_input, usize input_length, wchar_t *output, usize output_size) { u16 *out = cast(u16 *)output; String s = make_string(cast(u8 const *)multibyte_input, input_length); isize i = 0; isize output_len = 0; while (i < s.len) { Rune r = 0; isize width = utf8_decode(s.text + i, s.len - i, &r); if (r == GB_RUNE_INVALID) { return -1; } i += width; if (r < 0x10000) { if (out) { if (cast(usize)output_len+1 > output_size) { return -1; } out[output_len] = cast(u16)r; } output_len += 1; } else { if (out) { if (cast(usize)output_len+2 > output_size) { return -1; } u16 r1, r2; utf16_encode_surrogate_pair(r, &r1, &r2); out[output_len+0] = r1; out[output_len+1] = r2; } output_len += 2; } } return cast(int)output_len; } gb_internal int convert_widechar_to_multibyte(wchar_t const *widechar_input, usize input_length, char* output, usize output_size) { u16 const *in = cast(u16 const *)widechar_input; isize i = 0; isize output_len = 0; while (i < input_length) { Rune r; isize width = utf16_decode(in + i, input_length - i, &r); if (r == GB_RUNE_INVALID) { return -1; } i += width; u8 buf[4]; isize char_len = gb_utf8_encode_rune(buf, r); if(output) { if (cast(usize)output_len+cast(usize)char_len > output_size) { return -1; } gb_memmove(output + output_len, buf, char_len); } output_len += char_len; } return cast(int)output_len; } #else #error Implement system #endif gb_internal String16 string_to_string16(gbAllocator a, String s) { int len, len1; u16 *text; if (s.len < 1) { return make_string16(nullptr, 0); } len = convert_multibyte_to_widechar(cast(char *)s.text, cast(int)s.len, nullptr, 0); if (len == 0) { return make_string16(nullptr, 0); } text = gb_alloc_array(a, u16, len+1); len1 = convert_multibyte_to_widechar(cast(char *)s.text, cast(int)s.len, cast(wchar_t *)text, cast(int)len); if (len1 == 0) { gb_free(a, text); return make_string16(nullptr, 0); } text[len] = 0; return make_string16(text, len); } gb_internal String string16_to_string(gbAllocator a, String16 s) { int len, len1; u8 *text; if (s.len < 1) { return make_string(nullptr, 0); } len = convert_widechar_to_multibyte(cast(wchar_t *)s.text, cast(int)s.len, nullptr, 0); if (len == 0) { return make_string(nullptr, 0); } len += 1; // NOTE(bill): It needs an extra 1 for some reason text = gb_alloc_array(a, u8, len+1); len1 = convert_widechar_to_multibyte(cast(wchar_t *)s.text, cast(int)s.len, cast(char *)text, cast(int)len); if (len1 == 0) { gb_free(a, text); return make_string(nullptr, 0); } text[len] = 0; return make_string(text, len-1); } gb_internal String temporary_directory(gbAllocator allocator) { #if defined(GB_SYSTEM_WINDOWS) DWORD n = GetTempPathW(0, nullptr); if (n == 0) { return String{0}; } DWORD len = gb_max(MAX_PATH, n); u16 *b = gb_alloc_array(heap_allocator(), u16, len+1); defer (gb_free(heap_allocator(), b)); n = GetTempPathW(len, cast(wchar_t *)b); if (n == 3 && b[1] == ':' && b[2] == '\\') { } else if (n > 0 && b[n-1] == '\\') { n -= 1; } b[n] = 0; String16 s = make_string16(b, n); return string16_to_string(allocator, s); #else char const *tmp_env = gb_get_env("TMPDIR", allocator); if (tmp_env) { return make_string_c(tmp_env); } #if defined(P_tmpdir) String tmp_macro = make_string_c(P_tmpdir); if (tmp_macro.len != 0) { return copy_string(allocator, tmp_macro); } #endif return copy_string(allocator, str_lit("/tmp")); #endif } gb_internal bool is_printable(Rune r) { if (r <= 0xff) { if (0x20 <= r && r <= 0x7e) { return true; } if (0xa1 <= r && r <= 0xff) { return r != 0xad; } return false; } return false; } gb_global char const lower_hex[] = "0123456789abcdef"; gb_internal String quote_to_ascii(gbAllocator a, String str, u8 quote='"') { u8 *s = str.text; isize n = str.len; auto buf = array_make(a, 0, n); array_add(&buf, quote); for (isize width = 0; n > 0; s += width, n -= width) { Rune r = cast(Rune)s[0]; width = 1; if (r >= 0x80) { width = utf8_decode(s, n, &r); } if (width == 1 && r == GB_RUNE_INVALID) { array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'x'); array_add(&buf, cast(u8)lower_hex[s[0]>>4]); array_add(&buf, cast(u8)lower_hex[s[0]&0xf]); continue; } if (r == quote || r == '\\') { array_add(&buf, cast(u8)'\\'); array_add(&buf, u8(r)); continue; } if (r < 0x80 && is_printable(r)) { array_add(&buf, u8(r)); continue; } switch (r) { case '\a': case '\b': case '\f': case '\n': case '\r': case '\t': case '\v': default: if (r < ' ') { u8 b = cast(u8)r; array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'x'); array_add(&buf, cast(u8)lower_hex[b>>4]); array_add(&buf, cast(u8)lower_hex[b&0xf]); } if (r > GB_RUNE_MAX) { r = 0XFFFD; } if (r < 0x10000) { array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'u'); for (isize i = 12; i >= 0; i -= 4) { array_add(&buf, cast(u8)lower_hex[(r>>i)&0xf]); } } else { array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'U'); for (isize i = 28; i >= 0; i -= 4) { array_add(&buf, cast(u8)lower_hex[(r>>i)&0xf]); } } } } array_add(&buf, quote); String res = {}; res.text = buf.data; res.len = buf.count; return res; } gb_internal Rune decode_surrogate_pair(u16 r1, u16 r2) { static Rune const _surr1 = 0xd800; static Rune const _surr2 = 0xdc00; static Rune const _surr3 = 0xe000; static Rune const _surr_self = 0x10000; if (_surr1 <= r1 && r1 < _surr2 && _surr2 <= r2 && r2 < _surr3) { return (((r1-_surr1)<<10) | (r2 - _surr2)) + _surr_self; } return GB_RUNE_INVALID; } gb_internal String quote_to_ascii(gbAllocator a, String16 str, u8 quote='"') { static Rune const _surr1 = 0xd800; static Rune const _surr2 = 0xdc00; static Rune const _surr3 = 0xe000; static Rune const _surr_self = 0x10000; u16 *s = cast(u16 *)str.text; isize n = str.len; auto buf = array_make(a, 0, n*2); array_add(&buf, quote); for (isize width = 0; n > 0; s += width, n -= width) { Rune r = cast(Rune)s[0]; width = 1; if (r < _surr1 || _surr3 <= r) { r = cast(Rune)r; } else if (_surr1 <= r && r < _surr2) { if (n>1) { r = decode_surrogate_pair(s[0], s[1]); if (r != GB_RUNE_INVALID) { width = 2; } } else { r = GB_RUNE_INVALID; } } if (width == 1 && r == GB_RUNE_INVALID) { array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'u'); for (isize i = 12; i >= 0; i -= 4) { array_add(&buf, cast(u8)lower_hex[(s[0]>>i)&0xf]); } continue; } if (r == quote || r == '\\') { array_add(&buf, cast(u8)'\\'); array_add(&buf, u8(r)); continue; } if (r < 0x80 && is_printable(r)) { array_add(&buf, u8(r)); continue; } switch (r) { case '\a': case '\b': case '\f': case '\n': case '\r': case '\t': case '\v': default: if (r < ' ') { u8 b = cast(u8)r; array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'x'); array_add(&buf, cast(u8)lower_hex[b>>4]); array_add(&buf, cast(u8)lower_hex[b&0xf]); } if (r > GB_RUNE_MAX) { r = 0XFFFD; } if (r < 0x10000) { array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'u'); for (isize i = 12; i >= 0; i -= 4) { array_add(&buf, cast(u8)lower_hex[(r>>i)&0xf]); } } else { array_add(&buf, cast(u8)'\\'); array_add(&buf, cast(u8)'U'); for (isize i = 28; i >= 0; i -= 4) { array_add(&buf, cast(u8)lower_hex[(r>>i)&0xf]); } } } } array_add(&buf, quote); String res = {}; res.text = buf.data; res.len = buf.count; return res; } gb_internal bool unquote_char(String s, u8 quote, Rune *rune, bool *multiple_bytes, String *tail_string) { u8 c; if (s[0] == quote && (quote == '\'' || quote == '"')) { return false; } else if (s[0] >= 0x80) { Rune r = -1; isize size = utf8_decode(s.text, s.len, &r); *rune = r; if (multiple_bytes) *multiple_bytes = true; if (tail_string) *tail_string = make_string(s.text+size, s.len-size); return true; } else if (s[0] != '\\') { *rune = s[0]; if (tail_string) *tail_string = make_string(s.text+1, s.len-1); return true; } if (s.len <= 1) { return false; } c = s[1]; s = make_string(s.text+2, s.len-2); switch (c) { default: return false; case 'a': *rune = '\a'; break; case 'b': *rune = '\b'; break; case 'e': *rune = 0x1b; break; case 'f': *rune = '\f'; break; case 'n': *rune = '\n'; break; case 'r': *rune = '\r'; break; case 't': *rune = '\t'; break; case 'v': *rune = '\v'; break; case '\\': *rune = '\\'; break; case '\'': case '"': *rune = c; break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { isize i; i32 r = gb_digit_to_int(c); if (s.len < 2) { return false; } for (i = 0; i < 2; i++) { i32 d = gb_digit_to_int(s[i]); if (d < 0 || d > 7) { return false; } r = (r<<3) | d; } s = make_string(s.text+2, s.len-2); if (r > 0xff) { return false; } *rune = r; } break; case 'x': case 'u': case 'U': { Rune r = 0; isize i, count = 0; switch (c) { case 'x': count = 2; break; case 'u': count = 4; break; case 'U': count = 8; break; } if (s.len < count) { return false; } for (i = 0; i < count; i++) { i32 d = gb_hex_digit_to_int(s[i]); if (d < 0) { return false; } r = (r<<4) | d; } s = make_string(s.text+count, s.len-count); if (c == 'x') { *rune = r; break; } if (r > GB_RUNE_MAX) { return false; } *rune = r; if (multiple_bytes) *multiple_bytes = true; } break; } if (tail_string) *tail_string = s; return true; } gb_internal String strip_carriage_return(gbAllocator a, String s) { isize buf_len = s.len; u8 *buf = gb_alloc_array(a, u8, buf_len); isize i = 0; for (isize j = 0; j < s.len; j++) { u8 c = s.text[j]; if (c != '\r') { buf[i++] = c; } } return make_string(buf, i); } // 0 == failure // 1 == original memory // 2 == new allocation gb_internal i32 unquote_string(gbAllocator a, String *s_, u8 quote=0, bool has_carriage_return=false) { String s = *s_; isize n = s.len; if (quote == 0) { if (n < 2) { return 0; } quote = s[0]; if (quote != s[n-1]) { return 0; } s.text += 1; s.len -= 2; } if (quote == '`') { if (string_contains_char(s, '`')) { return 0; } if (has_carriage_return) { *s_ = strip_carriage_return(a, s); return 2; } *s_ = s; return 1; } if (quote != '"' && quote != '\'') { return 0; } if (string_contains_char(s, '\n')) { return 0; } if (!string_contains_char(s, '\\') && !string_contains_char(s, quote)) { if (quote == '"') { *s_ = s; return 1; } else if (quote == '\'') { Rune r = GB_RUNE_INVALID; isize size = utf8_decode(s.text, s.len, &r); if ((size == s.len) && (r != -1 || size != 1)) { *s_ = s; return 1; } } } { u8 rune_temp[4] = {}; isize buf_len = 3*s.len / 2; u8 *buf = gb_alloc_array(a, u8, buf_len); isize offset = 0; while (s.len > 0) { String tail_string = {}; Rune r = 0; bool multiple_bytes = false; bool success = unquote_char(s, quote, &r, &multiple_bytes, &tail_string); if (!success) { gb_free(a, buf); return 0; } s = tail_string; if (r < 0x80 || !multiple_bytes) { buf[offset++] = cast(u8)r; } else { isize size = gb_utf8_encode_rune(rune_temp, r); gb_memmove(buf+offset, rune_temp, size); offset += size; } if (quote == '\'' && s.len != 0) { gb_free(a, buf); return 0; } } *s_ = make_string(buf, offset); } return 2; } // Escape-process the already-delimiter-stripped body of a triple-quoted // string. `force_new` is true when `s` already points at a freshly allocated // buffer, so the "no escapes" fast path still reports a new allocation. // // 0 == failure // 1 == original memory // 2 == new allocation gb_internal i32 unquote_string_triple_content(gbAllocator a, String *s_, String s, bool has_carriage_return, bool force_new) { if (!string_contains_char(s, '\\')) { if (has_carriage_return) { *s_ = strip_carriage_return(a, s); return 2; } *s_ = s; return force_new ? 2 : 1; } if (has_carriage_return) { s = strip_carriage_return(a, s); } u8 rune_temp[4] = {}; isize buf_len = 3*s.len / 2 + 1; u8 *buf = gb_alloc_array(a, u8, buf_len); isize offset = 0; while (s.len > 0) { String tail_string = {}; Rune r = 0; bool multiple_bytes = false; // NOTE: quote==0 makes unquote_char treat `"` as an ordinary byte // (it only rejects a bare quote when quote is '"' or '\''), while // `\\` escapes — including `\"` — are still handled. bool success = unquote_char(s, 0, &r, &multiple_bytes, &tail_string); if (!success) { gb_free(a, buf); return 0; } s = tail_string; if (r < 0x80 || !multiple_bytes) { buf[offset++] = cast(u8)r; } else { isize size = gb_utf8_encode_rune(rune_temp, r); gb_memmove(buf+offset, rune_temp, size); offset += size; } } *s_ = make_string(buf, offset); return 2; } gb_internal bool triple_string_deindent(gbAllocator a, String body, String *out) { // Opening delimiter's line (up to the first newline) must be blank. isize first_nl = 0; while (body.text[first_nl] != '\n') { first_nl += 1; } for (isize i = 0; i < first_nl; i++) { u8 c = body.text[i]; if (c != ' ' && c != '\t' && c != '\r') { return false; } } // Closing delimiter's line: whitespace after the final newline is the // indentation prefix removed from each content line. isize last_nl = body.len - 1; while (body.text[last_nl] != '\n') { last_nl -= 1; } String indent = make_string(body.text+last_nl+1, body.len-(last_nl+1)); for (isize i = 0; i < indent.len; i++) { u8 c = indent.text[i]; if (c != ' ' && c != '\t') { return false; } } String content = make_string(body.text+first_nl+1, last_nl-(first_nl+1)); u8 *dbuf = gb_alloc_array(a, u8, content.len+1); isize dlen = 0; isize li = 0; for (;;) { isize le = li; while (le < content.len && content.text[le] != '\n') { le += 1; } isize len = le - li; if (len > 0 && content.text[li+len-1] == '\r') { len -= 1; // normalize CRLF line endings } bool blank = true; for (isize k = 0; k < len; k++) { u8 c = content.text[li+k]; if (c != ' ' && c != '\t') { blank = false; break; } } if (blank) { // A whitespace-only line contributes an empty line. } else { if (len < indent.len) { gb_free(a, dbuf); return false; // content line is less indented than the closing delimiter } for (isize k = 0; k < indent.len; k++) { if (content.text[li+k] != indent.text[k]) { gb_free(a, dbuf); return false; // indentation whitespace does not match } } for (isize k = indent.len; k < len; k++) { dbuf[dlen++] = content.text[li+k]; } } if (le >= content.len) { break; } dbuf[dlen++] = '\n'; li = le + 1; } *out = make_string(dbuf, dlen); return true; } enum TripleStringErrorKind { TripleStringError_None = 0, TripleStringError_InvalidLiteral, // malformed delimiters (shouldn't occur for a real token) TripleStringError_ContentOnOpeningLine, // text after the opening delimiter on its own line TripleStringError_ClosingNotOnOwnLine, // closing delimiter is not alone on its line TripleStringError_UnderIndented, // a content line is indented less than the closing delimiter TripleStringError_IndentationMismatch, // a line's indentation whitespace differs from the closing delimiter TripleStringError_InvalidEscape, // bad escape sequence (only reachable for `"""` strings) }; // `out_err` receives the failure reason and `out_err_offset` a byte offset into // the *token string* (i.e. into *s_ on entry) pointing at the offending // character, or -1 when no precise location is available. Both are only // meaningful when the function returns 0. // // 0 == failure // 1 == original memory // 2 == new allocation gb_internal i32 unquote_string_triple(gbAllocator a, String *s_, bool has_carriage_return=false, TripleStringErrorKind *out_err=nullptr, isize *out_err_offset=nullptr) { if (out_err) *out_err = TripleStringError_None; if (out_err_offset) *out_err_offset = -1; #define TRIPLE_FAIL(kind, off) do { \ if (out_err) *out_err = (kind); \ if (out_err_offset) *out_err_offset = (off); \ return 0; \ } while (0) String s = *s_; isize n = s.len; if (n < 6) { TRIPLE_FAIL(TripleStringError_InvalidLiteral, 0); } u8 quote = s[0]; if (quote != '"' && quote != '`') { TRIPLE_FAIL(TripleStringError_InvalidLiteral, 0); } if (!(s[0] == quote && s[1] == quote && s[2] == quote) || !(s[n-1] == quote && s[n-2] == quote && s[n-3] == quote)) { TRIPLE_FAIL(TripleStringError_InvalidLiteral, 0); } bool raw = (quote == '`'); String body = make_string(s.text+3, s.len-6); // body begins at token offset 3 // Single-line form: no indentation handling if (!string_contains_char(body, '\n')) { if (raw) { if (has_carriage_return) { *s_ = strip_carriage_return(a, body); return 2; } *s_ = body; return 1; } i32 r = unquote_string_triple_content(a, s_, body, has_carriage_return, false); if (r == 0) { TRIPLE_FAIL(TripleStringError_InvalidEscape, -1); } return r; } // Opening delimiter's line must be whitespace only. isize first_nl = 0; while (body.text[first_nl] != '\n') { first_nl += 1; } for (isize i = 0; i < first_nl; i++) { u8 c = body.text[i]; if (c != ' ' && c != '\t' && c != '\r') { TRIPLE_FAIL(TripleStringError_ContentOnOpeningLine, 3 + i); } } // Closing delimiter's line: whitespace after the final newline is the indent prefix. isize last_nl = body.len - 1; while (body.text[last_nl] != '\n') { last_nl -= 1; } String indent = make_string(body.text+last_nl+1, body.len-(last_nl+1)); for (isize i = 0; i < indent.len; i++) { u8 c = indent.text[i]; if (c != ' ' && c != '\t') { TRIPLE_FAIL(TripleStringError_ClosingNotOnOwnLine, 3 + (last_nl+1) + i); } } String content = make_string(body.text+first_nl+1, last_nl-(first_nl+1)); isize content_base = 3 + (first_nl+1); // token offset of the content start u8 *dbuf = gb_alloc_array(a, u8, content.len+1); isize dlen = 0; isize li = 0; for (;;) { isize le = li; while (le < content.len && content.text[le] != '\n') { le += 1; } isize len = le - li; if (len > 0 && content.text[li+len-1] == '\r') { len -= 1; // normalize CRLF } bool blank = true; for (isize k = 0; k < len; k++) { u8 c = content.text[li+k]; if (c != ' ' && c != '\t') { blank = false; break; } } if (!blank) { isize ws = 0; while (ws < len && (content.text[li+ws] == ' ' || content.text[li+ws] == '\t')) { ws += 1; } isize ws_end_off = content_base + li + ws; for (isize k = 0; k < indent.len; k++) { if (k >= len || content.text[li+k] != indent.text[k]) { u8 lc = 0; if (k < len) { lc = content.text[li+k]; } gb_free(a, dbuf); if (k < len && (lc == ' ' || lc == '\t')) { TRIPLE_FAIL(TripleStringError_IndentationMismatch, ws_end_off); } TRIPLE_FAIL(TripleStringError_UnderIndented, ws_end_off); } } for (isize k = indent.len; k < len; k++) { dbuf[dlen++] = content.text[li+k]; } } if (le >= content.len) { break; } dbuf[dlen++] = '\n'; li = le + 1; } String di = make_string(dbuf, dlen); if (raw) { *s_ = di; return 2; } i32 r = unquote_string_triple_content(a, s_, di, false, true); if (r == 0) { TRIPLE_FAIL(TripleStringError_InvalidEscape, -1); } return r; #undef TRIPLE_FAIL } gb_internal bool string_is_valid_identifier(String str) { if (str.len <= 0) return false; isize rune_count = 0; isize w = 0; isize offset = 0; while (offset < str.len) { Rune r = 0; w = utf8_decode(str.text, str.len, &r); if (r == GB_RUNE_INVALID) { return false; } if (rune_count == 0) { if (!rune_is_letter(r)) { return false; } } else { if (!rune_is_letter(r) && !rune_is_digit(r)) { return false; } } rune_count += 1; offset += w; } return true; }