This commit is contained in:
Jeroen van Rijn
2026-03-09 19:40:10 +01:00
parent df0d74efbe
commit d880404b84
3 changed files with 256 additions and 499 deletions

View File

@@ -1,394 +1,162 @@
package ucd
string_to_general_category :: proc "contextless"(
str: string,
) -> (gc: General_Category, err: Error) {
string_to_general_category :: proc "contextless" (str: string) -> (gc: General_Category, err: Error) {
switch str {
case "Lu":
gc = .Lu
case "Ll":
gc = .Ll
case "Lt":
gc = .Lt
case "Lm":
gc = .Lm
case "Lo":
gc = .Lo
case "Mn":
gc = .Mn
case "Mc":
gc = .Mc
case "Me":
gc = .Me
case "Nd":
gc = .Nd
case "Nl":
gc = .Nl
case "No":
gc = .No
case "Pc":
gc = .Pc
case "Pd":
gc = .Pd
case "Ps":
gc = .Ps
case "Pe":
gc = .Pe
case "Pi":
gc = .Pi
case "Pf":
gc = .Pf
case "Po":
gc = .Po
case "Sm":
gc = .Sm
case "Sc":
gc = .Sc
case "Sk":
gc = .Sk
case "So":
gc = .So
case "Zs":
gc = .Zs
case "Zl":
gc = .Zl
case "Zp":
gc = .Zp
case "Cc":
gc = .Cc
case "Cf":
gc = .Cf
case "Cs":
gc = .Cs
case "Co":
gc = .Co
case "Cn":
gc = .Cn
case:
err = UCD_Error.Invalid_General_Category
case "Lu": gc = .Lu
case "Ll": gc = .Ll
case "Lt": gc = .Lt
case "Lm": gc = .Lm
case "Lo": gc = .Lo
case "Mn": gc = .Mn
case "Mc": gc = .Mc
case "Me": gc = .Me
case "Nd": gc = .Nd
case "Nl": gc = .Nl
case "No": gc = .No
case "Pc": gc = .Pc
case "Pd": gc = .Pd
case "Ps": gc = .Ps
case "Pe": gc = .Pe
case "Pi": gc = .Pi
case "Pf": gc = .Pf
case "Po": gc = .Po
case "Sm": gc = .Sm
case "Sc": gc = .Sc
case "Sk": gc = .Sk
case "So": gc = .So
case "Zs": gc = .Zs
case "Zl": gc = .Zl
case "Zp": gc = .Zp
case "Cc": gc = .Cc
case "Cf": gc = .Cf
case "Cs": gc = .Cs
case "Co": gc = .Co
case "Cn": gc = .Cn
case: err = .Invalid_General_Category
}
return
}
string_to_age :: proc "contextless" (
str: string,
) -> (age: Age, err: Error) {
string_to_proplist_property :: proc(str: string) -> (prop: Prop_List_Property) {
switch str {
case "1.1":
age = .Age_1_1
return
case "2.0":
age = .Age_2_0
return
case "2.1":
age = .Age_2_1
return
case "3.0":
age = .Age_3_0
return
case "3.1":
age = .Age_3_1
return
case "3.2":
age = .Age_3_2
return
case "4.0":
age = .Age_4_0
return
case "4.1":
age = .Age_4_1
return
case "5.0":
age = .Age_5_0
return
case "5.1":
age = .Age_5_1
return
case "5.2":
age = .Age_5_2
return
case "6.0":
age = .Age_6_0
return
case "6.1":
age = .Age_6_1
return
case "6.2":
age = .Age_6_2
return
case "6.3":
age = .Age_6_3
return
case "7.0":
age = .Age_7_0
return
case "8.0":
age = .Age_8_0
return
case "9.0":
age = .Age_9_0
return
case "10.0":
age = .Age_10_0
return
case "11.0":
age = .Age_11_0
return
case "12.0":
age = .Age_12_0
return
case "12.1":
age = .Age_12_1
return
case "13.0":
age = .Age_13_0
return
case "14.0":
age = .Age_14_0
return
case "15.0":
age = .Age_15_0
return
case "15.1":
age = .Age_15_1
return
case "16.0":
age = .Age_16_0
return
case "17.0":
age = .Age_17_0
return
case "unassigned":
age = .Age_Unassigned
return
case:
// NOTE: Should this return an error instead?
unreachable()
case "White_Space": prop = .White_Space
case "Bidi_Control": prop = .Bidi_Control
case "Join_Control": prop = .Join_Control
case "Dash": prop = .Dash
case "Hyphen": prop = .Hyphen
case "Quotation_Mark": prop = .Quotation_Mark
case "Terminal_Punctuation": prop = .Terminal_Punctuation
case "Other_Math": prop = .Other_Math
case "Hex_Digit": prop = .Hex_Digit
case "ASCII_Hex_Digit": prop = .ASCII_Hex_Digit
case "Other_Alphabetic": prop = .Other_Alphabetic
case "Ideographic": prop = .Ideographic
case "Diacritic": prop = .Diacritic
case "Extender": prop = .Extender
case "Other_Lowercase": prop = .Other_Lowercase
case "Other_Uppercase": prop = .Other_Uppercase
case "Noncharacter_Code_Point": prop = .Noncharacter_Code_Point
case "Other_Grapheme_Extend": prop = .Other_Grapheme_Extend
case "IDS_Binary_Operator": prop = .IDS_Binary_Operator
case "IDS_Trinary_Operator": prop = .IDS_Trinary_Operator
case "IDS_Unary_Operator": prop = .IDS_Unary_Operator
case "Radical": prop = .Radical
case "Unified_Ideograph": prop = .Unified_Ideograph
case "Other_Default_Ignorable_Code_Point": prop = .Other_Default_Ignorable_Code_Point
case "Deprecated": prop = .Deprecated
case "Soft_Dotted": prop = .Soft_Dotted
case "Logical_Order_Exception": prop = .Logical_Order_Exception
case "Other_ID_Start": prop = .Other_ID_Start
case "Other_ID_Continue": prop = .Other_ID_Continue
case "ID_Compat_Math_Continue": prop = .ID_Compat_Math_Continue
case "ID_Compat_Math_Start": prop = .ID_Compat_Math_Start
case "Sentence_Terminal": prop = .Sentence_Terminal
case "Variation_Selector": prop = .Variation_Selector
case "Pattern_White_Space": prop = .Pattern_White_Space
case "Pattern_Syntax": prop = .Pattern_Syntax
case "Prepended_Concatenation_Mark": prop = .Prepended_Concatenation_Mark
case "Regional_Indicator": prop = .Regional_Indicator
case "Modifier_Combining_Mark": prop = .Modifier_Combining_Mark
case: prop = .Unknown_Property
}
return
}
string_to_paired_bracket_type :: proc "contextless"(str: string) -> Paired_Brack_Type {
@(deprecated="Unused?")
string_to_age :: proc "contextless" (str: string) -> (age: Age) {
switch str {
case "o":
return .Open
case "c":
return .Close
case "n":
return .None
case:
// TODO: Add error for this
unreachable()
case "1.1": age = .Age_1_1
case "2.0": age = .Age_2_0
case "2.1": age = .Age_2_1
case "3.0": age = .Age_3_0
case "3.1": age = .Age_3_1
case "3.2": age = .Age_3_2
case "4.0": age = .Age_4_0
case "4.1": age = .Age_4_1
case "5.0": age = .Age_5_0
case "5.1": age = .Age_5_1
case "5.2": age = .Age_5_2
case "6.0": age = .Age_6_0
case "6.1": age = .Age_6_1
case "6.2": age = .Age_6_2
case "6.3": age = .Age_6_3
case "7.0": age = .Age_7_0
case "8.0": age = .Age_8_0
case "9.0": age = .Age_9_0
case "10.0": age = .Age_10_0
case "11.0": age = .Age_11_0
case "12.0": age = .Age_12_0
case "12.1": age = .Age_12_1
case "13.0": age = .Age_13_0
case "14.0": age = .Age_14_0
case "15.0": age = .Age_15_0
case "15.1": age = .Age_15_1
case "16.0": age = .Age_16_0
case "17.0": age = .Age_17_0
case "unassigned": age = .Age_Unassigned
case: age = .Age_Unknown
}
return
}
string_to_bidi_class :: proc "contextless"(str: string) -> Bidi_Class {
@(deprecated="Unused?")
string_to_paired_bracket_type :: proc "contextless" (str: string) -> (pbt: Paired_Bracket_Type) {
switch str {
case "AL":
return .AL
case "AN":
return .AN
case "B":
return .B
case "BN":
return .BN
case "CS":
return .CS
case "EN":
return .EN
case "ES":
return .ES
case "ET":
return .ET
case "FSI":
return .FSI
case "L":
return .L
case "LRE":
return .LRE
case "LRI":
return .LRI
case "LRO":
return .LRO
case "NSM":
return .NSM
case "ON":
return .ON
case "PDF":
return .PDF
case "PDI":
return .PDI
case "R":
return .R
case "RLE":
return .RLE
case "RLI":
return .RLI
case "RLO":
return .RLO
case "S":
return .S
case "WS":
return .WS
case: unreachable() // TODO: Add error for this
case "o": pbt = .Open
case "c": pbt = .Close
case "n": pbt = .None
case: pbt = .Unknown
}
return
}
string_to_proplist_property :: proc(str: string) -> (
prop: PropList_Property,
err: UCD_Error,
) {
@(deprecated="Unused?")
string_to_bidi_class :: proc "contextless" (str: string) -> (class: Bidi_Class) {
switch str {
case "White_Space":
prop = .White_Space
case "Bidi_Control":
prop = .Bidi_Control
case "Join_Control":
prop = .Join_Control
case "Dash":
prop = .Dash
case "Hyphen":
prop = .Hyphen
case "Quotation_Mark":
prop = .Quotation_Mark
case "Terminal_Punctuation":
prop = .Terminal_Punctuation
case "Other_Math":
prop = .Other_Math
case "Hex_Digit":
prop = .Hex_Digit
case "ASCII_Hex_Digit":
prop = .ASCII_Hex_Digit
case "Other_Alphabetic":
prop = .Other_Alphabetic
case "Ideographic":
prop = .Ideographic
case "Diacritic":
prop = .Diacritic
case "Extender":
prop = .Extender
case "Other_Lowercase":
prop = .Other_Lowercase
case "Other_Uppercase":
prop = .Other_Uppercase
case "Noncharacter_Code_Point":
prop = .Noncharacter_Code_Point
case "Other_Grapheme_Extend":
prop = .Other_Grapheme_Extend
case "IDS_Binary_Operator":
prop = .IDS_Binary_Operator
case "IDS_Trinary_Operator":
prop = .IDS_Trinary_Operator
case "IDS_Unary_Operator":
prop = .IDS_Unary_Operator
case "Radical":
prop = .Radical
case "Unified_Ideograph":
prop = .Unified_Ideograph
case "Other_Default_Ignorable_Code_Point":
prop = .Other_Default_Ignorable_Code_Point
case "Deprecated":
prop = .Deprecated
case "Soft_Dotted":
prop = .Soft_Dotted
case "Logical_Order_Exception":
prop = .Logical_Order_Exception
case "Other_ID_Start":
prop = .Other_ID_Start
case "Other_ID_Continue":
prop = .Other_ID_Continue
case "ID_Compat_Math_Continue":
prop = .ID_Compat_Math_Continue
case "ID_Compat_Math_Start":
prop = .ID_Compat_Math_Start
case "Sentence_Terminal":
prop = .Sentence_Terminal
case "Variation_Selector":
prop = .Variation_Selector
case "Pattern_White_Space":
prop = .Pattern_White_Space
case "Pattern_Syntax":
prop = .Pattern_Syntax
case "Prepended_Concatenation_Mark":
prop = .Prepended_Concatenation_Mark
case "Regional_Indicator":
prop = .Regional_Indicator
case "Modifier_Combining_Mark":
prop = .Modifier_Combining_Mark
case:
err = .Unknown_Property
return
case "AL": class = .AL
case "AN": class = .AN
case "B": class = .B
case "BN": class = .BN
case "CS": class = .CS
case "EN": class = .EN
case "ES": class = .ES
case "ET": class = .ET
case "FSI": class = .FSI
case "L": class = .L
case "LRE": class = .LRE
case "LRI": class = .LRI
case "LRO": class = .LRO
case "NSM": class = .NSM
case "ON": class = .ON
case "PDF": class = .PDF
case "PDI": class = .PDI
case "R": class = .R
case "RLE": class = .RLE
case "RLI": class = .RLI
case "RLO": class = .RLO
case "S": class = .S
case "WS": class = .WS
case: class = .Unknown
}
return
}
return
}

View File

@@ -3,7 +3,7 @@ package ucd
import "core:os"
Age :: enum byte {
Nil = 0,
Age_Unknown = 0,
Age_1_1,
Age_2_0,
Age_2_1,
@@ -421,51 +421,49 @@ Block :: enum {
Combining_Class :: distinct byte
Paired_Brack_Type :: enum {
Nil,
Paired_Bracket_Type :: enum {
Unknown,
Open,
Close,
None,
}
Bidi_Class :: enum {
Nil, //
L, // Left-to-Right LRM
R, // Right-to-Left RLM
AL, // Right-to-Left Arabic ALM
EN, // European Number
ES, // European Number Separator
ET, // European Number Terminator
AN, // Arabic Number
CS, // Common Number Separator
NSM, // Nonspacing Mark
BN, // Boundary Neutral
B, // Paragraph Separator
S, // Segment Separator
WS, // Whitespace
ON, // Other Neutrals
LRE, // Left-to-Right Embedding LRE
LRO, // Left-to-Right Override LRO
RLE, // Right-to-Left Embedding RLE
RLO, // Right-to-Left Override RLO
PDF, // Pop Directional Format PDF
LRI, // Left-to-Right Isolate LRI
RLI, // Right-to-Left Isolate RLI
FSI, // First Strong Isolate FSI
PDI, // Pop Directional Isolate PDI
Unknown, //
L, // Left-to-Right LRM
R, // Right-to-Left RLM
AL, // Right-to-Left Arabic ALM
EN, // European Number
ES, // European Number Separator
ET, // European Number Terminator
AN, // Arabic Number
CS, // Common Number Separator
NSM, // Nonspacing Mark
BN, // Boundary Neutral
B, // Paragraph Separator
S, // Segment Separator
WS, // Whitespace
ON, // Other Neutrals
LRE, // Left-to-Right Embedding LRE
LRO, // Left-to-Right Override LRO
RLE, // Right-to-Left Embedding RLE
RLO, // Right-to-Left Override RLO
PDF, // Pop Directional Format PDF
LRI, // Left-to-Right Isolate LRI
RLI, // Right-to-Left Isolate RLI
FSI, // First Strong Isolate FSI
PDI, // Pop Directional Isolate PDI
}
Bidi :: struct {
bc: Bidi_Class,
bc: Bidi_Class,
bmg: Maybe(rune), // mirrored glyph
m: bool, // Bidi mirrored
c: bool, // Bidi control property
bpt : Paired_Brack_Type, // bidi paired bracket type
bpb : rune, // bidi paired bracket properties
m: bool, // Bidi mirrored
c: bool, // Bidi control property
pb: Paired_Bracket_Type, // bidi paired bracket type
bpb: rune, // bidi paired bracket properties
}
Decomposition_Type :: enum {
Nil = 0,
can,
@@ -490,71 +488,71 @@ Decomposition_Type :: enum {
Trinary_Bool :: enum {
Maybe = -1,
False = 0,
True = 1,
False = 0,
True = 1,
}
Decomposition_Mapping :: distinct [dynamic]rune
Decomposition :: struct {
dt: Decomposition_Type, // Decomposition type
dm: Decomposition_Mapping, // Decomposition Mapping
ce: bool, // Composition Exclusion
comp_ex: bool, // Full Composition Exclusion
nfc_quick_check: Trinary_Bool,
nfd_quick_check: bool,
dt: Decomposition_Type, // Decomposition type
dm: Decomposition_Mapping, // Decomposition Mapping
ce: bool, // Composition Exclusion
comp_ex: bool, // Full Composition Exclusion
nfc_quick_check: Trinary_Bool,
nfd_quick_check: bool,
nfkc_quick_check: Trinary_Bool,
nfkd_quick_check: bool,
}
Numeric_Type :: enum {
None = 0, // None
Decimal, // De
Digit, // Di
Numeric, // Nu
None = 0, // None
Decimal, // De
Digit, // Di
Numeric, // Nu
}
/*
Note: Value is NAN when numberator and denominator ar 0
*/
Numberic_Value :: struct {
numerator: int,
numerator: int,
denominator: int,
}
Char :: struct {
cp: rune,
name: string,
gc: General_Category,
ccc: Combining_Class,
bc: Bidi_Class,
dt: Decomposition_Type,
dm: Decomposition_Mapping,
nt: Numeric_Type,
nv: Numberic_Value,
bm: bool,
cp: rune,
name: string,
gc: General_Category,
ccc: Combining_Class,
bc: Bidi_Class,
dt: Decomposition_Type,
dm: Decomposition_Mapping,
nt: Numeric_Type,
nv: Numberic_Value,
bm: bool,
name1: string,
sum: string, // Simple uppercase mapping
slm: string, // Simple lowercase mapping
stm: string, // Simple titlecase_mapping
sum: string, // Simple uppercase mapping
slm: string, // Simple lowercase mapping
stm: string, // Simple titlecase_mapping
}
Char_Range :: struct {
first_cp: rune,
last_cp: rune,
name: string,
gc: General_Category,
ccc: Combining_Class,
bc: Bidi_Class,
dt: Decomposition_Type,
dm: Decomposition_Mapping,
nt: Numeric_Type,
nv: Numberic_Value,
bm: bool,
name1: string,
sum: string, // Simple uppercase mapping
slm: string, // Simple lowercase mapping
stm: string, // Simple titlecase_mapping
last_cp: rune,
name: string,
gc: General_Category,
ccc: Combining_Class,
bc: Bidi_Class,
dt: Decomposition_Type,
dm: Decomposition_Mapping,
nt: Numeric_Type,
nv: Numberic_Value,
bm: bool,
name1: string,
sum: string, // Simple uppercase mapping
slm: string, // Simple lowercase mapping
stm: string, // Simple titlecase_mapping
}
Chars :: union {
@@ -565,7 +563,8 @@ Chars :: union {
Unicode_Data :: distinct [dynamic]Chars
PropList_Property :: enum {
Prop_List_Property :: enum {
Unknown,
White_Space,
Bidi_Control,
Join_Control,
@@ -613,6 +612,7 @@ UCD_Error :: enum {
Element_Not_Repertoire,
Extra_Fields,
Unknown_Property,
Unknown_Bidi_Class,
NO_REPERTOIRE,
UNEXPECTED_STRING,
@@ -632,71 +632,63 @@ Error :: union #shared_nil {
Range_u16 :: struct {
first: u16,
last: u16,
last: u16,
}
Range_i32 :: struct {
first: i32,
last: i32,
last: i32,
}
Range_Rune :: struct {
first: rune,
last: rune,
last: rune,
}
Dynamic_Range :: struct {
single_16 : [dynamic]u16,
ranges_16 : [dynamic]Range_u16,
single_32 : [dynamic]i32,
ranges_32 : [dynamic]Range_i32,
single_16: [dynamic]u16,
ranges_16: [dynamic]Range_u16,
single_32: [dynamic]i32,
ranges_32: [dynamic]Range_i32,
}
append_to_dynamic_range :: proc(
dr: ^Dynamic_Range,
range: Range_Rune,
allocator := context.allocator,
) {
append_to_dynamic_range :: proc(dr: ^Dynamic_Range, range: Range_Rune, allocator := context.allocator) {
if range.first == range.last && range.first <= 0xFFFF {
if len(dr.single_16) == 0 {
dr.single_16 = make([dynamic]u16, 0, 512, allocator)
}
append(&dr.single_16, cast(u16) range.first)
append(&dr.single_16, cast(u16)range.first)
} else if range.first == range.last {
if len(dr.single_32) == 0 {
dr.single_32 = make([dynamic]i32, 0, 512, allocator)
}
append(&dr.single_32, cast(i32) range.first)
append(&dr.single_32, cast(i32)range.first)
} else if range.first <= 0xFFFF && range.last <= 0xFFFF {
if len(dr.ranges_16) == 0 {
dr.ranges_16 = make([dynamic]Range_u16, 0, 128, allocator)
}
r := Range_u16{ cast(u16)range.first, cast(u16) range.last}
r := Range_u16{ cast(u16)range.first, cast(u16)range.last}
append(&dr.ranges_16, r)
} else {
if len(dr.ranges_32) == 0 {
dr.ranges_32 = make([dynamic]Range_i32, 0, 128, allocator)
}
r := Range_i32{ cast(i32)range.first, cast(i32) range.last}
r := Range_i32{ cast(i32)range.first, cast(i32)range.last}
append(&dr.ranges_32, r)
}
}
destroy_dynamic_range :: proc (
dr: Dynamic_Range,
){
destroy_dynamic_range :: proc(dr: Dynamic_Range) {
delete(dr.ranges_16)
delete(dr.ranges_32)
delete(dr.single_16)
delete(dr.single_32)
}
destroy_general_category_ranges :: proc(
gcr: [General_Category]Dynamic_Range,
){
destroy_general_category_ranges :: proc(gcr: [General_Category]Dynamic_Range) {
for r in gcr {
destroy_dynamic_range(r)
}
}
}

View File

@@ -98,38 +98,39 @@ load_unicode_data :: proc(filename: string, allocator := context.allocator) -> (
nt = .None
}
case 9: // Bidi mirrored
case 9: // Bidi mirrored
case 10: // Unicode 1 Name (Obsolete as of 6.2.0)
case 11: // should be null
case 12:
case 13:
case 14:
case:
unreachable()
case:
err = .Extra_Fields
return
}
}
if is_range {
cr : Char_Range
cr.gc = gc
cr.first_cp = first_cp
cr.last_cp = cp
cr.name = name
cr.nt = nt
append(&unicode_data, cr)
append(&unicode_data, Char_Range {
gc = gc,
first_cp = first_cp,
last_cp = cp,
name = name,
nt = nt,
})
} else {
c : Char
c.gc = gc
c.cp = cp
c.name = name
c.nt = nt
append(&unicode_data, c)
append(&unicode_data, Char{
gc = gc,
cp = cp,
name = name,
nt = nt,
})
}
}
return
}
destroy_unicode_data :: proc(unicode_data: Unicode_Data){
destroy_unicode_data :: proc(unicode_data: Unicode_Data) {
for point in unicode_data {
switch p in point {
case Char:
@@ -143,9 +144,9 @@ destroy_unicode_data :: proc(unicode_data: Unicode_Data){
gc_ranges :: proc(ud: ^Unicode_Data, allocator := context.allocator) -> (lst: [General_Category]Dynamic_Range) {
range := Range_Rune {
range := Range_Rune{
first = -1,
last = -1,
last = -1,
}
gc: General_Category
@@ -174,6 +175,7 @@ gc_ranges :: proc(ud: ^Unicode_Data, allocator := context.allocator) -> (lst: [G
range.last = -1
}
}
if range.first != -1 {
append_to_dynamic_range(&lst[gc], range, allocator)
}
@@ -274,15 +276,10 @@ load_property_list :: proc(filename: string, allocator := context.allocator) ->
field := strings.trim_space(_field)
switch i {
case 0: // Code point or code point range
rr.first, rr.last = decode_rune(field) or_return
case 1:
prop = string_to_proplist_property(field) or_return
case:
err = UCD_Error.Extra_Fields
return
// Code point or code point range
case 0: rr.first, rr.last = decode_rune(field) or_return
case 1: prop = string_to_proplist_property(field) or_return
case: return {}, .Extra_Fields
}
}