diff --git a/core/unicode/tools/ucd/string_to.odin b/core/unicode/tools/ucd/string_to.odin index 7c9115f21..4f07d7e67 100644 --- a/core/unicode/tools/ucd/string_to.odin +++ b/core/unicode/tools/ucd/string_to.odin @@ -1,394 +1,162 @@ package ucd -string_to_general_category :: proc "contextless"( - str: string, -) -> (gc: General_Category, err: Error) { +string_to_general_category :: proc "contextless" (str: string) -> (gc: General_Category, err: Error) { switch str { - case "Lu": - gc = .Lu - case "Ll": - gc = .Ll - case "Lt": - gc = .Lt - case "Lm": - gc = .Lm - case "Lo": - gc = .Lo - case "Mn": - gc = .Mn - case "Mc": - gc = .Mc - case "Me": - gc = .Me - case "Nd": - gc = .Nd - case "Nl": - gc = .Nl - case "No": - gc = .No - case "Pc": - gc = .Pc - case "Pd": - gc = .Pd - case "Ps": - gc = .Ps - case "Pe": - gc = .Pe - case "Pi": - gc = .Pi - case "Pf": - gc = .Pf - case "Po": - gc = .Po - case "Sm": - gc = .Sm - case "Sc": - gc = .Sc - case "Sk": - gc = .Sk - case "So": - gc = .So - case "Zs": - gc = .Zs - case "Zl": - gc = .Zl - case "Zp": - gc = .Zp - case "Cc": - gc = .Cc - case "Cf": - gc = .Cf - case "Cs": - gc = .Cs - case "Co": - gc = .Co - case "Cn": - gc = .Cn - case: - err = UCD_Error.Invalid_General_Category + case "Lu": gc = .Lu + case "Ll": gc = .Ll + case "Lt": gc = .Lt + case "Lm": gc = .Lm + case "Lo": gc = .Lo + case "Mn": gc = .Mn + case "Mc": gc = .Mc + case "Me": gc = .Me + case "Nd": gc = .Nd + case "Nl": gc = .Nl + case "No": gc = .No + case "Pc": gc = .Pc + case "Pd": gc = .Pd + case "Ps": gc = .Ps + case "Pe": gc = .Pe + case "Pi": gc = .Pi + case "Pf": gc = .Pf + case "Po": gc = .Po + case "Sm": gc = .Sm + case "Sc": gc = .Sc + case "Sk": gc = .Sk + case "So": gc = .So + case "Zs": gc = .Zs + case "Zl": gc = .Zl + case "Zp": gc = .Zp + case "Cc": gc = .Cc + case "Cf": gc = .Cf + case "Cs": gc = .Cs + case "Co": gc = .Co + case "Cn": gc = .Cn + case: err = .Invalid_General_Category } return } - -string_to_age :: proc "contextless" ( - str: string, -) -> (age: Age, err: Error) { +string_to_proplist_property :: proc(str: string) -> (prop: Prop_List_Property) { switch str { - case "1.1": - age = .Age_1_1 - return - - case "2.0": - age = .Age_2_0 - return - - case "2.1": - age = .Age_2_1 - return - - case "3.0": - age = .Age_3_0 - return - - case "3.1": - age = .Age_3_1 - return - - case "3.2": - age = .Age_3_2 - return - - case "4.0": - age = .Age_4_0 - return - - case "4.1": - age = .Age_4_1 - return - - case "5.0": - age = .Age_5_0 - return - - case "5.1": - age = .Age_5_1 - return - - case "5.2": - age = .Age_5_2 - return - - case "6.0": - age = .Age_6_0 - return - - case "6.1": - age = .Age_6_1 - return - - case "6.2": - age = .Age_6_2 - return - - case "6.3": - age = .Age_6_3 - return - - case "7.0": - age = .Age_7_0 - return - - case "8.0": - age = .Age_8_0 - return - - case "9.0": - age = .Age_9_0 - return - - case "10.0": - age = .Age_10_0 - return - - case "11.0": - age = .Age_11_0 - return - - case "12.0": - age = .Age_12_0 - return - - case "12.1": - age = .Age_12_1 - return - - case "13.0": - age = .Age_13_0 - return - - case "14.0": - age = .Age_14_0 - return - - case "15.0": - age = .Age_15_0 - return - - case "15.1": - age = .Age_15_1 - return - - case "16.0": - age = .Age_16_0 - return - - case "17.0": - age = .Age_17_0 - return - - case "unassigned": - age = .Age_Unassigned - return - - case: - // NOTE: Should this return an error instead? - unreachable() + case "White_Space": prop = .White_Space + case "Bidi_Control": prop = .Bidi_Control + case "Join_Control": prop = .Join_Control + case "Dash": prop = .Dash + case "Hyphen": prop = .Hyphen + case "Quotation_Mark": prop = .Quotation_Mark + case "Terminal_Punctuation": prop = .Terminal_Punctuation + case "Other_Math": prop = .Other_Math + case "Hex_Digit": prop = .Hex_Digit + case "ASCII_Hex_Digit": prop = .ASCII_Hex_Digit + case "Other_Alphabetic": prop = .Other_Alphabetic + case "Ideographic": prop = .Ideographic + case "Diacritic": prop = .Diacritic + case "Extender": prop = .Extender + case "Other_Lowercase": prop = .Other_Lowercase + case "Other_Uppercase": prop = .Other_Uppercase + case "Noncharacter_Code_Point": prop = .Noncharacter_Code_Point + case "Other_Grapheme_Extend": prop = .Other_Grapheme_Extend + case "IDS_Binary_Operator": prop = .IDS_Binary_Operator + case "IDS_Trinary_Operator": prop = .IDS_Trinary_Operator + case "IDS_Unary_Operator": prop = .IDS_Unary_Operator + case "Radical": prop = .Radical + case "Unified_Ideograph": prop = .Unified_Ideograph + case "Other_Default_Ignorable_Code_Point": prop = .Other_Default_Ignorable_Code_Point + case "Deprecated": prop = .Deprecated + case "Soft_Dotted": prop = .Soft_Dotted + case "Logical_Order_Exception": prop = .Logical_Order_Exception + case "Other_ID_Start": prop = .Other_ID_Start + case "Other_ID_Continue": prop = .Other_ID_Continue + case "ID_Compat_Math_Continue": prop = .ID_Compat_Math_Continue + case "ID_Compat_Math_Start": prop = .ID_Compat_Math_Start + case "Sentence_Terminal": prop = .Sentence_Terminal + case "Variation_Selector": prop = .Variation_Selector + case "Pattern_White_Space": prop = .Pattern_White_Space + case "Pattern_Syntax": prop = .Pattern_Syntax + case "Prepended_Concatenation_Mark": prop = .Prepended_Concatenation_Mark + case "Regional_Indicator": prop = .Regional_Indicator + case "Modifier_Combining_Mark": prop = .Modifier_Combining_Mark + case: prop = .Unknown_Property } + return } - -string_to_paired_bracket_type :: proc "contextless"(str: string) -> Paired_Brack_Type { +@(deprecated="Unused?") +string_to_age :: proc "contextless" (str: string) -> (age: Age) { switch str { - case "o": - return .Open - case "c": - return .Close - case "n": - return .None - case: - // TODO: Add error for this - unreachable() + case "1.1": age = .Age_1_1 + case "2.0": age = .Age_2_0 + case "2.1": age = .Age_2_1 + case "3.0": age = .Age_3_0 + case "3.1": age = .Age_3_1 + case "3.2": age = .Age_3_2 + case "4.0": age = .Age_4_0 + case "4.1": age = .Age_4_1 + case "5.0": age = .Age_5_0 + case "5.1": age = .Age_5_1 + case "5.2": age = .Age_5_2 + case "6.0": age = .Age_6_0 + case "6.1": age = .Age_6_1 + case "6.2": age = .Age_6_2 + case "6.3": age = .Age_6_3 + case "7.0": age = .Age_7_0 + case "8.0": age = .Age_8_0 + case "9.0": age = .Age_9_0 + case "10.0": age = .Age_10_0 + case "11.0": age = .Age_11_0 + case "12.0": age = .Age_12_0 + case "12.1": age = .Age_12_1 + case "13.0": age = .Age_13_0 + case "14.0": age = .Age_14_0 + case "15.0": age = .Age_15_0 + case "15.1": age = .Age_15_1 + case "16.0": age = .Age_16_0 + case "17.0": age = .Age_17_0 + case "unassigned": age = .Age_Unassigned + case: age = .Age_Unknown } + return } -string_to_bidi_class :: proc "contextless"(str: string) -> Bidi_Class { +@(deprecated="Unused?") +string_to_paired_bracket_type :: proc "contextless" (str: string) -> (pbt: Paired_Bracket_Type) { switch str { - case "AL": - return .AL - case "AN": - return .AN - case "B": - return .B - case "BN": - return .BN - case "CS": - return .CS - case "EN": - return .EN - case "ES": - return .ES - case "ET": - return .ET - case "FSI": - return .FSI - case "L": - return .L - case "LRE": - return .LRE - case "LRI": - return .LRI - case "LRO": - return .LRO - case "NSM": - return .NSM - case "ON": - return .ON - case "PDF": - return .PDF - case "PDI": - return .PDI - case "R": - return .R - case "RLE": - return .RLE - case "RLI": - return .RLI - case "RLO": - return .RLO - case "S": - return .S - case "WS": - return .WS - case: unreachable() // TODO: Add error for this + case "o": pbt = .Open + case "c": pbt = .Close + case "n": pbt = .None + case: pbt = .Unknown } + return } -string_to_proplist_property :: proc(str: string) -> ( - prop: PropList_Property, - err: UCD_Error, -) { - +@(deprecated="Unused?") +string_to_bidi_class :: proc "contextless" (str: string) -> (class: Bidi_Class) { switch str { - case "White_Space": - prop = .White_Space - - case "Bidi_Control": - prop = .Bidi_Control - - case "Join_Control": - prop = .Join_Control - - case "Dash": - prop = .Dash - - case "Hyphen": - prop = .Hyphen - - case "Quotation_Mark": - prop = .Quotation_Mark - - case "Terminal_Punctuation": - prop = .Terminal_Punctuation - - case "Other_Math": - prop = .Other_Math - - case "Hex_Digit": - prop = .Hex_Digit - - case "ASCII_Hex_Digit": - prop = .ASCII_Hex_Digit - - case "Other_Alphabetic": - prop = .Other_Alphabetic - - case "Ideographic": - prop = .Ideographic - - case "Diacritic": - prop = .Diacritic - - case "Extender": - prop = .Extender - - case "Other_Lowercase": - prop = .Other_Lowercase - - case "Other_Uppercase": - prop = .Other_Uppercase - - - case "Noncharacter_Code_Point": - prop = .Noncharacter_Code_Point - - case "Other_Grapheme_Extend": - prop = .Other_Grapheme_Extend - - case "IDS_Binary_Operator": - prop = .IDS_Binary_Operator - - case "IDS_Trinary_Operator": - prop = .IDS_Trinary_Operator - - case "IDS_Unary_Operator": - prop = .IDS_Unary_Operator - - case "Radical": - prop = .Radical - - case "Unified_Ideograph": - prop = .Unified_Ideograph - - case "Other_Default_Ignorable_Code_Point": - prop = .Other_Default_Ignorable_Code_Point - - case "Deprecated": - prop = .Deprecated - - case "Soft_Dotted": - prop = .Soft_Dotted - - case "Logical_Order_Exception": - prop = .Logical_Order_Exception - - case "Other_ID_Start": - prop = .Other_ID_Start - - case "Other_ID_Continue": - prop = .Other_ID_Continue - - case "ID_Compat_Math_Continue": - prop = .ID_Compat_Math_Continue - - case "ID_Compat_Math_Start": - prop = .ID_Compat_Math_Start - - case "Sentence_Terminal": - prop = .Sentence_Terminal - - case "Variation_Selector": - prop = .Variation_Selector - - case "Pattern_White_Space": - prop = .Pattern_White_Space - - case "Pattern_Syntax": - prop = .Pattern_Syntax - - case "Prepended_Concatenation_Mark": - prop = .Prepended_Concatenation_Mark - - case "Regional_Indicator": - prop = .Regional_Indicator - - case "Modifier_Combining_Mark": - prop = .Modifier_Combining_Mark - - case: - err = .Unknown_Property - return + case "AL": class = .AL + case "AN": class = .AN + case "B": class = .B + case "BN": class = .BN + case "CS": class = .CS + case "EN": class = .EN + case "ES": class = .ES + case "ET": class = .ET + case "FSI": class = .FSI + case "L": class = .L + case "LRE": class = .LRE + case "LRI": class = .LRI + case "LRO": class = .LRO + case "NSM": class = .NSM + case "ON": class = .ON + case "PDF": class = .PDF + case "PDI": class = .PDI + case "R": class = .R + case "RLE": class = .RLE + case "RLI": class = .RLI + case "RLO": class = .RLO + case "S": class = .S + case "WS": class = .WS + case: class = .Unknown } - - return -} + return +} \ No newline at end of file diff --git a/core/unicode/tools/ucd/types.odin b/core/unicode/tools/ucd/types.odin index 0ba6f8909..5b87f96f8 100644 --- a/core/unicode/tools/ucd/types.odin +++ b/core/unicode/tools/ucd/types.odin @@ -3,7 +3,7 @@ package ucd import "core:os" Age :: enum byte { - Nil = 0, + Age_Unknown = 0, Age_1_1, Age_2_0, Age_2_1, @@ -421,51 +421,49 @@ Block :: enum { Combining_Class :: distinct byte -Paired_Brack_Type :: enum { - Nil, +Paired_Bracket_Type :: enum { + Unknown, Open, Close, None, } Bidi_Class :: enum { - Nil, // - L, // Left-to-Right LRM - R, // Right-to-Left RLM - AL, // Right-to-Left Arabic ALM - EN, // European Number - ES, // European Number Separator - ET, // European Number Terminator - AN, // Arabic Number - CS, // Common Number Separator - NSM, // Nonspacing Mark - BN, // Boundary Neutral - B, // Paragraph Separator - S, // Segment Separator - WS, // Whitespace - ON, // Other Neutrals - LRE, // Left-to-Right Embedding LRE - LRO, // Left-to-Right Override LRO - RLE, // Right-to-Left Embedding RLE - RLO, // Right-to-Left Override RLO - PDF, // Pop Directional Format PDF - LRI, // Left-to-Right Isolate LRI - RLI, // Right-to-Left Isolate RLI - FSI, // First Strong Isolate FSI - PDI, // Pop Directional Isolate PDI + Unknown, // + L, // Left-to-Right LRM + R, // Right-to-Left RLM + AL, // Right-to-Left Arabic ALM + EN, // European Number + ES, // European Number Separator + ET, // European Number Terminator + AN, // Arabic Number + CS, // Common Number Separator + NSM, // Nonspacing Mark + BN, // Boundary Neutral + B, // Paragraph Separator + S, // Segment Separator + WS, // Whitespace + ON, // Other Neutrals + LRE, // Left-to-Right Embedding LRE + LRO, // Left-to-Right Override LRO + RLE, // Right-to-Left Embedding RLE + RLO, // Right-to-Left Override RLO + PDF, // Pop Directional Format PDF + LRI, // Left-to-Right Isolate LRI + RLI, // Right-to-Left Isolate RLI + FSI, // First Strong Isolate FSI + PDI, // Pop Directional Isolate PDI } - Bidi :: struct { - bc: Bidi_Class, + bc: Bidi_Class, bmg: Maybe(rune), // mirrored glyph - m: bool, // Bidi mirrored - c: bool, // Bidi control property - bpt : Paired_Brack_Type, // bidi paired bracket type - bpb : rune, // bidi paired bracket properties + m: bool, // Bidi mirrored + c: bool, // Bidi control property + pb: Paired_Bracket_Type, // bidi paired bracket type + bpb: rune, // bidi paired bracket properties } - Decomposition_Type :: enum { Nil = 0, can, @@ -490,71 +488,71 @@ Decomposition_Type :: enum { Trinary_Bool :: enum { Maybe = -1, - False = 0, - True = 1, + False = 0, + True = 1, } Decomposition_Mapping :: distinct [dynamic]rune Decomposition :: struct { - dt: Decomposition_Type, // Decomposition type - dm: Decomposition_Mapping, // Decomposition Mapping - ce: bool, // Composition Exclusion - comp_ex: bool, // Full Composition Exclusion - nfc_quick_check: Trinary_Bool, - nfd_quick_check: bool, + dt: Decomposition_Type, // Decomposition type + dm: Decomposition_Mapping, // Decomposition Mapping + ce: bool, // Composition Exclusion + comp_ex: bool, // Full Composition Exclusion + nfc_quick_check: Trinary_Bool, + nfd_quick_check: bool, nfkc_quick_check: Trinary_Bool, nfkd_quick_check: bool, } Numeric_Type :: enum { - None = 0, // None - Decimal, // De - Digit, // Di - Numeric, // Nu + None = 0, // None + Decimal, // De + Digit, // Di + Numeric, // Nu } /* Note: Value is NAN when numberator and denominator ar 0 */ Numberic_Value :: struct { - numerator: int, + numerator: int, denominator: int, } Char :: struct { - cp: rune, - name: string, - gc: General_Category, - ccc: Combining_Class, - bc: Bidi_Class, - dt: Decomposition_Type, - dm: Decomposition_Mapping, - nt: Numeric_Type, - nv: Numberic_Value, - bm: bool, + cp: rune, + name: string, + gc: General_Category, + ccc: Combining_Class, + bc: Bidi_Class, + dt: Decomposition_Type, + dm: Decomposition_Mapping, + nt: Numeric_Type, + nv: Numberic_Value, + bm: bool, name1: string, - sum: string, // Simple uppercase mapping - slm: string, // Simple lowercase mapping - stm: string, // Simple titlecase_mapping + sum: string, // Simple uppercase mapping + slm: string, // Simple lowercase mapping + stm: string, // Simple titlecase_mapping } Char_Range :: struct { first_cp: rune, - last_cp: rune, - name: string, - gc: General_Category, - ccc: Combining_Class, - bc: Bidi_Class, - dt: Decomposition_Type, - dm: Decomposition_Mapping, - nt: Numeric_Type, - nv: Numberic_Value, - bm: bool, - name1: string, - sum: string, // Simple uppercase mapping - slm: string, // Simple lowercase mapping - stm: string, // Simple titlecase_mapping + last_cp: rune, + name: string, + gc: General_Category, + ccc: Combining_Class, + bc: Bidi_Class, + dt: Decomposition_Type, + dm: Decomposition_Mapping, + nt: Numeric_Type, + nv: Numberic_Value, + bm: bool, + name1: string, + sum: string, // Simple uppercase mapping + slm: string, // Simple lowercase mapping + stm: string, // Simple titlecase_mapping } Chars :: union { @@ -565,7 +563,8 @@ Chars :: union { Unicode_Data :: distinct [dynamic]Chars -PropList_Property :: enum { +Prop_List_Property :: enum { + Unknown, White_Space, Bidi_Control, Join_Control, @@ -613,6 +612,7 @@ UCD_Error :: enum { Element_Not_Repertoire, Extra_Fields, Unknown_Property, + Unknown_Bidi_Class, NO_REPERTOIRE, UNEXPECTED_STRING, @@ -632,71 +632,63 @@ Error :: union #shared_nil { Range_u16 :: struct { first: u16, - last: u16, + last: u16, } Range_i32 :: struct { first: i32, - last: i32, + last: i32, } Range_Rune :: struct { first: rune, - last: rune, + last: rune, } Dynamic_Range :: struct { - single_16 : [dynamic]u16, - ranges_16 : [dynamic]Range_u16, - single_32 : [dynamic]i32, - ranges_32 : [dynamic]Range_i32, + single_16: [dynamic]u16, + ranges_16: [dynamic]Range_u16, + single_32: [dynamic]i32, + ranges_32: [dynamic]Range_i32, } -append_to_dynamic_range :: proc( - dr: ^Dynamic_Range, - range: Range_Rune, - allocator := context.allocator, -) { +append_to_dynamic_range :: proc(dr: ^Dynamic_Range, range: Range_Rune, allocator := context.allocator) { if range.first == range.last && range.first <= 0xFFFF { if len(dr.single_16) == 0 { dr.single_16 = make([dynamic]u16, 0, 512, allocator) } - append(&dr.single_16, cast(u16) range.first) + append(&dr.single_16, cast(u16)range.first) } else if range.first == range.last { if len(dr.single_32) == 0 { dr.single_32 = make([dynamic]i32, 0, 512, allocator) } - append(&dr.single_32, cast(i32) range.first) + append(&dr.single_32, cast(i32)range.first) } else if range.first <= 0xFFFF && range.last <= 0xFFFF { if len(dr.ranges_16) == 0 { dr.ranges_16 = make([dynamic]Range_u16, 0, 128, allocator) } - r := Range_u16{ cast(u16)range.first, cast(u16) range.last} + r := Range_u16{ cast(u16)range.first, cast(u16)range.last} append(&dr.ranges_16, r) } else { if len(dr.ranges_32) == 0 { dr.ranges_32 = make([dynamic]Range_i32, 0, 128, allocator) } - r := Range_i32{ cast(i32)range.first, cast(i32) range.last} + r := Range_i32{ cast(i32)range.first, cast(i32)range.last} append(&dr.ranges_32, r) } } -destroy_dynamic_range :: proc ( - dr: Dynamic_Range, -){ +destroy_dynamic_range :: proc(dr: Dynamic_Range) { delete(dr.ranges_16) delete(dr.ranges_32) delete(dr.single_16) delete(dr.single_32) } -destroy_general_category_ranges :: proc( - gcr: [General_Category]Dynamic_Range, -){ +destroy_general_category_ranges :: proc(gcr: [General_Category]Dynamic_Range) { for r in gcr { destroy_dynamic_range(r) } -} +} \ No newline at end of file diff --git a/core/unicode/tools/ucd/ucd.odin b/core/unicode/tools/ucd/ucd.odin index ea6283343..328fe06b8 100644 --- a/core/unicode/tools/ucd/ucd.odin +++ b/core/unicode/tools/ucd/ucd.odin @@ -98,38 +98,39 @@ load_unicode_data :: proc(filename: string, allocator := context.allocator) -> ( nt = .None } - case 9: // Bidi mirrored + case 9: // Bidi mirrored case 10: // Unicode 1 Name (Obsolete as of 6.2.0) case 11: // should be null case 12: case 13: case 14: - case: - unreachable() + case: + err = .Extra_Fields + return } } if is_range { - cr : Char_Range - cr.gc = gc - cr.first_cp = first_cp - cr.last_cp = cp - cr.name = name - cr.nt = nt - append(&unicode_data, cr) + append(&unicode_data, Char_Range { + gc = gc, + first_cp = first_cp, + last_cp = cp, + name = name, + nt = nt, + }) } else { - c : Char - c.gc = gc - c.cp = cp - c.name = name - c.nt = nt - append(&unicode_data, c) + append(&unicode_data, Char{ + gc = gc, + cp = cp, + name = name, + nt = nt, + }) } } return } -destroy_unicode_data :: proc(unicode_data: Unicode_Data){ +destroy_unicode_data :: proc(unicode_data: Unicode_Data) { for point in unicode_data { switch p in point { case Char: @@ -143,9 +144,9 @@ destroy_unicode_data :: proc(unicode_data: Unicode_Data){ gc_ranges :: proc(ud: ^Unicode_Data, allocator := context.allocator) -> (lst: [General_Category]Dynamic_Range) { - range := Range_Rune { + range := Range_Rune{ first = -1, - last = -1, + last = -1, } gc: General_Category @@ -174,6 +175,7 @@ gc_ranges :: proc(ud: ^Unicode_Data, allocator := context.allocator) -> (lst: [G range.last = -1 } } + if range.first != -1 { append_to_dynamic_range(&lst[gc], range, allocator) } @@ -274,15 +276,10 @@ load_property_list :: proc(filename: string, allocator := context.allocator) -> field := strings.trim_space(_field) switch i { - case 0: // Code point or code point range - rr.first, rr.last = decode_rune(field) or_return - - case 1: - prop = string_to_proplist_property(field) or_return - - case: - err = UCD_Error.Extra_Fields - return + // Code point or code point range + case 0: rr.first, rr.last = decode_rune(field) or_return + case 1: prop = string_to_proplist_property(field) or_return + case: return {}, .Extra_Fields } }