From 09e1c0fa27a262d7fbfaa5b3e305054304847e75 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 16:19:13 +0200 Subject: [PATCH] [i18n] Add tests. --- core/text/i18n/doc.odin | 111 ++++++++++++ core/text/i18n/example/i18n_example.odin | 99 ----------- core/text/i18n/gettext.odin | 9 +- core/text/i18n/i18n.odin | 8 + core/text/i18n/qt_linguist.odin | 8 +- examples/all/all_main.odin | 4 + tests/core/Makefile | 5 +- tests/core/assets/I18N/duplicate-key.ts | 22 +++ tests/core/build.bat | 7 +- tests/core/text/i18n/test_core_text_i18n.odin | 165 ++++++++++++++++++ 10 files changed, 329 insertions(+), 109 deletions(-) create mode 100644 core/text/i18n/doc.odin delete mode 100644 core/text/i18n/example/i18n_example.odin create mode 100644 tests/core/assets/I18N/duplicate-key.ts create mode 100644 tests/core/text/i18n/test_core_text_i18n.odin diff --git a/core/text/i18n/doc.odin b/core/text/i18n/doc.odin new file mode 100644 index 000000000..cff1ce11f --- /dev/null +++ b/core/text/i18n/doc.odin @@ -0,0 +1,111 @@ +//+ignore +package i18n + +/* + The i18n package is flexible and easy to use. + + It has one call to get a translation: `get`, which the user can alias into something like `T`. + + `get`, referred to as `T` here, has a few different signatures. + All of them will return the key if the entry can't be found in the active translation catalog. + + - `T(key)` returns the translation of `key`. + - `T(key, n)` returns a pluralized translation of `key` according to value `n`. + + - `T(section, key)` returns the translation of `key` in `section`. + - `T(section, key, n)` returns a pluralized translation of `key` in `section` according to value `n`. + + By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use. + If you want to override which translation to use, for example in a language preview dialog, you can use the following: + + - `T(key, n, catalog)` returns the pluralized version of `key` from explictly supplied catalog. + - `T(section, key, n, catalog)` returns the pluralized version of `key` in `section` from explictly supplied catalog. + + If a catalog has translation contexts or sections, then ommitting it in the above calls looks up in section "". + + The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form. + Passing n != 1 returns plural form 1. + + Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser. + This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used. + + You can also assign it to a loaded catalog after parsing, of course. + + Some code examples follow. +*/ + +/* +```cpp +import "core:fmt" +import "core:text/i18n" + +T :: i18n.get + +mo :: proc() { + using fmt + + err: i18n.Error + + /* + Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo")) + defer i18n.destroy() + + if err != .None { return } + + /* + These are in the .MO catalog. + */ + println("-----") + println(T("")) + println("-----") + println(T("There are 69,105 leaves here.")) + println("-----") + println(T("Hellope, World!")) + println("-----") + // We pass 1 into `T` to get the singular format string, then 1 again into printf. + printf(T("There is %d leaf.\n", 1), 1) + // We pass 42 into `T` to get the plural format string, then 42 again into printf. + printf(T("There is %d leaf.\n", 42), 42) + + /* + This isn't in the translation catalog, so the key is passed back untranslated. + */ + println("-----") + println(T("Come visit us on Discord!")) +} + +qt :: proc() { + using fmt + + err: i18n.Error + + /* + Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts")) + defer i18n.destroy() + + if err != .None { + return + } + + /* + These are in the .TS catalog. As you can see they have sections. + */ + println("--- Page section ---") + println("Page:Text for translation =", T("Page", "Text for translation")) + println("-----") + println("Page:Also text to translate =", T("Page", "Also text to translate")) + println("-----") + println("--- installscript section ---") + println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall")) + println("-----") + println("--- apple_count section ---") + println("apple_count:%d apple(s) =") + println("\t 1 =", T("apple_count", "%d apple(s)", 1)) + println("\t 42 =", T("apple_count", "%d apple(s)", 42)) +} +``` +*/ \ No newline at end of file diff --git a/core/text/i18n/example/i18n_example.odin b/core/text/i18n/example/i18n_example.odin deleted file mode 100644 index 32eb38a7d..000000000 --- a/core/text/i18n/example/i18n_example.odin +++ /dev/null @@ -1,99 +0,0 @@ -package i18n_example - -import "core:mem" -import "core:fmt" -import "core:text/i18n" - -_T :: i18n.get - -mo :: proc() { - using fmt - - err: i18n.Error - - /* - Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. - */ - i18n.ACTIVE, err = i18n.parse_mo(#load("nl_NL.mo")) - defer i18n.destroy() - - if err != .None { return } - - /* - These are in the .MO catalog. - */ - println("-----") - println(_T("")) - println("-----") - println(_T("There are 69,105 leaves here.")) - println("-----") - println(_T("Hellope, World!")) - - /* - For ease of use, pluralized lookup can use both singular and plural form as key for the same translation. - This is a quirk of the GetText format which has separate keys for their different plurals. - */ - println("-----") - printf(_T("There is %d leaf.\n", 1), 1) - printf(_T("There is %d leaf.\n", 42), 42) - - printf(_T("There are %d leaves.\n", 1), 1) - printf(_T("There are %d leaves.\n", 42), 42) - - /* - This isn't. - */ - println("-----") - println(_T("Come visit us on Discord!")) -} - -qt :: proc() { - using fmt - - err: i18n.Error - - /* - Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. - */ - i18n.ACTIVE, err = i18n.parse_qt(#load("../../../../tests/core/assets/XML/nl_NL-qt-ts.ts")) - defer i18n.destroy() - - fmt.printf("parse_qt returned %v\n", err) - if err != .None { - return - } - - /* - These are in the .TS catalog. - */ - println("--- Page section ---") - println("Page:Text for translation =", _T("Page", "Text for translation")) - println("-----") - println("Page:Also text to translate =", _T("Page", "Also text to translate")) - println("-----") - println("--- installscript section ---") - println("installscript:99 bottles of beer on the wall =", _T("installscript", "99 bottles of beer on the wall")) - println("-----") - println("--- apple_count section ---") - println("apple_count:%d apple(s) =") - println("\t 1 =", _T("apple_count", "%d apple(s)", 1)) - println("\t 42 =", _T("apple_count", "%d apple(s)", 42)) -} - -main :: proc() { - using fmt - - track: mem.Tracking_Allocator - mem.tracking_allocator_init(&track, context.allocator) - context.allocator = mem.tracking_allocator(&track) - - // mo() - qt() - - if len(track.allocation_map) > 0 { - println() - for _, v in track.allocation_map { - printf("%v Leaked %v bytes.\n", v.location, v.size) - } - } -} \ No newline at end of file diff --git a/core/text/i18n/gettext.odin b/core/text/i18n/gettext.odin index 54c5a1111..eed73855b 100644 --- a/core/text/i18n/gettext.odin +++ b/core/text/i18n/gettext.odin @@ -8,6 +8,9 @@ package i18n A from-scratch implementation based after the specification found here: https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html + Options are ignored as they're not applicable to this format. + They're part of the signature for consistency with other catalog formats. + List of contributors: Jeroen van Rijn: Initial implementation. */ @@ -15,7 +18,7 @@ import "core:os" import "core:strings" import "core:bytes" -parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_mo_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator /* An MO file should have at least a 4-byte magic, 2 x 2 byte version info, @@ -115,7 +118,7 @@ parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allo return } -parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_mo_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator data, data_ok := os.read_entire_file(filename) @@ -123,7 +126,7 @@ parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allo if !data_ok { return {}, .File_Error } - return parse_mo_from_slice(data, pluralizer) + return parse_mo_from_slice(data, options, pluralizer, allocator) } parse_mo :: proc { parse_mo_file, parse_mo_from_slice } diff --git a/core/text/i18n/i18n.odin b/core/text/i18n/i18n.odin index 36204efd9..e007401af 100644 --- a/core/text/i18n/i18n.odin +++ b/core/text/i18n/i18n.odin @@ -74,6 +74,14 @@ Error :: enum { } +Parse_Options :: struct { + merge_sections: bool, +} + +DEFAULT_PARSE_OPTIONS :: Parse_Options{ + merge_sections = false, +} + /* Several ways to use: - get(key), which defaults to the singular form and i18n.ACTIVE catalog, or diff --git a/core/text/i18n/qt_linguist.odin b/core/text/i18n/qt_linguist.odin index 65d51444e..0a241c1aa 100644 --- a/core/text/i18n/qt_linguist.odin +++ b/core/text/i18n/qt_linguist.odin @@ -27,7 +27,7 @@ TS_XML_Options := xml.Options{ expected_doctype = "TS", } -parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator ts, xml_err := xml.parse(data, TS_XML_Options) @@ -59,7 +59,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = return translation, .TS_File_Expected_Context_Name, } - section_name := ts.elements[section_name_id].value + section_name := "" if options.merge_sections else ts.elements[section_name_id].value if section_name not_in translation.k_v { translation.k_v[section_name] = {} @@ -139,7 +139,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = return } -parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_qt_linguist_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator data, data_ok := os.read_entire_file(filename) @@ -147,7 +147,7 @@ parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int = if !data_ok { return {}, .File_Error } - return parse_qt_linguist_from_slice(data, pluralizer) + return parse_qt_linguist_from_slice(data, options, pluralizer, allocator) } parse_qt :: proc { parse_qt_linguist_file, parse_qt_linguist_from_slice } \ No newline at end of file diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index 27f199062..36acf7714 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -56,6 +56,7 @@ import csv "core:encoding/csv" import hxa "core:encoding/hxa" import json "core:encoding/json" import varint "core:encoding/varint" +import xml "core:encoding/xml" import fmt "core:fmt" import hash "core:hash" @@ -100,6 +101,7 @@ import strings "core:strings" import sync "core:sync" import testing "core:testing" import scanner "core:text/scanner" +import i18n "core:text/i18n" import thread "core:thread" import time "core:time" @@ -158,6 +160,7 @@ _ :: csv _ :: hxa _ :: json _ :: varint +_ :: xml _ :: fmt _ :: hash _ :: image @@ -192,6 +195,7 @@ _ :: strings _ :: sync _ :: testing _ :: scanner +_ :: i18n _ :: thread _ :: time _ :: unicode diff --git a/tests/core/Makefile b/tests/core/Makefile index 2c24fef75..1405ae5c6 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -26,9 +26,10 @@ noise_test: $(ODIN) run math/noise -out:test_noise encoding_test: - $(ODIN) run encoding/hxa -collection:tests=.. -out:test_hxa - $(ODIN) run encoding/json -out:test_json + $(ODIN) run encoding/hxa -out:test_hxa -collection:tests=.. + $(ODIN) run encoding/json -out:test_json $(ODIN) run encoding/varint -out:test_varint + $(ODIN) run encoding/xml -out:test_xml math_test: $(ODIN) run math/test_core_math.odin -file -collection:tests=.. -out:test_core_math diff --git a/tests/core/assets/I18N/duplicate-key.ts b/tests/core/assets/I18N/duplicate-key.ts new file mode 100644 index 000000000..a38824d01 --- /dev/null +++ b/tests/core/assets/I18N/duplicate-key.ts @@ -0,0 +1,22 @@ + + + + + Page + + %d apple(s) + commenting + Tekst om te vertalen + + + + apple_count + + %d apple(s) + + %d appel + %d appels + + + + diff --git a/tests/core/build.bat b/tests/core/build.bat index 8e4ba1d15..77ff38038 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -64,4 +64,9 @@ echo --- echo --- echo Running core:reflect tests echo --- -%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe \ No newline at end of file +%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe + +echo --- +echo Running core:text/i18n tests +echo --- +%PATH_TO_ODIN% run text\i18n %COMMON% -out:test_core_i18n.exe \ No newline at end of file diff --git a/tests/core/text/i18n/test_core_text_i18n.odin b/tests/core/text/i18n/test_core_text_i18n.odin new file mode 100644 index 000000000..ba668c4fd --- /dev/null +++ b/tests/core/text/i18n/test_core_text_i18n.odin @@ -0,0 +1,165 @@ +package test_core_text_i18n + +import "core:mem" +import "core:fmt" +import "core:os" +import "core:testing" +import "core:text/i18n" + +TEST_count := 0 +TEST_fail := 0 + +when ODIN_TEST { + expect :: testing.expect + log :: testing.log +} else { + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } +} +T :: i18n.get + +Test :: struct { + section: string, + key: string, + val: string, + n: int, +} + +Test_Suite :: struct { + file: string, + loader: proc(string, i18n.Parse_Options, proc(int) -> int, mem.Allocator) -> (^i18n.Translation, i18n.Error), + err: i18n.Error, + options: i18n.Parse_Options, + tests: []Test, +} + +TESTS := []Test_Suite{ + { + file = "assets/I18N/nl_NL.mo", + loader = i18n.parse_mo_file, + tests = { + // These are in the catalog. + { "", "There are 69,105 leaves here.", "Er zijn hier 69.105 bladeren.", 1 }, + { "", "Hellope, World!", "Hallo, Wereld!", 1 }, + { "", "There is %d leaf.\n", "Er is %d blad.\n", 1 }, + { "", "There are %d leaves.\n", "Er is %d blad.\n", 1 }, + { "", "There is %d leaf.\n", "Er zijn %d bladeren.\n", 42 }, + { "", "There are %d leaves.\n", "Er zijn %d bladeren.\n", 42 }, + + // This isn't in the catalog, so should ruturn the key. + { "", "Come visit us on Discord!", "Come visit us on Discord!", 1 }, + }, + }, + + // QT Linguist with default loader options. + { + file = "assets/I18N/nl_NL-qt-ts.ts", + loader = i18n.parse_qt_linguist_file, + tests = { + // These are in the catalog. + { "Page", "Text for translation", "Tekst om te vertalen", 1}, + { "Page", "Also text to translate", "Ook tekst om te vertalen", 1}, + { "installscript", "99 bottles of beer on the wall", "99 flessen bier op de muur", 1}, + { "apple_count", "%d apple(s)", "%d appel", 1}, + { "apple_count", "%d apple(s)", "%d appels", 42}, + + // These aren't in the catalog, so should ruturn the key. + { "", "Come visit us on Discord!", "Come visit us on Discord!", 1 }, + { "Fake_Section", "Come visit us on Discord!", "Come visit us on Discord!", 1 }, + }, + }, + + // QT Linguist, merging sections. + { + file = "assets/I18N/nl_NL-qt-ts.ts", + loader = i18n.parse_qt_linguist_file, + options = {merge_sections = true}, + tests = { + // All of them are now in section "", lookup with original section should return the key. + { "", "Text for translation", "Tekst om te vertalen", 1}, + { "", "Also text to translate", "Ook tekst om te vertalen", 1}, + { "", "99 bottles of beer on the wall", "99 flessen bier op de muur", 1}, + { "", "%d apple(s)", "%d appel", 1}, + { "", "%d apple(s)", "%d appels", 42}, + + // All of them are now in section "", lookup with original section should return the key. + { "Page", "Text for translation", "Text for translation", 1}, + { "Page", "Also text to translate", "Also text to translate", 1}, + { "installscript", "99 bottles of beer on the wall", "99 bottles of beer on the wall", 1}, + { "apple_count", "%d apple(s)", "%d apple(s)", 1}, + { "apple_count", "%d apple(s)", "%d apple(s)", 42}, + }, + }, + + // QT Linguist, merging sections. Expecting .Duplicate_Key error because same key exists in more than 1 section. + { + file = "assets/I18N/duplicate-key.ts", + loader = i18n.parse_qt_linguist_file, + options = {merge_sections = true}, + err = .Duplicate_Key, + }, + + // QT Linguist, not merging sections. Shouldn't return error despite same key existing in more than 1 section. + { + file = "assets/I18N/duplicate-key.ts", + loader = i18n.parse_qt_linguist_file, + }, +} + +@test +tests :: proc(t: ^testing.T) { + using fmt + + cat: ^i18n.Translation + err: i18n.Error + + for suite in TESTS { + cat, err = suite.loader(suite.file, suite.options, nil, context.allocator) + + msg := fmt.tprintf("Expected loading %v to return %v, got %v", suite.file, suite.err, err) + expect(t, err == suite.err, msg) + + if err == .None { + for test in suite.tests { + val := T(test.section, test.key, test.n, cat) + + msg = fmt.tprintf("Expected key `%v` from section `%v`'s form for value `%v` to equal `%v`, got `%v`", test.key, test.section, test.n, test.val, val) + expect(t, val == test.val, msg) + } + } + i18n.destroy(cat) + } +} + +main :: proc() { + using fmt + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + t := testing.T{} + tests(&t) + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + if TEST_fail > 0 { + os.exit(1) + } + + if len(track.allocation_map) > 0 { + println() + for _, v in track.allocation_map { + printf("%v Leaked %v bytes.\n", v.location, v.size) + } + } +} \ No newline at end of file