From 11b08b4d861b8e6407730eafb5f245948c46c262 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 3 Mar 2022 15:38:59 +0000 Subject: [PATCH] Initial work on `core:text/template` --- core/text/template/execute.odin | 670 ++++++++++++++++++++++++++++ core/text/template/function.odin | 91 ++++ core/text/template/parse/ast.odin | 162 +++++++ core/text/template/parse/parse.odin | 528 ++++++++++++++++++++++ core/text/template/scan/scan.odin | 608 +++++++++++++++++++++++++ 5 files changed, 2059 insertions(+) create mode 100644 core/text/template/execute.odin create mode 100644 core/text/template/function.odin create mode 100644 core/text/template/parse/ast.odin create mode 100644 core/text/template/parse/parse.odin create mode 100644 core/text/template/scan/scan.odin diff --git a/core/text/template/execute.odin b/core/text/template/execute.odin new file mode 100644 index 000000000..c594daed8 --- /dev/null +++ b/core/text/template/execute.odin @@ -0,0 +1,670 @@ +package text_template + +import "core:io" +import "core:reflect" +import "core:strings" +import "core:mem" +import "core:mem/virtual" +import "core:fmt" + +import parse "parse" + +Template :: struct { + tree: ^parse.Tree, + + using config: Config, +} + +Config :: struct { + flags: Flags, + allocator: mem.Allocator, + left_delim: string, + right_delim: string, +} + +Flags :: distinct bit_set[Flag; u64] +Flag :: enum u64 { + Emit_Comments, +} + +Parse_Error :: parse.Error +Execution_Error :: enum { + Accumulated_Errors, + + Invalid_Node, + Invalid_Command, + Invalid_Value, + + Undeclared_Variable, + Undefined_Function, + + Invalid_Break, + Invalid_Continue, + + Invalid_Argument_Count, + Invalid_Argument_Type, + Out_Of_Bounds_Access, +} +Error :: union { + io.Error, + Execution_Error, +} + +create_from_string :: proc(input: string, config: Maybe(Config) = nil) -> (t: ^Template, err: Parse_Error) { + cfg, _ := config.? + if cfg.allocator.procedure == nil { + cfg.allocator = context.allocator + } + + t = new(Template, cfg.allocator) + t.config = cfg + t.tree = nil + t.tree, err = parse.parse(input, t.left_delim, t.right_delim, .Emit_Comments in t.flags, t.allocator) + return +} + +must :: proc(t: ^Template, err: Parse_Error) -> ^Template { + assert(t != nil) + assert(err == nil) + return t +} + +destroy :: proc(t: ^Template) { + allocator := t.allocator + parse.destroy_tree(t.tree) + free(t, allocator) +} + +execute :: proc(t: ^Template, w: io.Writer, data: any) -> Error { + s := State{ + tmpl = t, + w = w, + } + context.allocator = virtual.arena_allocator(&s.arena) + defer free_all(context.allocator) + + err := walk(&s, data,t.tree.root) + if err == nil && s.error_count > 0 { + err = .Accumulated_Errors + } + return err +} + + +State :: struct { + arena: virtual.Growing_Arena, + + tmpl: ^Template, + w: io.Writer, + at: ^parse.Node, + vars: [dynamic]Variable, + depth: int, + error_count: int, +} +Variable :: struct { + name: string, + value: any, +} + +walk :: proc(s: ^State, dot: any, node: ^parse.Node) -> Error { + s.at = node + + switch n in node.variant { + case ^parse.Node_Comment: + // ignore + case ^parse.Node_Text: + io.write_string(s.w, n.text) or_return + return nil + case ^parse.Node_List: + for elem in n.nodes { + walk(s, dot, elem) or_return + } + return nil + case ^parse.Node_Action: + val := eval_pipeline(s, dot, n.pipe) or_return + if len(n.pipe.decl) == 0 { + print_value(s, n, val) + } + return nil + case ^parse.Node_Pipeline: + print_value(s, n, eval_pipeline(s, dot, n) or_return) + return nil + + case ^parse.Node_Import: + panic("TODO Node_Import") + + case ^parse.Node_If: + return walk_if_or_with(s, .If, dot, n.pipe, n.list, n.else_list) + case ^parse.Node_With: + return walk_if_or_with(s, .With, dot, n.pipe, n.list, n.else_list) + case ^parse.Node_For: + return walk_for(s, dot, n) + + case ^parse.Node_Break: + return .Invalid_Break + case ^parse.Node_Continue: + return .Invalid_Continue + + case ^parse.Node_Else, + ^parse.Node_End, + ^parse.Node_Nil, + ^parse.Node_Bool, + ^parse.Node_Number, + ^parse.Node_String, + ^parse.Node_Variable, + ^parse.Node_Identifier, + ^parse.Node_Operator, + ^parse.Node_Dot, + ^parse.Node_Field: + return .Invalid_Node + + case ^parse.Node_Command: + return .Invalid_Node + case ^parse.Node_Chain: + return .Invalid_Node + } + + return .Invalid_Node +} + +mark_vars :: proc(s: ^State) -> int { + return len(s.vars) +} +pop_vars :: proc(s: ^State, n: int) { + resize(&s.vars, n) +} + +@(private, deferred_in_out=pop_vars) +SCOPE :: proc(s: ^State) -> int { + return mark_vars(s) +} +walk_if_or_with :: proc(s: ^State, kind: enum{If, With}, dot: any, + pipe: ^parse.Node_Pipeline, + list: ^parse.Node_List, + else_list: ^parse.Node_List) -> Error { + SCOPE(s) + + val := eval_pipeline(s, dot, pipe) or_return + truth, ok := is_true(val) + if !ok { + error(s, "if/with cannot use %v", val) + } + if truth { + if kind == .With { + return walk(s, val, list) + } else { + return walk(s, dot, list) + } + } else if else_list != nil { + return walk(s, dot, else_list) + } + return nil +} + +walk_for :: proc(s: ^State, dot: any, f: ^parse.Node_For) -> Error { + s.at = f + SCOPE(s) + + val, _ := indirect(eval_pipeline(s, dot, f.pipe) or_return) + + mark := mark_vars(s) + + the_body :: proc(s: ^State, f: ^parse.Node_For, elem, index: any, mark: int) -> Error { + if len(f.pipe.decl) > 0 { + set_top_var(s, 1, index) or_return + } + if len(f.pipe.decl) > 1 { + set_top_var(s, 2, elem) or_return + } + defer pop_vars(s, mark) + return walk(s, elem, f.list) + } + + original_id := val.id + ti := reflect.type_info_base(type_info_of(val.id)) + val.id = ti.id + #partial switch info in ti.variant { + case reflect.Type_Info_Array, + reflect.Type_Info_Slice, + reflect.Type_Info_Dynamic_Array: + n := reflect.length(val) + if n == 0 { + break + } + for i in 0.. (v: any, is_nil: bool) { + v = val + for v != nil { + ti := reflect.type_info_base(type_info_of(v.id)) + info, ok := ti.variant.(reflect.Type_Info_Pointer) + if !ok { + break + } + + ptr := (^rawptr)(v.data)^ + if ptr == nil { + return v, true + } + v = any{ptr, info.elem.id} + } + return v, false +} + + +error :: proc(s: ^State, format: string, args: ..any) { + s.error_count += 1 + + assert(s.at != nil) + + // NOTE(bill): the line and column are recalculated + // each time here because errors are usually an early + // out for this execution system + + pos := int(s.at.pos) + text := s.tmpl.tree.input[:pos] + col := strings.last_index(text, "\n") + if col < 0 { + col = pos + } else { + col += 1 + col = pos - col + } + + line := 1 + strings.count(text, "\n") + + name := s.tmpl.tree.name + if name == "" { + name = "" + } + fmt.eprintf("%s:%d:%d: ", name, line, col) + fmt.eprintf(format, ..args) + fmt.eprintln() +} + +is_true :: proc(val: any) -> (truth, ok: bool) { + check_trivial :: proc(v: any) -> (bool, bool) { + data := reflect.as_bytes(v) + for v in data { + if v != 0 { + return true, true + } + } + return false, true + } + + if val == nil { + return false, true + } + + ti := reflect.type_info_base(type_info_of(val.id)) + switch v in ti.variant { + case reflect.Type_Info_Named: + unreachable() + case reflect.Type_Info_Integer, + reflect.Type_Info_Rune, + reflect.Type_Info_Float, + reflect.Type_Info_Complex, + reflect.Type_Info_Quaternion, + reflect.Type_Info_Boolean, + reflect.Type_Info_Bit_Set, + reflect.Type_Info_Enum: + return check_trivial(val) + + case reflect.Type_Info_String: + if v.is_cstring { + cstr := (^cstring)(val.data)^ + if cstr == nil { + return false, true + } + return ([^]u8)(cstr)[0] != 0, true + } + str := (^string)(val.data)^ + return len(str) > 0, true + + case reflect.Type_Info_Any: + return false, false + case reflect.Type_Info_Type_Id: + return (^typeid)(val.data)^ != nil, true + case reflect.Type_Info_Pointer, + reflect.Type_Info_Multi_Pointer: + return (^rawptr)(val.data)^ != nil, true + + case reflect.Type_Info_Procedure: + return + + case reflect.Type_Info_Array: + return v.count > 0, true + case reflect.Type_Info_Enumerated_Array: + return v.count > 0, true + case reflect.Type_Info_Dynamic_Array: + a := (^mem.Raw_Dynamic_Array)(val.data) + return a.len > 0, true + case reflect.Type_Info_Slice: + a := (^mem.Raw_Slice)(val.data) + return a.len > 0, true + case reflect.Type_Info_Tuple: + return + case reflect.Type_Info_Struct: + // All structs are always non nil + return true, true + case reflect.Type_Info_Union: + return reflect.union_variant_typeid(val) != nil, true + case reflect.Type_Info_Map: + m := (^mem.Raw_Map)(val.data) + return m.entries.len > 0, true + case reflect.Type_Info_Simd_Vector: + return v.count > 0, true + case reflect.Type_Info_Relative_Pointer: + return check_trivial(val) + case reflect.Type_Info_Relative_Slice: + return check_trivial(val) + case reflect.Type_Info_Matrix: + return check_trivial(val) + } + return +} + + +eval_pipeline :: proc(s: ^State, dot: any, pipe: ^parse.Node_Pipeline) -> (value: any, err: Error) { + if pipe == nil { + return + } + s.at = pipe + value = nil + for cmd in pipe.cmds { + value = eval_command(s, dot, cmd, value) or_return + } + for var in pipe.decl { + if pipe.is_assign { + set_var(s, var.name, value) or_return + } else { + push_var(s, var.name, value) + } + } + return +} + +set_var :: proc(s: ^State, name: string, value: any) -> Error { + for i := mark_vars(s)-1; i >= 0; i -= 1 { + if s.vars[i].name == name { + s.vars[i].value = value + return nil + } + } + return .Undeclared_Variable +} + +set_top_var :: proc(s: ^State, n: int, value: any) -> Error { + if len(s.vars) > 0 { + s.vars[len(s.vars)-n].value = value + return nil + } + return .Undeclared_Variable +} + +push_var :: proc(s: ^State, name: string, value: any) { + append(&s.vars, Variable{name, value}) +} +get_var :: proc(s: ^State, name: string) -> (value: any, err: Error) { + for i := mark_vars(s)-1; i >= 0; i -= 1 { + if s.vars[i].name == name { + return s.vars[i].value, nil + } + } + error(s, "undeclared variable $%s", name) + return nil, .Undeclared_Variable + +} + +eval_command :: proc(s: ^State, dot: any, cmd: ^parse.Node_Command, final: any) -> (value: any, err: Error) { + first_word := cmd.args[0] + #partial switch n in first_word.variant { + case ^parse.Node_Field: + s.at = n + return eval_fields(s, dot, n.idents) + case ^parse.Node_Chain: + return eval_chain(s, dot, n) + case ^parse.Node_Identifier: + return eval_function(s, dot, n.ident, cmd, final) + case ^parse.Node_Operator: + return eval_function(s, dot, n.value, cmd, final) + case ^parse.Node_Pipeline: + return eval_pipeline(s, dot, n) + case ^parse.Node_Variable: + s.at = n + return get_var(s, n.name) + } + s.at = first_word + #partial switch n in first_word.variant { + case ^parse.Node_Bool: + return new_any(n.ok), nil + case ^parse.Node_Dot: + return dot, nil + case ^parse.Node_Nil: + return nil, nil + case ^parse.Node_Number: + if i, ok := n.i.?; ok { + return new_any(i), nil + } + if u, ok := n.u.?; ok { + return new_any(u), nil + } + if f, ok := n.f.?; ok { + return new_any(f), nil + } + return eval_function(s, dot, n.text, cmd, final) + case ^parse.Node_String: + return new_any(n.text), nil + } + + error(s, "cannot evaluate command %v", first_word.variant) + return nil, .Invalid_Command +} + +eval_chain :: proc(s: ^State, dot: any, chain: ^parse.Node_Chain) -> (value: any, err: Error) { + s.at = chain + return eval_fields(s, eval_arg(s, dot, chain.node) or_return, chain.fields[:]) +} + +eval_fields :: proc(s: ^State, dot: any, idents: []string) -> (value: any, err: Error) { + value = dot + for ident in idents { + value = eval_field(s, value, ident) or_return + } + return +} + +eval_field :: proc(s: ^State, dot: any, ident: string) -> (value: any, err: Error) { + if dot == nil { + return nil, nil + } + + ti := reflect.type_info_base(type_info_of(dot.id)) + #partial switch info in ti.variant { + case reflect.Type_Info_Struct: + value = reflect.struct_field_value_by_name(dot, ident, true) + if value != nil { + return + } + case reflect.Type_Info_Pointer: + if dot.data != nil { + deref := (^rawptr)(dot.data)^ + return eval_field(s, {deref, info.elem.id}, ident) + } + case reflect.Type_Info_Map: + key_type := reflect.type_info_base(info.key) + switch key_type.id { + case typeid_of(string), typeid_of(cstring): + gs := reflect.type_info_base(info.generated_struct).variant.(reflect.Type_Info_Struct) + ed := reflect.type_info_base(gs.types[1]).variant.(reflect.Type_Info_Dynamic_Array) + entry_type := ed.elem.variant.(reflect.Type_Info_Struct) + key_offset := entry_type.offsets[2] + value_offset := entry_type.offsets[3] + entry_size := uintptr(ed.elem_size) + + rm := (^mem.Raw_Map)(dot.data) + + data := uintptr(rm.entries.data) + for i in 0.. (value: any, err: Error) { + cmd_args := cmd.args[1:] + + switch name { + case "+", "-": + if len(cmd_args) < 1 { + error(s, "%q expects at least 1 argument", name) + return nil, .Invalid_Argument_Count + } + // TODO + + return nil, nil + case "*": + if len(cmd_args) < 1 { + error(s, "%q expects at least 2 arguments, got %d", name, len(cmd_args)) + return nil, .Invalid_Argument_Count + } + // TODO + return nil, nil + } + + function, ok := builtin_funcs[name] + if !ok { + error(s, "%q is not a defined function", name) + err = .Undefined_Function + return + } + + if function == nil { + switch name { + case "and": + // TODO + case "or": + // TODO + case: + panic("unhandled built-in procedure") + } + } + + n := len(cmd_args) + if final != nil { + n += 1 + } + args_to_call := make([dynamic]any, 0, n) + for arg in cmd_args { + append(&args_to_call, eval_arg(s, dot, arg) or_return) + } + if final != nil { + append(&args_to_call, final) + } + + return function(args_to_call[:]) +} + +eval_arg :: proc(s: ^State, dot: any, arg: ^parse.Node) -> (value: any, err: Error) { + s.at = arg + #partial switch n in arg.variant { + case ^parse.Node_Dot: + return dot, nil + case ^parse.Node_Nil: + return nil, nil + case ^parse.Node_Bool: + return new_any(n.ok), nil + case ^parse.Node_Number: + if i, ok := n.i.?; ok { + return new_any(i), nil + } + if u, ok := n.u.?; ok { + return new_any(u), nil + } + if f, ok := n.f.?; ok { + return new_any(f), nil + } + case ^parse.Node_String: + return new_any(n.text), nil + + case ^parse.Node_Field: + return eval_fields(s, dot, n.idents) + case ^parse.Node_Variable: + return get_var(s, n.name) + case ^parse.Node_Pipeline: + return eval_pipeline(s, dot, n) + case ^parse.Node_Chain: + return eval_chain(s, dot, n) + } + return nil, .Invalid_Node +} + + +print_value :: proc(s: ^State, n: ^parse.Node, val: any) { + s.at = n + if val == nil { + io.write_string(s.w, "nil") + } else { + fmt.wprint(s.w, val) + } +} \ No newline at end of file diff --git a/core/text/template/function.odin b/core/text/template/function.odin new file mode 100644 index 000000000..87958b072 --- /dev/null +++ b/core/text/template/function.odin @@ -0,0 +1,91 @@ +package text_template + +import "core:mem" +import "core:fmt" +import "core:reflect" + +Function :: #type proc(args: []any) -> (value: any, err: Error) + + +@(private) +new_any :: proc(x: $T) -> any { + ptr := new_clone(x) + return any{ptr, typeid_of(T)} +} + +builtin_funcs: map[string]Function + +@(private, init) +init_builtin_funcs :: proc() { + builtin_funcs["and"] = nil // requires shortcircuiting behaviour so implemented internally + builtin_funcs["or"] = nil // requires shortcircuiting behaviour so implemented internally + builtin_funcs["not"] = proc(args: []any) -> (value: any, err: Error) { + if len(args) != 1 { + err = .Invalid_Argument_Count + return + } + t, _ := is_true(args[0]) + return new_any(t), nil + } + + builtin_funcs["index"] = proc(args: []any) -> (value: any, err: Error) { + if len(args) < 2 { + err = .Invalid_Argument_Count + return + } + + arg := args[0] + for idx in args[1:] { + i, ok := reflect.as_int(idx) + if !ok { + err = .Invalid_Argument_Type + return + } + if reflect.length(arg) < i { + return nil, .Out_Of_Bounds_Access + } + arg = reflect.index(arg, i) + } + return arg, nil + } + + builtin_funcs["len"] = proc(args: []any) -> (value: any, err: Error) { + if len(args) != 1 { + err = .Invalid_Argument_Count + return + } + + n := reflect.length(args[0]) + return new_any(n), nil + } + + + builtin_funcs["print"] = proc(args: []any) -> (value: any, err: Error) { + return new_any(fmt.aprint(..args)), nil + } + builtin_funcs["println"] = proc(args: []any) -> (value: any, err: Error) { + return new_any(fmt.aprintln(..args)), nil + } + builtin_funcs["printf"] = proc(args: []any) -> (value: any, err: Error) { + if len(args) < 1 { + err = .Invalid_Argument_Count + return + } + format_any := args[0] + format_any.id = reflect.typeid_base(format_any.id) + format: string + switch v in format_any { + case string: + format = v + case cstring: + format = string(v) + case: + err = .Invalid_Argument_Type + return + } + + other_args := args[1:] + return new_any(fmt.aprintf(format, ..other_args)), nil + } +} + diff --git a/core/text/template/parse/ast.odin b/core/text/template/parse/ast.odin new file mode 100644 index 000000000..3314b4140 --- /dev/null +++ b/core/text/template/parse/ast.odin @@ -0,0 +1,162 @@ +package text_template_parse + +import "../scan" +import "core:strings" + +Pos :: scan.Pos +Token :: scan.Token +Token_Kind :: scan.Token_Kind + +new_node :: proc($T: typeid, pos: Pos = 0, allocator := context.allocator) -> ^T { + n := new(T, allocator) + n.pos = pos + n.variant = n + return n +} + +Node :: struct { + pos: Pos, + variant: union{ + ^Node_Text, + ^Node_Comment, + ^Node_Action, + ^Node_Pipeline, + ^Node_Chain, + ^Node_Command, + ^Node_Import, + ^Node_Dot, + ^Node_Field, + ^Node_Identifier, + ^Node_Operator, + ^Node_If, + ^Node_For, + ^Node_List, + ^Node_Nil, + ^Node_Bool, + ^Node_Number, + ^Node_String, + ^Node_Variable, + ^Node_With, + ^Node_Break, + ^Node_Continue, + + // Dummy nodes + ^Node_Else, + ^Node_End, + }, +} + +Node_Branch :: struct{ + using base: Node, + pipe: ^Node_Pipeline, + list: ^Node_List, + else_list: ^Node_List, +} + +Node_Text :: struct{ + using base: Node, + text: string, +} +Node_Action :: struct{ + using base: Node, + pipe: ^Node_Pipeline, +} +Node_Bool :: struct{ + using base: Node, + ok: bool, +} +Node_Chain :: struct{ + using base: Node, + node: ^Node, + fields: [dynamic]string, +} +Node_Command :: struct{ + using base: Node, + args: [dynamic]^Node, +} +Node_Dot :: struct{ + using base: Node, +} +Node_Field :: struct{ + using base: Node, + idents: []string, +} +Node_Identifier :: struct{ + using base: Node, + ident: string, +} +Node_Operator :: struct{ + using base: Node, + value: string, +} + + +Node_If :: distinct Node_Branch +Node_For :: distinct Node_Branch +Node_With :: distinct Node_Branch + +Node_List :: struct{ + using base: Node, + nodes: [dynamic]^Node, +} +Node_Nil :: struct{ + using base: Node, +} +Node_Number :: struct{ + using base: Node, + text: string, + i: Maybe(i64), + u: Maybe(u64), + f: Maybe(f64), +} +Node_Pipeline :: struct{ + using base: Node, + is_assign: bool, + decl: [dynamic]^Node_Variable, + cmds: [dynamic]^Node_Command, +} +Node_String :: struct{ + using base: Node, + quoted: string, + text: string, // after processing +} +Node_Import :: struct{ + using base: Node, + name: string, // unquoted + pipe: ^Node_Pipeline, +} +Node_Variable :: struct{ + using base: Node, + name: string, +} +Node_Comment :: struct{ + using base: Node, + text: string, +} +Node_Break :: struct{ + using base: Node, +} +Node_Continue :: struct{ + using base: Node, +} + +Node_Else :: struct { + using base: Node, +} + +Node_End :: struct { + using base: Node, +} + +chain_add :: proc(c: ^Node_Chain, field: string) { + field := field + if len(field) == 0 || field[0] != '.' { + panic("not a .field") + } + field = field[1:] + if field == "" { + panic("empty field") + } + append(&c.fields, field) +} + diff --git a/core/text/template/parse/parse.odin b/core/text/template/parse/parse.odin new file mode 100644 index 000000000..40ee12aff --- /dev/null +++ b/core/text/template/parse/parse.odin @@ -0,0 +1,528 @@ +package text_template_parse + +import "core:io" +import "core:fmt" +import "core:mem" +import "core:mem/virtual" +import "core:strconv" +import "../scan" + +Error :: enum { + None, + + Unexpected_Token, + Unexpected_EOF, + + Expected_End, + + Invalid_Node, + + Invalid_Character, + Invalid_Number, + Invalid_String, + + Empty_Command, + Missing_Value, + Non_Executable_Command, + Undefined_Variable, + Unexpected_Operand, + Invalid_For_Initialization, + Too_Many_Declarations, +} +Tree :: struct { + general_allocator: mem.Allocator, + + arena: virtual.Growing_Arena, + name: string, + + tokens: []Token, // general_allocator + + root: ^Node_List, + input: string, + offset: uint, + + for_loop_depth: uint, + + vars: [dynamic]string, +} + +@(require_results) +errorf :: proc(t: ^Tree, err: Error, format: string, args: ..any) -> Error { + if err != nil { + fmt.eprintf(format, ..args) + fmt.eprintln() + } + return err +} + +@(require_results) +unexpected_token :: proc(t: ^Tree, token: Token) -> Error { + return errorf(t, .Unexpected_Token, "unexpected token: %s", token.value) +} + + +peek :: proc(t: ^Tree, n: uint = 0) -> Token { + if t.offset+n < len(t.tokens) { + return t.tokens[t.offset+n] + } + return Token{.EOF, "", Pos(len(t.input)), 0} +} +next :: proc(t: ^Tree) -> (token: Token) { + if t.offset < len(t.tokens) { + token = t.tokens[t.offset] + t.offset += 1 + return + } + return Token{.EOF, "", Pos(len(t.input)), 0} +} +backup :: proc(t: ^Tree, n: uint = 1) { + if n > t.offset { + t.offset = 0 + } else { + t.offset -= n + } +} + +next_non_space :: proc(t: ^Tree) -> (token: Token) { + for { + token = next(t) + if token.kind != .Space { + break + } + } + return +} +peek_non_space :: proc(t: ^Tree, offset: uint = 0) -> (token: Token) { + i := offset + for { + if t.offset+i < len(t.tokens) { + token = t.tokens[t.offset+i] + } else { + token = Token{.EOF, "", Pos(len(t.input)), 0} + } + if token.kind != .Space { + break + } + i += 1 + } + return +} +peek_after_non_space :: proc(t: ^Tree) -> (token: Token) { + return peek_non_space(t, 1) +} + +expect :: proc(t: ^Tree, expected: Token_Kind, ctx: string) -> (token: Token, err: Error) { + token = next_non_space(t) + if token.kind != expected { + err = errorf(t, .Unexpected_Token, "unexpected token, expected %s, got %s", expected, token.value) + } + return +} + + +parse :: proc(input: string, left_delim, right_delim: string, emit_comments: bool = false, general_allocator := context.allocator) -> (t: ^Tree, err: Error) { + t = new(Tree, general_allocator) + t.general_allocator = general_allocator + t.vars.allocator = general_allocator + t.input = input + + + s := scan.init(&scan.Scanner{}, t.name, input, left_delim, right_delim, emit_comments) + s.tokens.allocator = t.general_allocator + scan.run(s) + t.tokens = s.tokens[:] // general_allocator + + context.allocator = virtual.arena_allocator(&t.arena) + + t.root = new_node(Node_List) + for peek(t).kind != .EOF { + if peek(t).kind == .Left_Delim && peek_after_non_space(t).kind == .Declare { + // TODO + continue + } + node := text_or_action(t) or_return + if node != nil { + append(&t.root.nodes, node) + } else { + break + } + } + + return +} + +destroy_tree :: proc(t: ^Tree) { + if t != nil { + virtual.arena_destroy(&t.arena) + + ga := t.general_allocator + delete(t.tokens, ga) + delete(t.vars) + free(t, ga) + } +} + + +text_or_action :: proc(t: ^Tree) -> (node: ^Node, err: Error) { + #partial switch token := next_non_space(t); token.kind { + case .Text: + n := new_node(Node_Text, token.pos) + n.text = token.value + return n, nil + case .Left_Delim: + return action(t) + case .Comment: + n := new_node(Node_Comment, token.pos) + n.text = token.value + return n, nil + case: + return nil, unexpected_token(t, token) + } + return nil, nil +} + +parse_list :: proc(t: ^Tree) -> (list: ^Node_List, next: ^Node, err: Error) { + list = new_node(Node_List, peek_non_space(t).pos) + for peek_non_space(t).kind != .EOF { + node := text_or_action(t) or_return + #partial switch n in node.variant { + case ^Node_Else: + next = n + return + case ^Node_End: + next = n + return + } + append(&list.nodes, node) + } + err = errorf(t, .Unexpected_EOF, "unexpected EOF") + return +} + +parse_control :: proc(t: ^Tree, allow_else_if: bool, ctx: string) -> (pipe: ^Node_Pipeline, list, else_list: ^Node_List, err: Error) { + pipe = pipeline(t, ctx, .Right_Delim) or_return + + if ctx == "for" { + t.for_loop_depth += 1 + } + + next_node: ^Node + list, next_node = parse_list(t) or_return + + if ctx == "for" { + t.for_loop_depth -= 1 + } + + #partial switch n in next_node.variant { + case ^Node_End: + // We are done + + case ^Node_Else: + if allow_else_if && peek(t).kind == .If { + // {{if a}}...{{else if b}}...{{end}} + // is translated into + // {{if a}}...{{else}}{{if b}}...{{end}}{{end}} + next(t) + else_list = new_node(Node_List, next_node.pos) + append(&else_list.nodes, parse_if(t) or_return) + break + } + else_list, next_node = parse_list(t) or_return + if _, ok := next_node.variant.(^Node_End); !ok { + errorf(t, .Expected_End, "expected end") or_return + } + } + return +} + +// {{if pipeline}} list {{end}} +// {{if pipeline}} list {{else}} list {{end}} +// {{if pipeline}} list {{else if pipeline}} list {{end}} +parse_if :: proc(t: ^Tree) -> (node: ^Node_If, err: Error) { + pipe, list, else_list := parse_control(t, true, "if") or_return + node = new_node(Node_If, pipe.pos) + node.pipe = pipe + node.list = list + node.else_list = else_list + return +} + +// {{for pipeline}} list {{end}} +// {{for pipeline}} list {{else}} list {{end}} +parse_for :: proc(t: ^Tree) -> (node: ^Node_For, err: Error) { + pipe, list, else_list := parse_control(t, false, "for") or_return + node = new_node(Node_For, pipe.pos) + node.pipe = pipe + node.list = list + node.else_list = else_list + return +} + +// {{with pipeline}} list {{end}} +// {{with pipeline}} list {{else}} list {{end}} +parse_with :: proc(t: ^Tree) -> (node: ^Node_With, err: Error) { + pipe, list, else_list := parse_control(t, false, "with") or_return + node = new_node(Node_With, pipe.pos) + node.pipe = pipe + node.list = list + node.else_list = else_list + return +} + + +// {{else}} +parse_else :: proc(t: ^Tree) -> (node: ^Node_Else, err: Error) { + p := peek_non_space(t) + if p.kind == .If { + node = new_node(Node_Else, p.pos) + return + } + token := expect(t, .Right_Delim, "else") or_return + node = new_node(Node_Else, token.pos) + return +} +// {{end}} +parse_end :: proc(t: ^Tree) -> (node: ^Node_End, err: Error) { + token := expect(t, .Right_Delim, "end") or_return + node = new_node(Node_End, token.pos) + return +} + + + + +action :: proc(t: ^Tree) -> (^Node, Error) { + // TODO actions + #partial switch token := next_non_space(t); token.kind { + case .If: return parse_if(t) + case .For: return parse_for(t) + case .With: return parse_with(t) + case .Else: return parse_else(t) + case .End: return parse_end(t) + + case .Block: + return nil, .Invalid_Node + case .Break: + return nil, .Invalid_Node + case .Continue: + return nil, .Invalid_Node + case .Include: + return nil, .Invalid_Node + } + backup(t) + + return pipeline(t, "command", .Right_Delim) +} + + +pipeline :: proc(t: ^Tree, ctx: string, end: Token_Kind) -> (pipe: ^Node_Pipeline, err: Error) { + pipe = new_node(Node_Pipeline, peek_non_space(t).pos) + + decls: for v := peek_non_space(t); v.kind == .Variable; /**/ { + next_non_space(t) + + token_after_variable := peek(t) // could be space + next := peek_non_space(t) + switch { + case next.kind == .Assign, next.kind == .Declare: + pipe.is_assign = next.kind == .Assign + next_non_space(t) + append(&t.vars, v.value) + append(&pipe.decl, parse_variable(t, v) or_return) + + case next.kind == .Char && next.value == ",": + next_non_space(t) + append(&t.vars, v.value) + append(&pipe.decl, parse_variable(t, v) or_return) + if ctx == "for" && len(pipe.decl) < 2 { + #partial switch peek_non_space(t).kind { + case .Variable, .Right_Delim, .Right_Paren: + v = peek_non_space(t) + continue decls + } + errorf(t, .Invalid_For_Initialization, "for can only initialize variables") or_return + } + errorf(t, .Too_Many_Declarations, "too many declarations in %s", ctx) or_return + + case token_after_variable.kind == .Space: + backup(t, 2) + case: + backup(t, 1) + } + + break decls + } + + for { + #partial switch tok := next_non_space(t); tok.kind { + case end: + if len(pipe.cmds) == 0 { + errorf(t, .Missing_Value, "missing value for %s", ctx) or_return + } + for c, i in pipe.cmds[1:] { + #partial switch n in c.variant { + case ^Node_Bool, ^Node_Dot, ^Node_Nil, ^Node_Number, ^Node_String: + errorf(t, .Non_Executable_Command, "non executable command in pipeline stage for %d", i+2) or_return + } + } + return + case .Bool, .Char, .Dot, .Field, .Identifier, .Operator, .Number, .Nil, .Raw_String, .String, .Variable, .Left_Paren: + backup(t) + append(&pipe.cmds, command(t) or_return) + case: + err = unexpected_token(t, tok) + return + } + } +} + +command :: proc(t: ^Tree) -> (cmd: ^Node_Command, err: Error) { + cmd = new_node(Node_Command, peek_non_space(t).pos) + loop: for { + op := operand(t) or_return + if op != nil { + append(&cmd.args, op) + } + #partial switch token := next(t); token.kind { + case .Space: + continue loop + case .Right_Delim, .Right_Paren: + backup(t) + case .Pipe: + break loop + case: + errorf(t, .Unexpected_Operand, "unexpected operand %s", token.value) or_return + } + break loop + } + if len(cmd.args) == 0 { + err = errorf(t, .Empty_Command, "empty command") + } + return +} + +operand :: proc(t: ^Tree) -> (node: ^Node, err: Error) { + node = term(t) or_return + if node == nil { + return + } + if p := peek(t); p.kind == .Field { + chain := new_node(Node_Chain, p.pos) + chain.node = node + for peek(t).kind == .Field { + chain_add(chain, next(t).value) + } + + #partial switch n in node.variant { + case ^Node_Field: + f := new_node(Node_Field, chain.pos) + resize(&chain.fields, len(chain.fields)+len(n.idents)) + copy(chain.fields[len(n.idents):], chain.fields[:]) + copy(chain.fields[:], n.idents) + f.idents = chain.fields[:] + node = f + case: + node = chain + } + + } + return +} + +// literal (number, string, nil, boolean) +// function (identifier) +// operator (function-like thing) +// . +// .field +// $ +// $ +// '(' pipeline ')' +term :: proc(t: ^Tree) -> (^Node, Error) { + #partial switch token := next_non_space(t); token.kind { + case .Identifier: + n := new_node(Node_Identifier, token.pos) + n.ident = token.value + return n, nil + case .Operator: + n := new_node(Node_Operator, token.pos) + n.value = token.value + return n, nil + case .Dot: return new_node(Node_Dot, token.pos), nil + case .Nil: return new_node(Node_Nil, token.pos), nil + case .Variable: + return parse_variable(t, token) + case .Field: + f := new_node(Node_Field, token.pos) + f.idents = make([]string, 1) + f.idents[0] = token.value[1:] + return f, nil + case .Bool: + b := new_node(Node_Bool, token.pos) + b.ok = token.value == "true" + return b, nil + case .Char, .Number: + return parse_number(t, token) + case .String, .Raw_String: + text, _, ok := strconv.unquote_string(token.value) + if !ok { + return nil, errorf(t, .Invalid_String, "invalid string literal: %s", token.value) + } + n := new_node(Node_String, token.pos) + n.quoted = token.value + n.text = text + return n, nil + case .Left_Paren: + return pipeline(t, "parenthesized pipeline", .Right_Paren) + } + backup(t) + return nil, nil +} + + +parse_number :: proc(t: ^Tree, token: Token) -> (^Node_Number, Error) { + text := token.value + n := new_node(Node_Number, token.pos) + n.text = text + if token.kind == .Char { + r, _, tail, ok := strconv.unquote_char(text[:], text[0]) + if !ok || tail != "" { + return nil, errorf(t, .Invalid_Character, "invalid character literal: %s", text) + } + n.i = i64(r) + n.u = u64(r) + n.f = f64(r) + return n, nil + } + + + if u, ok := strconv.parse_u64(text); ok { + n.u = u + } + if i, ok := strconv.parse_i64(text); ok { + n.i = i + if i == 0 { + n.u = 0 + } + } + if n.u == nil && n.i == nil { + if f, ok := strconv.parse_f64(text); ok { + n.f = f + } + } + if n.u == nil && n.i == nil && n.f == nil { + return nil, errorf(t, .Invalid_Number, "invalid number syntax: %q", text) + } + return n, nil + +} + +parse_variable :: proc(t: ^Tree, token: Token) -> (^Node_Variable, Error) { + v := new_node(Node_Variable, token.pos) + v.name = token.value + for var in t.vars { + if var == v.name { + return v, nil + } + } + return nil, errorf(t, .Undefined_Variable, "undefined variable %q", v.name) +} \ No newline at end of file diff --git a/core/text/template/scan/scan.odin b/core/text/template/scan/scan.odin new file mode 100644 index 000000000..375e5ef5e --- /dev/null +++ b/core/text/template/scan/scan.odin @@ -0,0 +1,608 @@ +package text_template_scan + +import "core:fmt" +import "core:unicode" +import "core:unicode/utf8" +import "core:strings" + +Pos :: distinct int + +Token_Kind :: enum { + Error, + EOF, + + Comment, + Space, + + Left_Delim, + Right_Delim, + + Identifier, + Field, + Left_Paren, + Right_Paren, + + Bool, + Char, + Number, + Pipe, + Raw_String, + String, + Text, + Variable, + Operator, + + Declare, // := + Assign, // - + + _Keyword, + Dot, + Block, + Break, + Continue, + Define, + Else, + End, + For, + If, + Include, + Nil, + With, +} + + +keywords := map[string]Token_Kind { + "." = .Dot, + "block" = .Block, + "break" = .Break, + "continue" = .Continue, + "define" = .Define, + "else" = .Else, + "end" = .End, + "for" = .For, + "if" = .If, + "include" = .Include, + "nil" = .Nil, + "with" = .With, +} + +Token :: struct { + kind: Token_Kind, + value: string, + pos: Pos, + line: int, +} + +token_to_string :: proc(using tok: Token, allocator := context.temp_allocator) -> string { + context.allocator = allocator + switch { + case kind == .EOF: + return fmt.tprint("EOF") + case kind == .Error: + return fmt.tprint(value) + case kind > ._Keyword: + return fmt.tprintf("<%s>", value) + case len(value) > 10: + return fmt.tprintf("%.10q...", value) + } + return fmt.tprintf("%q", value) +} + +Scanner :: struct { + name: string, + input: string, + left_delim: string, + right_delim: string, + pos: Pos, + start: Pos, + width: Pos, + tokens: [dynamic]Token, + paren_depth: int, + line: int, + start_line: int, + emit_comments: bool, +} + +next :: proc(s: ^Scanner) -> rune { + if int(s.pos) >= len(s.input) { + s.width = 0 + return utf8.RUNE_EOF + } + r, w := utf8.decode_rune_in_string(s.input[s.pos:]) + s.width = Pos(w) + s.pos += s.width + if r == '\n' { + s.line += 1 + } + return r +} + +backup :: proc(s: ^Scanner) { + s.pos -= s.width + if s.width == 1 && s.input[s.pos] == '\n' { + s.line -= 1 + } +} + +peek :: proc(s: ^Scanner) -> rune { + r := next(s) + backup(s) + return r +} + +emit :: proc(s: ^Scanner, kind: Token_Kind) { + append(&s.tokens, Token{ + kind = kind, + pos = s.start, + line = s.start_line, + value = s.input[s.start:s.pos], + }) + s.start = s.pos + s.start_line = s.line +} + +ignore :: proc(s: ^Scanner) { + s.line += strings.count(s.input[s.start:s.pos], "\n") + s.start = s.pos + s.start_line = s.line +} + +accept :: proc(s: ^Scanner, valid: string) -> bool { + if strings.contains_rune(valid, next(s)) >= 0 { + return true + } + backup(s) + return false +} +accept_run :: proc(s: ^Scanner, valid: string) { + for strings.contains_rune(valid, next(s)) >= 0 { + // Okay + } + backup(s) +} + + +// State procedures + +DEFAULT_LEFT_DELIM :: "{{" +DEFAULT_RIGHT_DELIM :: "}}" + +LEFT_COMMENT :: "/*" +RIGHT_COMMENT :: "*/" + + +init :: proc(s: ^Scanner, name, input: string, left: string = "", right: string = "", + emit_comments: bool = false) -> ^Scanner { + s.name = name + s.input = input + s.left_delim = left if left != "" else DEFAULT_LEFT_DELIM + s.right_delim = right if right != "" else DEFAULT_RIGHT_DELIM + s.emit_comments = emit_comments + reset(s) + return s +} + +destroy :: proc(s: ^Scanner) { + delete(s.tokens) + s.tokens = {} +} + + +reset :: proc(s: ^Scanner) { + clear(&s.tokens) + s.pos = 0 + s.start = 0 + s.width = 0 + s.paren_depth = 0 + s.line = 1 + s.start_line = 1 +} + + +// Finite State Machine Scanning + +Scan_State :: enum { + None, + + Comment, + Space, + + Identifier, + Field, + Left_Delim, + Right_Delim, + + Char, + Number, + Raw_String, + String, + Text, + Variable, + + Inside_Action, +} + +step :: proc(s: ^Scanner, state: Scan_State) -> Scan_State { + scan_error :: proc(s: ^Scanner, value: string) -> Scan_State { + append(&s.tokens, Token{ + kind = .Error, + pos = s.start, + line = s.start_line, + value = value, + }) + return nil + } + scan_variable_or_field :: proc(s: ^Scanner, kind: Token_Kind) -> Scan_State { + if at_terminator(s) { + if kind == .Variable { + emit(s, kind) + } else { + emit(s, .Dot) + } + return .Inside_Action + } + for { + r := next(s) + if !is_alpha_numeric(r) { + backup(s) + break + } + } + if !at_terminator(s) { + return scan_error(s, "bad character") + } + emit(s, kind) + return .Inside_Action + } + + switch state { + case .None: + return nil + + case .Comment: + s.pos += Pos(len(LEFT_COMMENT)) + i := strings.index(s.input[s.pos:], RIGHT_COMMENT) + if i < 0 { + return scan_error(s, "unclosed comment") + } + s.pos += Pos(i + len(RIGHT_COMMENT)) + delim, trim_space := at_right_delim(s) + if !delim { + return scan_error(s, "comment ends before closing delimiter") + } + if s.emit_comments { + emit(s, .Comment) + } + if trim_space { + s.pos += TRIM_MARKER_LEN + } + s.pos += Pos(len(s.right_delim)) + if trim_space { + s.pos += left_trim_length(s.input[s.pos:]) + } + ignore(s) + return .Text + + case .Space: + space_count: int + for { + r := peek(s) + if !is_space(r) { + break + } + next(s) + space_count += 1 + } + + if has_right_trim_marker(s.input[s.pos-1:]) && strings.has_prefix(s.input[s.pos-1+TRIM_MARKER_LEN:], s.right_delim) { + backup(s) + if space_count == 1 { + return .Right_Delim + } + } + emit(s, .Space) + return .Inside_Action + + case .Identifier: + identifier_loop: for { + r := next(s) + if is_alpha_numeric(r) { + // Okay + } else { + backup(s) + word := s.input[s.start:s.pos] + if !at_terminator(s) { + return scan_error(s, "bad character") + } + + if kw := keywords[word]; kw > ._Keyword { + emit(s, kw) + } else if word == "true" || word == "false" { + emit(s, .Bool) + } else { + emit(s, .Identifier) + } + break identifier_loop + } + } + return .Inside_Action + case .Field: + return scan_variable_or_field(s, .Field) + + case .Left_Delim: + s.pos += Pos(len(s.left_delim)) + trim_space := has_left_trim_marker(s.input[s.pos:]) + after_marker := TRIM_MARKER_LEN if trim_space else 0 + if strings.has_prefix(s.input[s.pos+after_marker:], LEFT_COMMENT) { + s.pos += after_marker + ignore(s) + return .Comment + } + emit(s, .Left_Delim) + s.pos += after_marker + ignore(s) + s.paren_depth = 0 + return .Inside_Action + + case .Right_Delim: + trim_space := has_right_trim_marker(s.input[s.pos:]) + if trim_space { + s.pos += TRIM_MARKER_LEN + ignore(s) + } + s.pos += Pos(len(s.right_delim)) + emit(s, .Right_Delim) + if trim_space { + s.pos += left_trim_length(s.input[s.pos:]) + ignore(s) + } + return .Text + + case .Char: + char_loop: for { + switch next(s) { + case '\\': + if r := next(s); r != utf8.RUNE_EOF && r != '\n' { + break + } + fallthrough + case utf8.RUNE_EOF, '\n': + return scan_error(s, "unterminated character constant") + case '\'': + break char_loop + } + } + emit(s, .Char) + return .Inside_Action + + case .Number: + accept(s, "+-") + digits := "0123456789_" + if accept(s, "0") { + switch { + case accept(s, "bB"): + digits = "01_" + case accept(s, "oO"): + digits = "01234567_" + case accept(s, "xX"): + digits = "0123456789ABCDEFabcdef_" + } + } + accept_run(s, digits) + if accept(s, ".") { + accept_run(s, digits) + } + if len(digits) == 10+1 && accept(s, "eE") { + accept(s, "+-") + accept_run(s, digits) + } + if is_alpha_numeric(peek(s)) { + next(s) + return scan_error(s, "bad number syntax") + } + emit(s, .Number) + return .Inside_Action + + case .Raw_String: + raw_string_loop: for { + switch next(s) { + case utf8.RUNE_EOF: + return scan_error(s, "unterminated raw quoted string") + case '`': + break raw_string_loop + } + } + emit(s, .Raw_String) + return .Inside_Action + + case .String: + string_loop: for { + switch next(s) { + case '\\': + if r := next(s); r != utf8.RUNE_EOF && r != '\n' { + break + } + fallthrough + case utf8.RUNE_EOF, '\n': + return scan_error(s, "unterminated quoted string") + case '"': + break string_loop + } + } + emit(s, .String) + return .Inside_Action + + case .Text: + s.width = 0 + if x := strings.index(s.input[s.pos:], s.left_delim); x >= 0 { + ldn := Pos(len(s.left_delim)) + s.pos += Pos(x) + trim_length := Pos(0) + if has_left_trim_marker(s.input[s.pos+ldn:]) { + trim_length = right_trim_length(s.input[s.start:s.pos]) + } + s.pos -= trim_length + if s.pos > s.start { + s.line += strings.count(s.input[s.start:s.pos], "\n") + emit(s, .Text) + } + s.pos += trim_length + ignore(s) + return .Left_Delim + } + s.pos = Pos(len(s.input)) + // EOF + if s.pos > s.start { + s.line += strings.count(s.input[s.start:s.pos], "\n") + emit(s, .Text) + } + emit(s, .EOF) + + case .Variable: + if at_terminator(s) { + emit(s, .Variable) + return .Inside_Action + } + return scan_variable_or_field(s, .Variable) + + case .Inside_Action: + if delim, _ := at_right_delim(s); delim { + if s.paren_depth == 0 { + return .Right_Delim + } + return scan_error(s, "unclosed left paren") + } + + rp := peek(s) + switch r := next(s); { + case r == utf8.RUNE_EOF: + return scan_error(s, "unclosed action") + case is_space(r): + backup(s) // Just in case of " -}}" + return .Space + case r == '.': + // Look for a '.field' + if s.pos < Pos(len(s.input)) { + if r := s.input[s.pos]; r < '0' || r > '9' { + return .Field + } + } + // it's a number + fallthrough + + case (r == '+' || r == '-') && ('0' <= rp && rp <= '9'): + fallthrough + case '0' <= r && r <= '9': + backup(s) + return .Number + case r == '+', r == '-', r == '*': + emit(s, .Operator) + case is_alpha_numeric(r): + backup(s) + return .Identifier + case r == '|': + emit(s, .Pipe) + case r == '"': + return .String + case r == '`': + return .Raw_String + case r == '\'': + return .Char + case r == '(': + emit(s, .Left_Paren) + s.paren_depth += 1 + case r == ')': + emit(s, .Right_Paren) + s.paren_depth -= 1 + if s.paren_depth < 0 { + return scan_error(s, "unexpected right parenthesis ')'") + } + case r == '$': + return .Variable + case r == ':': + if next(s) != '=' { + return scan_error(s, "expected :=") + } + emit(s, .Declare) + case r == '=': + emit(s, .Assign) + case r <= unicode.MAX_ASCII && unicode.is_print(r): + emit(s, .Char) + case: + return scan_error(s, "unrecognized character in action") + } + + return .Inside_Action + } + return nil +} + + + +run :: proc(s: ^Scanner) { + state := Scan_State.Text + for state != nil { + state = step(s, state) + } +} + +@private TRIM_MARKER :: '-' +@private TRIM_MARKER_LEN :: Pos(2) // includes space + +is_space :: proc(r: rune) -> bool { + switch r { + case ' ', '\t', '\r', '\n': + return true + } + return false +} + +is_alpha_numeric :: proc(r: rune) -> bool { + return r == '_' || unicode.is_letter(r) || unicode.is_digit(r) +} + +left_trim_length :: proc(s: string) -> Pos { + return Pos(len(s) - len(strings.trim_left_proc(s, is_space))) +} +right_trim_length :: proc(s: string) -> Pos { + return Pos(len(s) - len(strings.trim_right_proc(s, is_space))) +} + +has_left_trim_marker :: proc(s: string) -> bool { + return len(s) >= 2 && s[0] == TRIM_MARKER && is_space(rune(s[1])) +} + +has_right_trim_marker :: proc(s: string) -> bool { + return len(s) >= 2 && is_space(rune(s[0])) && s[1] == TRIM_MARKER +} + +at_right_delim :: proc(s: ^Scanner) -> (delim, trim_spaces: bool) { + if has_right_trim_marker(s.input[s.pos:]) && strings.has_prefix(s.input[s.pos+TRIM_MARKER_LEN:], s.right_delim) { + delim = true + trim_spaces = true + return + } + if strings.has_prefix(s.input[s.pos:], s.right_delim) { + delim = true + trim_spaces = false + return + } + delim = false + trim_spaces = false + return +} + +at_terminator :: proc(s: ^Scanner) -> bool { + r := peek(s) + if is_space(r) { + return true + } + + switch r { + case utf8.RUNE_EOF, '.', ',', '(', ')', '|', ':': + return true + } + + rd, _ := utf8.decode_rune_in_string(s.right_delim) + return rd == r +} \ No newline at end of file