Move html escaping calls and fix existing generator

core/encoding/entity/entity.odin

@@ -21,6 +21,7 @@ package encoding_unicode_entity
 		Jeroen van Rijn: Initial implementation.
 */
 
+import "base:runtime"
 import "core:unicode/utf8"
 import "core:unicode"
 import "core:strings"
@@ -141,8 +142,10 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
 					write_string(&builder, entity)
 				} else {
 					if .No_Entity_Decode not_in options {
-						if decoded, ok := xml_decode_entity(entity); ok {
-							write_rune(&builder, decoded)
+						if decoded, count, ok := xml_decode_entity(entity); ok {
+							for i in 0..<count {
+								write_rune(&builder, decoded[i])
+							}
 							continue
 						}
 					}
@@ -212,17 +215,16 @@ advance :: proc(t: ^Tokenizer) -> (err: Error) {
 	}
 }
 
-xml_decode_entity :: proc(entity: string) -> (decoded: rune, ok: bool) {
+xml_decode_entity :: proc(entity: string) -> (decoded: [2]rune, rune_count: int, ok: bool) {
 	entity := entity
-	if len(entity) == 0 { return -1, false }
+	if len(entity) == 0 { return }
 
-	switch entity[0] {
-	case '#':
+	if entity[0] == '#' {
 		base  := 10
 		val   := 0
 		entity = entity[1:]
 
-		if len(entity) == 0 { return -1, false }
+		if len(entity) == 0 { return }
 
 		if entity[0] == 'x' || entity[0] == 'X' {
 			base = 16
@@ -237,30 +239,275 @@ xml_decode_entity :: proc(entity: string) -> (decoded: rune, ok: bool) {
 				val += int(r - '0')
 
 			case 'a'..='f':
-				if base == 10 { return -1, false }
+				if base == 10 { return }
 				val *= base
 				val += int(r - 'a' + 10)
 
 			case 'A'..='F':
-				if base == 10 { return -1, false }
+				if base == 10 { return }
 				val *= base
 				val += int(r - 'A' + 10)
 
 			case:
-				return -1, false
+				return
 			}
 
-			if val > MAX_RUNE_CODEPOINT { return -1, false }
+			if val > MAX_RUNE_CODEPOINT { return  }
 			entity = entity[1:]
 		}
-		return rune(val), true
-
-	case:
-		// Named entity.
-		return named_xml_entity_to_rune(entity)
+		return rune(val), 1, true
 	}
+	// Named entity.
+	return named_xml_entity_to_rune(entity)
 }
 
+
+// escape_html escapes special characters like '&' to become '&amp;'.
+// It escapes only 5 different characters: & ' < > and ".
+@(require_results)
+escape_html :: proc(s: string, allocator := context.allocator, loc := #caller_location) -> (output: string, was_allocation: bool) {
+	/*
+		& -> &amp;
+		' -> &#39; // &#39; is shorter than &apos; (NOTE: &apos; was not available until HTML 5)
+		< -> &lt;
+		> -> &gt;
+		" -> &#34; // &#34; is shorter than &quot;
+	*/
+
+	b := transmute([]byte)s
+
+	extra_bytes_needed := 0
+
+	for c in b {
+		switch c {
+		case '&':  extra_bytes_needed += 4
+		case '\'': extra_bytes_needed += 4
+		case '<':  extra_bytes_needed += 3
+		case '>':  extra_bytes_needed += 3
+		case '"':  extra_bytes_needed += 4
+		}
+	}
+
+	if extra_bytes_needed == 0 {
+		return s, false
+	}
+
+	t, err := make([]byte, len(s) + extra_bytes_needed, allocator, loc)
+	if err != nil {
+		return
+	}
+	was_allocation = true
+
+	w := 0
+	for c in b {
+		s := ""
+		switch c {
+		case '&':  s = "&amp;"
+		case '\'': s = "&#39;"
+		case '<':  s = "&lt;"
+		case '>':  s = "&gt;"
+		case '"':  s = "&#34;"
+		}
+		if s != "" {
+			copy(t[w:], s)
+			w += len(s)
+		} else {
+			t[w] = c
+			w += 1
+		}
+	}
+	output = string(t[0:w])
+	return
+}
+
+
+@(require_results)
+unescape_html :: proc(s: string, allocator := context.allocator, loc := #caller_location) -> (output: string, was_allocation: bool, err: runtime.Allocator_Error) {
+	@(require_results)
+	do_append :: proc(s: string, amp_idx: int, buf: ^[dynamic]byte) -> (n: int) {
+		s, amp_idx := s, amp_idx
+
+		n += len(s[:amp_idx])
+		if buf != nil { append(buf, s[:amp_idx]) }
+		s = s[amp_idx:]
+		for len(s) > 0 {
+			b, w, j := unescape_entity(s)
+			n += w
+			if buf != nil { append(buf, ..b[:w]) }
+
+			s = s[j:]
+
+			amp_idx = strings.index_byte(s, '&')
+			if amp_idx < 0 {
+				n += len(s)
+				if buf != nil { append(buf, s) }
+				break
+			}
+			n += amp_idx
+			if buf != nil { append(buf, s[:amp_idx]) }
+			s = s[amp_idx:]
+		}
+
+		return
+	}
+
+	s := s
+	amp_idx := strings.index_byte(s, '&')
+	if amp_idx < 0 {
+		return s, false, nil
+	}
+
+	// NOTE(bill): this does a two pass in order to minimize the allocations required
+	bytes_required := do_append(s, amp_idx, nil)
+
+	buf := make([dynamic]byte, 0, bytes_required, allocator, loc) or_return
+	was_allocation = true
+
+	_ = do_append(s, amp_idx, &buf)
+
+	assert(len(buf) == cap(buf))
+	output = string(buf[:])
+
+	return
+}
+
+// Returns an unescaped string of an encoded XML/HTML entity.
+@(require_results)
+unescape_entity :: proc(s: string) -> (b: [8]byte, w: int, j: int) {
+	s := s
+	if len(s) < 2 {
+		return
+	}
+	if s[0] != '&' {
+		return
+	}
+	j = 1
+
+	if s[j] == '#' { // scan numbers
+		j += 1
+		if len(s) <= 3 { // remove `&#.`
+			return
+		}
+		c := s[j]
+		hex := false
+		if c == 'x' || c == 'X' {
+			hex = true
+			j += 1
+		}
+
+		x := rune(0)
+		scan_number: for j < len(s) {
+			c = s[j]
+			j += 1
+			if hex {
+				switch c {
+				case '0'..='9': x = 16*x + rune(c) - '0';      continue scan_number
+				case 'a'..='f': x = 16*x + rune(c) - 'a' + 10; continue scan_number
+				case 'A'..='F': x = 16*x + rune(c) - 'A' + 10; continue scan_number
+				}
+			} else {
+				switch c {
+				case '0'..='9': x = 10*x + rune(c) - '0'; continue scan_number
+				}
+			}
+
+			// Keep the ';' to check for cases which require it and cases which might not
+			if c != ';' {
+				j -= 1
+			}
+			break scan_number
+		}
+
+
+		if j <= 3 { // no replacement characters found
+			return
+		}
+
+		@(static, rodata)
+		windows_1252_replacement_table := [0xa0 - 0x80]rune{ // Windows-1252 -> UTF-8
+			'\u20ac', '\u0081', '\u201a', '\u0192',
+			'\u201e', '\u2026', '\u2020', '\u2021',
+			'\u02c6', '\u2030', '\u0160', '\u2039',
+			'\u0152', '\u008d', '\u017d', '\u008f',
+			'\u0090', '\u2018', '\u2019', '\u201c',
+			'\u201d', '\u2022', '\u2013', '\u2014',
+			'\u02dc', '\u2122', '\u0161', '\u203a',
+			'\u0153', '\u009d', '\u017e', '\u0178',
+		}
+
+		switch x {
+		case 0x80..<0xa0:
+			x = windows_1252_replacement_table[x-0x80]
+		case 0, 0xd800..=0xdfff:
+			x = utf8.RUNE_ERROR
+		case:
+			if x > 0x10ffff {
+				x = utf8.RUNE_ERROR
+			}
+
+		}
+
+		b1, w1 := utf8.encode_rune(x)
+		w += copy(b[:], b1[:w1])
+		return
+	}
+
+	// Lookup by entity names
+
+	scan_ident: for j < len(s) { // scan over letters and digits
+		c := s[j]
+		j += 1
+
+		switch c {
+		case 'a'..='z', 'A'..='Z', '0'..='9':
+			continue scan_ident
+		}
+		// Keep the ';' to check for cases which require it and cases which might not
+		if c != ';' {
+			j -= 1
+		}
+		break scan_ident
+	}
+
+	entity_name := s[1:j]
+	if len(entity_name) == 0 {
+		return
+	}
+
+	if entity_name[len(entity_name)-1] == ';' {
+		entity_name = entity_name[:len(entity_name)-1]
+	}
+
+	if r2, _, ok := named_xml_entity_to_rune(entity_name); ok {
+		b1, w1 := utf8.encode_rune(r2[0])
+		w += copy(b[w:], b1[:w1])
+		if r2[1] != 0 {
+			b2, w2 := utf8.encode_rune(r2[1])
+			w += copy(b[w:], b2[:w2])
+		}
+		return
+	}
+
+	// The longest entities that do not end with a semicolon are <=6 bytes long
+	LONGEST_ENTITY_WITHOUT_SEMICOLON :: 6
+
+	n := min(len(entity_name)-1, LONGEST_ENTITY_WITHOUT_SEMICOLON)
+	for i := n; i > 1; i -= 1 {
+		if r2, _, ok := named_xml_entity_to_rune(entity_name[:i]); ok {
+			b1, w1 := utf8.encode_rune(r2[0])
+			w += copy(b[w:], b1[:w1])
+			if r2[1] != 0 {
+				b2, w2 := utf8.encode_rune(r2[1])
+				w += copy(b[w:], b2[:w2])
+			}
+			return
+		}
+	}
+
+	return
+}
+
+
 // Private XML helper to extract `&<stuff>;` entity.
 @(private="file")
 _extract_xml_entity :: proc(t: ^Tokenizer) -> (entity: string, err: Error) {

core/html/escape.odin

@@ -1,248 +0,0 @@
-package html
-
-import "base:runtime"
-import "core:fmt"
-import "core:strings"
-import "core:unicode/utf8"
-
-// escape_string escapes special characters like '&' to become '&'.
-// It escapes only 5 different characters: & ' < > and ".
-@(require_results)
-escape_string :: proc(s: string, allocator := context.allocator, loc := #caller_location) -> (output: string, was_allocation: bool) {
-	/*
-		& -> &amp;
-		' -> &#39; // &#39; is shorter than &apos; (NOTE: &apos; was not available until HTML 5)
-		< -> &lt;
-		> -> &gt;
-		" -> &#34; // &#34; is shorter than &quot;
-	*/
-
-	b := transmute([]byte)s
-
-	extra_bytes_needed := 0
-
-	for c in b {
-		switch c {
-		case '&':  extra_bytes_needed += 4
-		case '\'': extra_bytes_needed += 4
-		case '<':  extra_bytes_needed += 3
-		case '>':  extra_bytes_needed += 3
-		case '"':  extra_bytes_needed += 4
-		}
-	}
-
-	if extra_bytes_needed == 0 {
-		return s, false
-	}
-
-	t, err := make([]byte, len(s) + extra_bytes_needed, allocator, loc)
-	if err != nil {
-		return
-	}
-	was_allocation = true
-
-	w := 0
-	for c in b {
-		s := ""
-		switch c {
-		case '&':  s = "&amp;"
-		case '\'': s = "&#39;"
-		case '<':  s = "&lt;"
-		case '>':  s = "&gt;"
-		case '"':  s = "&#34;"
-		}
-		if s != "" {
-			copy(t[w:], s)
-			w += len(s)
-		} else {
-			t[w] = c
-			w += 1
-		}
-	}
-	output = string(t[0:w])
-	return
-}
-
-@(require_results)
-unescape_string :: proc(s: string, entity_map: Entity_Map, allocator := context.allocator, loc := #caller_location) -> (output: string, was_allocation: bool, err: runtime.Allocator_Error) {
-	@(require_results)
-	do_append :: proc(s: string, amp_idx: int, entity_map: Entity_Map, buf: ^[dynamic]byte) -> (n: int) {
-		s, amp_idx := s, amp_idx
-
-		n += len(s[:amp_idx])
-		if buf != nil { append(buf, s[:amp_idx]) }
-		s = s[amp_idx:]
-		for len(s) > 0 {
-			b, w, j := unescape_entity(s, entity_map)
-			n += w
-			if buf != nil { append(buf, ..b[:w]) }
-
-			s = s[j:]
-
-			amp_idx = strings.index_byte(s, '&')
-			if amp_idx < 0 {
-				n += len(s)
-				if buf != nil { append(buf, s) }
-				break
-			}
-			n += amp_idx
-			if buf != nil { append(buf, s[:amp_idx]) }
-			s = s[amp_idx:]
-		}
-
-		return
-	}
-
-	s := s
-	amp_idx := strings.index_byte(s, '&')
-	if amp_idx < 0 {
-		return s, false, nil
-	}
-
-	// NOTE(bill): this does a two pass in order to minimize the allocations required
-	bytes_required := do_append(s, amp_idx, entity_map, nil)
-
-	buf := make([dynamic]byte, 0, bytes_required, allocator, loc) or_return
-	was_allocation = true
-
-	_ = do_append(s, amp_idx, entity_map, &buf)
-
-	assert(len(buf) == cap(buf))
-	output = string(buf[:])
-
-	return
-}
-
-// Returns an unescaped string of an encoded HTML entity.
-@(require_results)
-unescape_entity :: proc(s: string, entity_map: Entity_Map) -> (b: [8]byte, w: int, j: int) {
-	s := s
-	if len(s) < 2 {
-		return
-	}
-	if s[0] != '&' {
-		return
-	}
-	j = 1
-
-	if s[j] == '#' { // scan numbers
-		j += 1
-		if len(s) <= 3 { // remove `&#.`
-			return
-		}
-		c := s[j]
-		hex := false
-		if c == 'x' || c == 'X' {
-			hex = true
-			j += 1
-		}
-
-		x := rune(0)
-		scan_number: for j < len(s) {
-			c = s[j]
-			j += 1
-			if hex {
-				switch c {
-				case '0'..='9': x = 16*x + rune(c) - '0';      continue scan_number
-				case 'a'..='f': x = 16*x + rune(c) - 'a' + 10; continue scan_number
-				case 'A'..='F': x = 16*x + rune(c) - 'A' + 10; continue scan_number
-				}
-			} else {
-				switch c {
-				case '0'..='9': x = 10*x + rune(c) - '0'; continue scan_number
-				}
-			}
-
-			// Keep the ';' to check for cases which require it and cases which might not
-			if c != ';' {
-				j -= 1
-			}
-			break scan_number
-		}
-
-
-		if j <= 3 { // no replacement characters found
-			return
-		}
-
-		@(static, rodata)
-		windows_1252_replacement_table := [0xa0 - 0x80]rune{ // Windows-1252 -> UTF-8
-			'\u20ac', '\u0081', '\u201a', '\u0192',
-			'\u201e', '\u2026', '\u2020', '\u2021',
-			'\u02c6', '\u2030', '\u0160', '\u2039',
-			'\u0152', '\u008d', '\u017d', '\u008f',
-			'\u0090', '\u2018', '\u2019', '\u201c',
-			'\u201d', '\u2022', '\u2013', '\u2014',
-			'\u02dc', '\u2122', '\u0161', '\u203a',
-			'\u0153', '\u009d', '\u017e', '\u0178',
-		}
-
-		switch x {
-		case 0x80..<0xa0:
-			x = windows_1252_replacement_table[x-0x80]
-		case 0, 0xd800..=0xdfff:
-			x = utf8.RUNE_ERROR
-		case:
-			if x > 0x10ffff {
-				x = utf8.RUNE_ERROR
-			}
-
-		}
-
-		b1, w1 := utf8.encode_rune(x)
-		w += copy(b[:], b1[:w1])
-		return
-	}
-
-	// Lookup by entity names
-
-	scan_ident: for j < len(s) { // scan over letters and digits
-		c := s[j]
-		j += 1
-
-		switch c {
-		case 'a'..='z', 'A'..='Z', '0'..='9':
-			continue scan_ident
-		}
-		// Keep the ';' to check for cases which require it and cases which might not
-		if c != ';' {
-			j -= 1
-		}
-		break scan_ident
-	}
-
-	entity_name := s[1:j]
-	if len(entity_name) == 0 {
-		return
-	}
-
-	if r, ok := entity_map.entity1[entity_name]; ok {
-		b1, w1 := utf8.encode_rune(r)
-		copy(b[:], b1[:w1])
-		w = w1
-		return
-	}
-
-	if r2, ok := entity_map.entity2[entity_name]; ok {
-		b1, w1 := utf8.encode_rune(r2[0])
-		b2, w2 := utf8.encode_rune(r2[1])
-		w += copy(b[w:], b1[:w1])
-		w += copy(b[w:], b2[:w2])
-		return
-	}
-
-	// The longest entities that do not end with a semicolon are <=6 bytes long
-	LONGEST_ENTITY_WITHOUT_SEMICOLON :: 6
-
-	n := min(len(entity_name)-1, LONGEST_ENTITY_WITHOUT_SEMICOLON)
-	for i := n; i > 1; i -= 1 {
-		if r, ok := entity_map.entity1[entity_name[:i]]; ok {
-			b1, w1 := utf8.encode_rune(r)
-			copy(b[:], b1[:w1])
-			w = w1
-			return
-		}
-	}
-
-	return
-}

core/unicode/tools/generate_entity_table.odin

@@ -18,7 +18,7 @@ OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, }, expected_doctype = "un
 
 Entity :: struct {
 	name:        string,
-	codepoint:   rune,
+	codepoints:  [2]rune,
 	description: string,
 }
 
@@ -57,7 +57,10 @@ generate_encoding_entity_table :: proc() {
 	fmt.printf("Found `<charlist>` with %v children.\n", len(charlist.value))
 
 	entity_map: map[string]Entity
+	defer delete(entity_map)
+
 	names: [dynamic]string
+	defer delete(names)
 
 	min_name_length := max(int)
 	max_name_length := min(int)
@@ -78,7 +81,13 @@ generate_encoding_entity_table :: proc() {
 			fmt.eprintln("`<character id=\"...\">` attribute not found.")
 			os.exit(1)
 		} else {
-			codepoint := strconv.atoi(codepoint_string)
+			r1, _, r2 := strings.partition(codepoint_string, "-")
+
+			codepoint, codepoint2: int
+			codepoint, _ = strconv.parse_int(r1)
+			if r2 != "" {
+				codepoint2, _ = strconv.parse_int(r1)
+			}
 
 			desc, desc_ok := xml.find_child_by_ident(doc, id, "description")
 			description := ""
@@ -115,7 +124,7 @@ generate_encoding_entity_table :: proc() {
 
 				e := Entity{
 					name        = name,
-					codepoint   = rune(codepoint),
+					codepoints  = {rune(codepoint), rune(codepoint2)},
 					description = description,
 				}
 
@@ -162,22 +171,22 @@ generate_encoding_entity_table :: proc() {
 		entity_name - a string, like "copy" that describes a user-encoded Unicode entity as used in XML.
 
 	Returns:
-		"decoded" - The decoded rune if found by name, or -1 otherwise.
-		"ok"      - true if found, false if not.
+		"decoded"    - The decoded runes if found by name, or all zero otherwise.
+		"rune_count" - The number of decoded runes
+		"ok"         - true if found, false if not.
 
 	IMPORTANT: XML processors (including browsers) treat these names as case-sensitive. So do we.
 */
-named_xml_entity_to_rune :: proc(name: string) -> (decoded: rune, ok: bool) {
+named_xml_entity_to_rune :: proc(name: string) -> (decoded: [2]rune, rune_count: int, ok: bool) {
 	/*
 		Early out if the name is too short or too long.
 		min as a precaution in case the generated table has a bogus value.
 	*/
 	if len(name) < min(1, XML_NAME_TO_RUNE_MIN_LENGTH) || len(name) > XML_NAME_TO_RUNE_MAX_LENGTH {
-		return -1, false
+		return
 	}
 
-	switch rune(name[0]) {
-`)
+	switch rune(name[0]) {`)
 
 	prefix := '?'
 	should_close := false
@@ -200,13 +209,17 @@ named_xml_entity_to_rune :: proc(name: string) -> (decoded: rune, ok: bool) {
 			fmt.wprintf(w, " ")
 		}
 		fmt.wprintf(w, " // %v\n", e.description)
-		fmt.wprintf(w, "\t\t\treturn %v, true\n", rune_to_string(e.codepoint))
+		if e.codepoints[1] != 0 {
+			fmt.wprintf(w, "\t\t\treturn {{%q, %q}}, 2, true\n", e.codepoints[0], e.codepoints[1])
+		} else {
+			fmt.wprintf(w, "\t\t\treturn {{%q, 0}}, 1, true\n", e.codepoints[0])
+		}
 
 		should_close = true
 	}
 	fmt.wprintln(w, "\t\t}")
 	fmt.wprintln(w, "\t}")
-	fmt.wprintln(w, "\treturn -1, false")
+	fmt.wprintln(w, "\treturn")
 	fmt.wprintln(w, "}\n")
 	fmt.wprintln(w, GENERATED)
 
@@ -221,12 +234,6 @@ named_xml_entity_to_rune :: proc(name: string) -> (decoded: rune, ok: bool) {
 	} else {
 		fmt.printf("Failed to write generated \"%v\".\n", generated_filename)
 	}
-
-	delete(entity_map)
-	delete(names)
-	for &name in names {
-		free(&name)
-	}
 }
 
 GENERATED :: `/*
@@ -262,14 +269,6 @@ TABLE_FILE_PROLOG :: `/*
 */
 `
 
-rune_to_string :: proc(r: rune) -> (res: string) {
-	res = fmt.tprintf("%08x", int(r))
-	for len(res) > 2 && res[:2] == "00" {
-		res = res[2:]
-	}
-	return fmt.tprintf("rune(0x%v)", res)
-}
-
 is_dotted_name :: proc(name: string) -> (dotted: bool) {
 	for r in name {
 		if r == '.' { return true}