mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-05-12 15:41:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
Jeroen van Rijn
2022-07-28 16:01:18 +02:00
committed by GitHub
parent 9746e25784 96eecaab54
commit 674ebe395f
619 changed files with 122594 additions and 30627 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/FUNDING.yml vendored
View File

@@ -1,3 +1,4 @@
# These are supported funding model platforms
github: odin-lang
patreon: gingerbill

78
.github/workflows/ci.yml vendored
View File

@@ -1,5 +1,5 @@
name: CI
on: [push, pull_request]
on: [push, pull_request, workflow_dispatch]
jobs:
build_linux:
@@ -7,9 +7,9 @@ jobs:
steps:
- uses: actions/checkout@v1
- name: Download LLVM, botan
run: sudo apt-get install llvm-11 clang-11 llvm libbotan-2-dev botan
run: sudo apt-get install llvm-11 clang-11 libbotan-2-dev botan
- name: build odin
run: make release
run: ./build_odin.sh release
- name: Odin version
run: ./odin version
timeout-minutes: 1
@@ -17,13 +17,16 @@ jobs:
run: ./odin report
timeout-minutes: 1
- name: Odin check
run: ./odin check examples/demo/demo.odin -vet
run: ./odin check examples/demo -vet
timeout-minutes: 10
- name: Odin run
run: ./odin run examples/demo/demo.odin
run: ./odin run examples/demo
timeout-minutes: 10
- name: Odin run -debug
run: ./odin run examples/demo/demo.odin -debug
run: ./odin run examples/demo -debug
timeout-minutes: 10
- name: Odin check examples/all
run: ./odin check examples/all -strict-style
timeout-minutes: 10
- name: Core library tests
run: |
@@ -35,6 +38,20 @@ jobs:
cd tests/vendor
make
timeout-minutes: 10
- name: Odin issues tests
run: |
cd tests/issues
./run.sh
timeout-minutes: 10
- name: Odin check examples/all for Linux i386
run: ./odin check examples/all -vet -strict-style -target:linux_i386
timeout-minutes: 10
- name: Odin check examples/all for FreeBSD amd64
run: ./odin check examples/all -vet -strict-style -target:freebsd_amd64
timeout-minutes: 10
- name: Odin check examples/all for OpenBSD amd64
run: ./odin check examples/all -vet -strict-style -target:openbsd_amd64
timeout-minutes: 10
build_macOS:
runs-on: macos-latest
steps:
@@ -46,7 +63,7 @@ jobs:
TMP_PATH=$(xcrun --show-sdk-path)/user/include
echo "CPATH=$TMP_PATH" >> $GITHUB_ENV
- name: build odin
run: make release
run: ./build_odin.sh release
- name: Odin version
run: ./odin version
timeout-minutes: 1
@@ -54,13 +71,16 @@ jobs:
run: ./odin report
timeout-minutes: 1
- name: Odin check
run: ./odin check examples/demo/demo.odin -vet
run: ./odin check examples/demo -vet
timeout-minutes: 10
- name: Odin run
run: ./odin run examples/demo/demo.odin
run: ./odin run examples/demo
timeout-minutes: 10
- name: Odin run -debug
run: ./odin run examples/demo/demo.odin -debug
run: ./odin run examples/demo -debug
timeout-minutes: 10
- name: Odin check examples/all
run: ./odin check examples/all -strict-style
timeout-minutes: 10
- name: Core library tests
run: |
@@ -72,8 +92,19 @@ jobs:
cd tests/vendor
make
timeout-minutes: 10
- name: Odin issues tests
run: |
cd tests/issues
./run.sh
timeout-minutes: 10
- name: Odin check examples/all for Darwin arm64
run: ./odin check examples/all -vet -strict-style -target:darwin_arm64
timeout-minutes: 10
- name: Odin check examples/all for Linux arm64
run: ./odin check examples/all -vet -strict-style -target:linux_arm64
timeout-minutes: 10
build_windows:
runs-on: windows-latest
runs-on: windows-2019
steps:
- uses: actions/checkout@v1
- name: build Odin
@@ -91,19 +122,25 @@ jobs:
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin check examples/demo/demo.odin -vet
odin check examples/demo -vet
timeout-minutes: 10
- name: Odin run
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin run examples/demo/demo.odin
odin run examples/demo
timeout-minutes: 10
- name: Odin run -debug
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin run examples/demo/demo.odin -debug
odin run examples/demo -debug
timeout-minutes: 10
- name: Odin check examples/all
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin check examples/all -strict-style
timeout-minutes: 10
- name: Core library tests
shell: cmd
@@ -126,3 +163,16 @@ jobs:
cd tests\core\math\big
call build.bat
timeout-minutes: 10
- name: Odin issues tests
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
cd tests\issues
call run.bat
timeout-minutes: 10
- name: Odin check examples/all for Windows 32bits
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin check examples/all -strict-style -target:windows_i386
timeout-minutes: 10

14
.github/workflows/nightly.yml vendored
View File

@@ -7,7 +7,7 @@ on:
jobs:
build_windows:
runs-on: windows-latest
runs-on: windows-2019
steps:
- uses: actions/checkout@v1
- name: build Odin
@@ -19,7 +19,7 @@ jobs:
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
odin run examples/demo/demo.odin
odin run examples/demo
- name: Copy artifacts
run: |
rm bin/llvm/windows/LLVM-C.lib
@@ -41,11 +41,11 @@ jobs:
steps:
- uses: actions/checkout@v1
- name: (Linux) Download LLVM
run: sudo apt-get install llvm-11 clang-11 llvm
run: sudo apt-get install llvm-11 clang-11
- name: build odin
run: make nightly
- name: Odin run
run: ./odin run examples/demo/demo.odin
run: ./odin run examples/demo
- name: Copy artifacts
run: |
mkdir dist
@@ -72,7 +72,7 @@ jobs:
- name: build odin
run: make nightly
- name: Odin run
run: ./odin run examples/demo/demo.odin
run: ./odin run examples/demo
- name: Copy artifacts
run: |
mkdir dist
@@ -129,7 +129,7 @@ jobs:
run: |
echo Authorizing B2 account
b2 authorize-account "$APPID" "$APPKEY"
echo Uploading artifcates to B2
chmod +x ./ci/upload_create_nightly.sh
./ci/upload_create_nightly.sh "$BUCKET" windows-amd64 windows_artifacts/
@@ -141,7 +141,7 @@ jobs:
echo Creating nightly.json
python3 ci/create_nightly_json.py "$BUCKET" > nightly.json
echo Uploading nightly.json
b2 upload-file "$BUCKET" nightly.json nightly.json

43
.github/workflows/stale.yml vendored Normal file
View File

@@ -0,0 +1,43 @@
name: "Close Stale Issues & PRs"
on:
workflow_dispatch:
schedule:
- cron: "0 21 * * *"
permissions:
issues: write
pull-requests: write
jobs:
stale:
runs-on: ubuntu-latest
steps:
- name: Close Stale Issues
uses: actions/stale@v4.1.0
with:
# stale-issue-message: |
# Hello!
#
# I am marking this issue as stale as it has not received any engagement from the community or maintainers 120 days. That does not imply that the issue has no merit! If you feel strongly about this issue
# - open a PR referencing and resolving the issue;
# - leave a comment on it and discuss ideas how you could contribute towards resolving it;
# - leave a comment and describe in detail why this issue is critical for your use case;
# - open a new issue with updated details and a plan on resolving the issue.
#
# The motivation for this automation is to help prioritize issues in the backlog and not ignore, reject, or belittle anyone..
#
# stale-pr-message: |
# Hello!
#
# I am marking this PR as stale as it has not received any engagement from the community or maintainers 120 days. That does not imply that the issue has no merit! If you feel strongly about this issue
# - leave a comment on it and discuss ideas how you could contribute towards resolving it;
# - leave a comment and describe in detail why this issue is critical for your use case;
#
# The motivation for this automation is to help prioritize issues in the backlog and not ignore, reject, or belittle anyone..
days-before-stale: 120
days-before-close: 30
exempt-draft-pr: true
ascending: true
operations-per-run: 1000
exempt-issue-labels: "ignore"

7
.gitignore vendored
View File

@@ -7,6 +7,9 @@
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# For macOS
.DS_Store
# Build results
[Dd]ebug/
[Dd]ebugPublic/
@@ -266,6 +269,8 @@ bin/
# - Linux/MacOS
odin
odin.dSYM
*.bin
demo.bin
# shared collection
shared/
@@ -276,3 +281,5 @@ shared/
*.ll
*.sublime-workspace
examples/bug/
build.sh

2
LICENSE
View File

@@ -1,4 +1,4 @@
Copyright (c) 2016-2021 Ginger Bill. All rights reserved.
Copyright (c) 2016-2022 Ginger Bill. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

73
Makefile
View File

@@ -1,80 +1,19 @@
GIT_SHA=$(shell git rev-parse --short HEAD)
DISABLED_WARNINGS=-Wno-switch -Wno-macro-redefined -Wno-unused-value
LDFLAGS=-pthread -ldl -lm -lstdc++
CFLAGS=-std=c++14 -DGIT_SHA=\"$(GIT_SHA)\"
CFLAGS:=$(CFLAGS) -DODIN_VERSION_RAW=\"dev-$(shell date +"%Y-%m")\"
CC=clang
OS=$(shell uname)
ifeq ($(OS), Darwin)
ARCH=$(shell uname -m)
LLVM_CONFIG=
# allow for arm only llvm's with version 13
ifeq ($(ARCH), arm64)
LLVM_VERSIONS = "13.%.%"
else
# allow for x86 / amd64 all llvm versions begining from 11
LLVM_VERSIONS = "13.%.%" "12.0.1" "11.1.0"
endif
LLVM_VERSION_PATTERN_SEPERATOR = )|(
LLVM_VERSION_PATTERNS_ESCAPED_DOT = $(subst .,\.,$(LLVM_VERSIONS))
LLVM_VERSION_PATTERNS_REPLACE_PERCENT = $(subst %,.*,$(LLVM_VERSION_PATTERNS_ESCAPED_DOT))
LLVM_VERSION_PATTERN_REMOVE_ELEMENTS = $(subst " ",$(LLVM_VERSION_PATTERN_SEPERATOR),$(LLVM_VERSION_PATTERNS_REPLACE_PERCENT))
LLMV_VERSION_PATTERN_REMOVE_SINGLE_STR = $(subst ",,$(LLVM_VERSION_PATTERN_REMOVE_ELEMENTS))
LLVM_VERSION_PATTERN = "^(($(LLMV_VERSION_PATTERN_REMOVE_SINGLE_STR)))"
ifneq ($(shell llvm-config --version | grep -E $(LLVM_VERSION_PATTERN)),)
LLVM_CONFIG=llvm-config
else
ifeq ($(ARCH), arm64)
$(error "Requirement: llvm-config must be base version 13 for arm64")
else
$(error "Requirement: llvm-config must be base version greater than 11 for amd64/x86")
endif
endif
LDFLAGS:=$(LDFLAGS) -liconv
CFLAGS:=$(CFLAGS) $(shell $(LLVM_CONFIG) --cxxflags --ldflags)
LDFLAGS:=$(LDFLAGS) -lLLVM-C
endif
ifeq ($(OS), Linux)
LLVM_CONFIG=llvm-config-11
ifneq ($(shell which llvm-config-11 2>/dev/null),)
LLVM_CONFIG=llvm-config-11
else ifneq ($(shell which llvm-config-11-64 2>/dev/null),)
LLVM_CONFIG=llvm-config-11-64
else
ifneq ($(shell llvm-config --version | grep '^11\.'),)
LLVM_CONFIG=llvm-config
else
$(error "Requirement: llvm-config must be version 11")
endif
endif
CFLAGS:=$(CFLAGS) $(shell $(LLVM_CONFIG) --cxxflags --ldflags)
LDFLAGS:=$(LDFLAGS) $(shell $(LLVM_CONFIG) --libs core native --system-libs)
endif
all: debug demo
all: debug
demo:
./odin run examples/demo/demo.odin
./odin run examples/demo/demo.odin -file
report:
./odin report
debug:
$(CC) src/main.cpp src/libtommath.cpp $(DISABLED_WARNINGS) $(CFLAGS) -g $(LDFLAGS) -o odin
./build_odin.sh debug
release:
$(CC) src/main.cpp src/libtommath.cpp $(DISABLED_WARNINGS) $(CFLAGS) -O3 $(LDFLAGS) -o odin
./build_odin.sh release
release_native:
$(CC) src/main.cpp src/libtommath.cpp $(DISABLED_WARNINGS) $(CFLAGS) -O3 -march=native $(LDFLAGS) -o odin
./build_odin.sh release-native
nightly:
$(CC) src/main.cpp src/libtommath.cpp $(DISABLED_WARNINGS) $(CFLAGS) -DNIGHTLY -O3 $(LDFLAGS) -o odin
./build_odin.sh nightly

16
README.md
View File

@@ -11,7 +11,7 @@
<img src="https://img.shields.io/badge/platforms-Windows%20|%20Linux%20|%20macOS-green.svg">
</a>
<br>
<a href="https://discord.gg/hnwN2Rj">
<a href="https://discord.gg/odinlang">
<img src="https://img.shields.io/discord/568138951836172421?logo=discord">
</a>
<a href="https://github.com/odin-lang/odin/actions">
@@ -58,6 +58,10 @@ main :: proc() {
Instructions for downloading and installing the Odin compiler and libraries.
#### [Nightly Builds](https://odin-lang.org/docs/nightly/)
Get the latest nightly builds of Odin.
### Learning Odin
#### [Overview of Odin](https://odin-lang.org/docs/overview)
@@ -68,6 +72,10 @@ An overview of the Odin programming language.
Answers to common questions about Odin.
#### [Packages](https://pkg.odin-lang.org/)
Documentation for all the official packages part of the [core](https://pkg.odin-lang.org/core/) and [vendor](https://pkg.odin-lang.org/vendor/) library collections.
#### [The Odin Wiki](https://github.com/odin-lang/Odin/wiki)
A wiki maintained by the Odin community.
@@ -76,12 +84,6 @@ A wiki maintained by the Odin community.
Get live support and talk with other odiners on the Odin Discord.
### References
#### [Language Specification](https://odin-lang.org/docs/spec/)
The official Odin Language specification.
### Articles
#### [The Odin Blog](https://odin-lang.org/news/)

2
build.bat
View File

@@ -58,7 +58,7 @@ set libs= ^
set linker_flags= -incremental:no -opt:ref -subsystem:console
if %release_mode% EQU 0 ( rem Debug
set linker_flags=%linker_flags% -debug
set linker_flags=%linker_flags% -debug /NATVIS:src\odin_compiler.natvis
) else ( rem Release
set linker_flags=%linker_flags% -debug
)

150
build_odin.sh Executable file
View File

@@ -0,0 +1,150 @@
#!/usr/bin/env bash
set -eu
GIT_SHA=$(git rev-parse --short HEAD)
DISABLED_WARNINGS="-Wno-switch -Wno-macro-redefined -Wno-unused-value"
LDFLAGS="-pthread -lm -lstdc++"
CFLAGS="-std=c++14 -DGIT_SHA=\"$GIT_SHA\""
CFLAGS="$CFLAGS -DODIN_VERSION_RAW=\"dev-$(date +"%Y-%m")\""
CC=clang
OS=$(uname)
panic() {
printf "%s\n" "$1"
exit 1
}
version() { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; }
config_darwin() {
ARCH=$(uname -m)
LLVM_CONFIG=llvm-config
# allow for arm only llvm's with version 13
if [ ARCH == arm64 ]; then
MIN_LLVM_VERSION=("13.0.0")
else
# allow for x86 / amd64 all llvm versions begining from 11
MIN_LLVM_VERSION=("11.1.0")
fi
if [ $(version $($LLVM_CONFIG --version)) -lt $(version $MIN_LLVM_VERSION) ]; then
if [ ARCH == arm64 ]; then
panic "Requirement: llvm-config must be base version 13 for arm64"
else
panic "Requirement: llvm-config must be base version greater than 11 for amd64/x86"
fi
fi
LDFLAGS="$LDFLAGS -liconv -ldl"
CFLAGS="$CFLAGS $($LLVM_CONFIG --cxxflags --ldflags)"
LDFLAGS="$LDFLAGS -lLLVM-C"
}
config_freebsd() {
LLVM_CONFIG=/usr/local/bin/llvm-config11
CFLAGS="$CFLAGS $($LLVM_CONFIG --cxxflags --ldflags)"
LDFLAGS="$LDFLAGS $($LLVM_CONFIG --libs core native --system-libs)"
}
config_openbsd() {
LLVM_CONFIG=/usr/local/bin/llvm-config
LDFLAGS="$LDFLAGS -liconv"
CFLAGS="$CFLAGS $($LLVM_CONFIG --cxxflags --ldflags)"
LDFLAGS="$LDFLAGS $($LLVM_CONFIG --libs core native --system-libs)"
}
config_linux() {
if which llvm-config > /dev/null 2>&1; then
LLVM_CONFIG=llvm-config
elif which llvm-config-11 > /dev/null 2>&1; then
LLVM_CONFIG=llvm-config-11
elif which llvm-config-11-64 > /dev/null 2>&1; then
LLVM_CONFIG=llvm-config-11-64
else
panic "Unable to find LLVM-config"
fi
MIN_LLVM_VERSION=("11.0.0")
if [ $(version $($LLVM_CONFIG --version)) -lt $(version $MIN_LLVM_VERSION) ]; then
echo "Tried to use " $(which $LLVM_CONFIG) "version" $($LLVM_CONFIG --version)
panic "Requirement: llvm-config must be base version greater than 11"
fi
LDFLAGS="$LDFLAGS -ldl"
CFLAGS="$CFLAGS $($LLVM_CONFIG --cxxflags --ldflags)"
LDFLAGS="$LDFLAGS $($LLVM_CONFIG --libs core native --system-libs)"
}
build_odin() {
case $1 in
debug)
EXTRAFLAGS="-g"
;;
release)
EXTRAFLAGS="-O3"
;;
release-native)
EXTRAFLAGS="-O3 -march=native"
;;
nightly)
EXTRAFLAGS="-DNIGHTLY -O3"
;;
*)
panic "Build mode unsupported!"
esac
set -x
$CC src/main.cpp src/libtommath.cpp $DISABLED_WARNINGS $CFLAGS $EXTRAFLAGS $LDFLAGS -o odin
set +x
}
run_demo() {
./odin run examples/demo/demo.odin -file
}
case $OS in
Linux)
config_linux
;;
Darwin)
config_darwin
;;
OpenBSD)
config_openbsd
;;
FreeBSD)
config_freebsd
;;
*)
panic "Platform unsupported!"
esac
if [[ $# -eq 0 ]]; then
build_odin debug
run_demo
exit 0
fi
if [[ $# -eq 1 ]]; then
case $1 in
report)
if [[ ! -f "./odin" ]]; then
build_odin debug
fi
./odin report
exit 0
;;
*)
build_odin $1
;;
esac
run_demo
exit 0
else
panic "Too many arguments!"
fi

9
core/bufio/scanner.odin
View File

@@ -8,6 +8,7 @@ import "core:intrinsics"
// Extra errors returns by scanning procedures
Scanner_Extra_Error :: enum i32 {
None,
Negative_Advance,
Advanced_Too_Far,
Bad_Read_Count,
@@ -15,7 +16,7 @@ Scanner_Extra_Error :: enum i32 {
Too_Short,
}
Scanner_Error :: union {
Scanner_Error :: union #shared_nil {
io.Error,
Scanner_Extra_Error,
}
@@ -68,7 +69,7 @@ scanner_destroy :: proc(s: ^Scanner) {
// Returns the first non-EOF error that was encounted by the scanner
scanner_error :: proc(s: ^Scanner) -> Scanner_Error {
switch s._err {
case .EOF, .None:
case .EOF, nil:
return nil
}
return s._err
@@ -93,10 +94,6 @@ scanner_text :: proc(s: ^Scanner) -> string {
// scanner_scan advances the scanner
scanner_scan :: proc(s: ^Scanner) -> bool {
set_err :: proc(s: ^Scanner, err: Scanner_Error) {
err := err
if err == .None {
err = nil
}
switch s._err {
case nil, .EOF:
s._err = err

132
core/builtin/builtin.odin
View File

@@ -1,90 +1,90 @@
// This is purely for documentation
package builtin
nil :: nil;
false :: 0!==0;
true :: 0==0;
nil :: nil
false :: 0!=0
true :: 0==0
ODIN_OS :: ODIN_OS;
ODIN_ARCH :: ODIN_ARCH;
ODIN_ENDIAN :: ODIN_ENDIAN;
ODIN_VENDOR :: ODIN_VENDOR;
ODIN_VERSION :: ODIN_VERSION;
ODIN_ROOT :: ODIN_ROOT;
ODIN_DEBUG :: ODIN_DEBUG;
ODIN_OS :: ODIN_OS
ODIN_ARCH :: ODIN_ARCH
ODIN_ENDIAN :: ODIN_ENDIAN
ODIN_VENDOR :: ODIN_VENDOR
ODIN_VERSION :: ODIN_VERSION
ODIN_ROOT :: ODIN_ROOT
ODIN_DEBUG :: ODIN_DEBUG
byte :: u8; // alias
byte :: u8 // alias
bool :: bool;
b8 :: b8;
b16 :: b16;
b32 :: b32;
b64 :: b64;
bool :: bool
b8 :: b8
b16 :: b16
b32 :: b32
b64 :: b64
i8 :: i8;
u8 :: u8;
i16 :: i16;
u16 :: u16;
i32 :: i32;
u32 :: u32;
i64 :: i64;
u64 :: u64;
i8 :: i8
u8 :: u8
i16 :: i16
u16 :: u16
i32 :: i32
u32 :: u32
i64 :: i64
u64 :: u64
i128 :: i128;
u128 :: u128;
i128 :: i128
u128 :: u128
rune :: rune;
rune :: rune
f16 :: f16;
f32 :: f32;
f64 :: f64;
f16 :: f16
f32 :: f32
f64 :: f64
complex32 :: complex32;
complex64 :: complex64;
complex128 :: complex128;
complex32 :: complex32
complex64 :: complex64
complex128 :: complex128
quaternion64 :: quaternion64;
quaternion128 :: quaternion128;
quaternion256 :: quaternion256;
quaternion64 :: quaternion64
quaternion128 :: quaternion128
quaternion256 :: quaternion256
int :: int;
uint :: uint;
uintptr :: uintptr;
int :: int
uint :: uint
uintptr :: uintptr
rawptr :: rawptr;
string :: string;
cstring :: cstring;
any :: any;
rawptr :: rawptr
string :: string
cstring :: cstring
any :: any
typeid :: typeid;
typeid :: typeid
// Endian Specific Types
i16le :: i16le;
u16le :: u16le;
i32le :: i32le;
u32le :: u32le;
i64le :: i64le;
u64le :: u64le;
i128le :: i128le;
u128le :: u128le;
i16le :: i16le
u16le :: u16le
i32le :: i32le
u32le :: u32le
i64le :: i64le
u64le :: u64le
i128le :: i128le
u128le :: u128le
i16be :: i16be;
u16be :: u16be;
i32be :: i32be;
u32be :: u32be;
i64be :: i64be;
u64be :: u64be;
i128be :: i128be;
u128be :: u128be;
i16be :: i16be
u16be :: u16be
i32be :: i32be
u32be :: u32be
i64be :: i64be
u64be :: u64be
i128be :: i128be
u128be :: u128be
f16le :: f16le;
f32le :: f32le;
f64le :: f64le;
f16le :: f16le
f32le :: f32le
f64le :: f64le
f16be :: f16be;
f32be :: f32be;
f64be :: f64be;
f16be :: f16be
f32be :: f32be
f64be :: f64be

8
core/bytes/buffer.odin
View File

@@ -161,6 +161,10 @@ buffer_write :: proc(b: ^Buffer, p: []byte) -> (n: int, err: io.Error) {
return copy(b.buf[m:], p), nil
}
buffer_write_ptr :: proc(b: ^Buffer, ptr: rawptr, size: int) -> (n: int, err: io.Error) {
return buffer_write(b, ([^]byte)(ptr)[:size])
}
buffer_write_string :: proc(b: ^Buffer, s: string) -> (n: int, err: io.Error) {
b.last_read = .Invalid
m, ok := _buffer_try_grow(b, len(s))
@@ -229,6 +233,10 @@ buffer_read :: proc(b: ^Buffer, p: []byte) -> (n: int, err: io.Error) {
return
}
buffer_read_ptr :: proc(b: ^Buffer, ptr: rawptr, size: int) -> (n: int, err: io.Error) {
return buffer_read(b, ([^]byte)(ptr)[:size])
}
buffer_read_at :: proc(b: ^Buffer, p: []byte, offset: int) -> (n: int, err: io.Error) {
b.last_read = .Invalid

92
core/bytes/bytes.odin
View File

@@ -10,7 +10,14 @@ clone :: proc(s: []byte, allocator := context.allocator, loc := #caller_location
return c[:len(s)]
}
ptr_from_slice :: proc(str: []byte) -> ^byte {
clone_safe :: proc(s: []byte, allocator := context.allocator, loc := #caller_location) -> (data: []byte, err: mem.Allocator_Error) {
c := make([]byte, len(s), allocator, loc) or_return
copy(c, s)
return c[:len(s)], nil
}
ptr_from_slice :: ptr_from_bytes
ptr_from_bytes :: proc(str: []byte) -> ^byte {
d := transmute(mem.Raw_String)str
return d.data
}
@@ -134,6 +141,25 @@ join :: proc(a: [][]byte, sep: []byte, allocator := context.allocator) -> []byte
return b
}
join_safe :: proc(a: [][]byte, sep: []byte, allocator := context.allocator) -> (data: []byte, err: mem.Allocator_Error) {
if len(a) == 0 {
return nil, nil
}
n := len(sep) * (len(a) - 1)
for s in a {
n += len(s)
}
b := make([]byte, n, allocator) or_return
i := copy(b, a[0])
for s in a[1:] {
i += copy(b[i:], sep)
i += copy(b[i:], s)
}
return b, nil
}
concatenate :: proc(a: [][]byte, allocator := context.allocator) -> []byte {
if len(a) == 0 {
return nil
@@ -151,6 +177,24 @@ concatenate :: proc(a: [][]byte, allocator := context.allocator) -> []byte {
return b
}
concatenate_safe :: proc(a: [][]byte, allocator := context.allocator) -> (data: []byte, err: mem.Allocator_Error) {
if len(a) == 0 {
return nil, nil
}
n := 0
for s in a {
n += len(s)
}
b := make([]byte, n, allocator) or_return
i := 0
for s in a {
i += copy(b[i:], s)
}
return b, nil
}
@private
_split :: proc(s, sep: []byte, sep_save, n: int, allocator := context.allocator) -> [][]byte {
s, n := s, n
@@ -218,61 +262,37 @@ split_after_n :: proc(s, sep: []byte, n: int, allocator := context.allocator) ->
@private
_split_iterator :: proc(s: ^[]byte, sep: []byte, sep_save, n: int) -> (res: []byte, ok: bool) {
s, n := s, n
if n == 0 {
return
}
if sep == nil {
_split_iterator :: proc(s: ^[]byte, sep: []byte, sep_save: int) -> (res: []byte, ok: bool) {
if len(sep) == 0 {
res = s[:]
ok = true
s^ = s[len(s):]
return
}
if n < 0 {
n = count(s^, sep) + 1
}
n -= 1
i := 0
for ; i < n; i += 1 {
m := index(s^, sep)
if m < 0 {
break
}
m := index(s^, sep)
if m < 0 {
// not found
res = s[:]
ok = len(res) != 0
s^ = s[len(s):]
} else {
res = s[:m+sep_save]
ok = true
s^ = s[m+len(sep):]
return
}
res = s[:]
ok = res != nil
s^ = s[len(s):]
return
}
split_iterator :: proc(s: ^[]byte, sep: []byte) -> ([]byte, bool) {
return _split_iterator(s, sep, 0, -1)
}
split_n_iterator :: proc(s: ^[]byte, sep: []byte, n: int) -> ([]byte, bool) {
return _split_iterator(s, sep, 0, n)
return _split_iterator(s, sep, 0)
}
split_after_iterator :: proc(s: ^[]byte, sep: []byte) -> ([]byte, bool) {
return _split_iterator(s, sep, len(sep), -1)
return _split_iterator(s, sep, len(sep))
}
split_after_n_iterator :: proc(s: ^[]byte, sep: []byte, n: int) -> ([]byte, bool) {
return _split_iterator(s, sep, len(sep), n)
}
index_byte :: proc(s: []byte, c: byte) -> int {
for i := 0; i < len(s); i += 1 {

8
core/c/c.odin
View File

@@ -7,20 +7,20 @@ char :: builtin.u8 // assuming -funsigned-char
schar :: builtin.i8
short :: builtin.i16
int :: builtin.i32
long :: builtin.i32 when (ODIN_OS == "windows" || size_of(builtin.rawptr) == 4) else builtin.i64
long :: builtin.i32 when (ODIN_OS == .Windows || size_of(builtin.rawptr) == 4) else builtin.i64
longlong :: builtin.i64
uchar :: builtin.u8
ushort :: builtin.u16
uint :: builtin.u32
ulong :: builtin.u32 when (ODIN_OS == "windows" || size_of(builtin.rawptr) == 4) else builtin.u64
ulong :: builtin.u32 when (ODIN_OS == .Windows || size_of(builtin.rawptr) == 4) else builtin.u64
ulonglong :: builtin.u64
bool :: builtin.bool
size_t :: builtin.uint
ssize_t :: builtin.int
wchar_t :: builtin.u16 when (ODIN_OS == "windows") else builtin.u32
wchar_t :: builtin.u16 when (ODIN_OS == .Windows) else builtin.u32
float :: builtin.f32
double :: builtin.f64
@@ -48,7 +48,7 @@ int_least64_t :: builtin.i64
uint_least64_t :: builtin.u64
// Same on Windows, Linux, and FreeBSD
when ODIN_ARCH == "i386" || ODIN_ARCH == "amd64" {
when ODIN_ARCH == .i386 || ODIN_ARCH == .amd64 {
int_fast8_t :: builtin.i8
uint_fast8_t :: builtin.u8
int_fast16_t :: builtin.i32

4
core/c/frontend/preprocessor/preprocess.odin
View File

@@ -519,7 +519,7 @@ join_adjacent_string_literals :: proc(cpp: ^Preprocessor, initial_tok: ^Token) {
quote_string :: proc(s: string) -> []byte {
b := strings.make_builder(0, len(s)+2)
b := strings.builder_make(0, len(s)+2)
io.write_quoted_string(strings.to_writer(&b), s, '"')
return b.buf[:]
}
@@ -1276,7 +1276,7 @@ preprocess_internal :: proc(cpp: ^Preprocessor, tok: ^Token) -> ^Token {
if start.file != nil {
dir = filepath.dir(start.file.name)
}
path := filepath.join(dir, filename)
path := filepath.join({dir, filename})
if os.exists(path) {
tok = include_file(cpp, tok, path, start.next.next)
continue

4
core/c/libc/complex.odin
View File

@@ -2,9 +2,9 @@ package libc
// 7.3 Complex arithmetic
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"

4
core/c/libc/ctype.odin
View File

@@ -1,8 +1,8 @@
package libc
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"

23
core/c/libc/errno.odin
View File

@@ -2,9 +2,9 @@ package libc
// 7.5 Errors
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
@@ -14,7 +14,7 @@ when ODIN_OS == "windows" {
// EDOM,
// EILSEQ
// ERANGE
when ODIN_OS == "linux" || ODIN_OS == "freebsd" {
when ODIN_OS == .Linux || ODIN_OS == .FreeBSD {
@(private="file")
@(default_calling_convention="c")
foreign libc {
@@ -27,7 +27,20 @@ when ODIN_OS == "linux" || ODIN_OS == "freebsd" {
ERANGE :: 34
}
when ODIN_OS == "windows" {
when ODIN_OS == .OpenBSD {
@(private="file")
@(default_calling_convention="c")
foreign libc {
@(link_name="__errno")
_get_errno :: proc() -> ^int ---
}
EDOM :: 33
EILSEQ :: 84
ERANGE :: 34
}
when ODIN_OS == .Windows {
@(private="file")
@(default_calling_convention="c")
foreign libc {
@@ -40,7 +53,7 @@ when ODIN_OS == "windows" {
ERANGE :: 34
}
when ODIN_OS == "darwin" {
when ODIN_OS == .Darwin {
@(private="file")
@(default_calling_convention="c")
foreign libc {

24
core/c/libc/math.odin
View File

@@ -4,9 +4,9 @@ package libc
import "core:intrinsics"
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
@@ -211,19 +211,19 @@ _signbitf :: #force_inline proc(x: float) -> int {
return int(transmute(uint32_t)x >> 31)
}
isfinite :: #force_inline proc(x: $T) where intrinsics.type_is_float(T) {
isfinite :: #force_inline proc(x: $T) -> bool where intrinsics.type_is_float(T) {
return fpclassify(x) == FP_INFINITE
}
isinf :: #force_inline proc(x: $T) where intrinsics.type_is_float(T) {
isinf :: #force_inline proc(x: $T) -> bool where intrinsics.type_is_float(T) {
return fpclassify(x) > FP_INFINITE
}
isnan :: #force_inline proc(x: $T) where intrinsics.type_is_float(T) {
isnan :: #force_inline proc(x: $T) -> bool where intrinsics.type_is_float(T) {
return fpclassify(x) == FP_NAN
}
isnormal :: #force_inline proc(x: $T) where intrinsics.type_is_float(T) {
isnormal :: #force_inline proc(x: $T) -> bool where intrinsics.type_is_float(T) {
return fpclassify(x) == FP_NORMAL
}
@@ -231,27 +231,27 @@ isnormal :: #force_inline proc(x: $T) where intrinsics.type_is_float(T) {
// implemented as the relational comparisons, as that would produce an invalid
// "sticky" state that propagates and affects maths results. These need
// to be implemented natively in Odin assuming isunordered to prevent that.
isgreater :: #force_inline proc(x, y: $T) where intrinsics.type_is_float(T) {
isgreater :: #force_inline proc(x, y: $T) -> bool where intrinsics.type_is_float(T) {
return !isunordered(x, y) && x > y
}
isgreaterequal :: #force_inline proc(x, y: $T) where intrinsics.type_is_float(T) {
isgreaterequal :: #force_inline proc(x, y: $T) -> bool where intrinsics.type_is_float(T) {
return !isunordered(x, y) && x >= y
}
isless :: #force_inline proc(x, y: $T) where intrinsics.type_is_float(T) {
isless :: #force_inline proc(x, y: $T) -> bool where intrinsics.type_is_float(T) {
return !isunordered(x, y) && x < y
}
islessequal :: #force_inline proc(x, y: $T) where intrinsics.type_is_float(T) {
islessequal :: #force_inline proc(x, y: $T) -> bool where intrinsics.type_is_float(T) {
return !isunordered(x, y) && x <= y
}
islessgreater :: #force_inline proc(x, y: $T) where intrinsics.type_is_float(T) {
islessgreater :: #force_inline proc(x, y: $T) -> bool where intrinsics.type_is_float(T) {
return !isunordered(x, y) && x <= y
}
isunordered :: #force_inline proc(x, y: $T) where intrinsics.type_is_float(T) {
isunordered :: #force_inline proc(x, y: $T) -> bool where intrinsics.type_is_float(T) {
if isnan(x) {
// Force evaluation of y to propagate exceptions for ordering semantics.
// To ensure correct semantics of IEEE 754 this cannot be compiled away.

6
core/c/libc/setjmp.odin
View File

@@ -2,14 +2,14 @@ package libc
// 7.13 Nonlocal jumps
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
}
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
@(default_calling_convention="c")
foreign libc {
// 7.13.1 Save calling environment

10
core/c/libc/signal.odin
View File

@@ -2,9 +2,9 @@ package libc
// 7.14 Signal handling
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
@@ -21,7 +21,7 @@ foreign libc {
raise :: proc(sig: int) -> int ---
}
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
SIG_ERR :: rawptr(~uintptr(0))
SIG_DFL :: rawptr(uintptr(0))
SIG_IGN :: rawptr(uintptr(1))
@@ -34,7 +34,7 @@ when ODIN_OS == "windows" {
SIGTERM :: 15
}
when ODIN_OS == "linux" || ODIN_OS == "freebsd" {
when ODIN_OS == .Linux || ODIN_OS == .FreeBSD {
SIG_ERR :: rawptr(~uintptr(0))
SIG_DFL :: rawptr(uintptr(0))
SIG_IGN :: rawptr(uintptr(1))
@@ -47,7 +47,7 @@ when ODIN_OS == "linux" || ODIN_OS == "freebsd" {
SIGTERM :: 15
}
when ODIN_OS == "darwin" {
when ODIN_OS == .Darwin {
SIG_ERR :: rawptr(~uintptr(0))
SIG_DFL :: rawptr(uintptr(0))
SIG_IGN :: rawptr(uintptr(1))

257
core/c/libc/stdatomic.odin
View File

@@ -47,29 +47,30 @@ kill_dependency :: #force_inline proc(value: $T) -> T {
// 7.17.4 Fences
atomic_thread_fence :: #force_inline proc(order: memory_order) {
switch (order) {
case .relaxed:
return
case .consume:
intrinsics.atomic_fence_acq()
case .acquire:
intrinsics.atomic_fence_acq()
case .release:
intrinsics.atomic_fence_rel()
case .acq_rel:
intrinsics.atomic_fence_acqrel()
case .seq_cst:
intrinsics.atomic_fence_acqrel()
assert(order != .relaxed)
assert(order != .consume)
#partial switch order {
case .acquire: intrinsics.atomic_thread_fence(.Acquire)
case .release: intrinsics.atomic_thread_fence(.Release)
case .acq_rel: intrinsics.atomic_thread_fence(.Acq_Rel)
case .seq_cst: intrinsics.atomic_thread_fence(.Seq_Cst)
}
}
atomic_signal_fence :: #force_inline proc(order: memory_order) {
atomic_thread_fence(order)
assert(order != .relaxed)
assert(order != .consume)
#partial switch order {
case .acquire: intrinsics.atomic_signal_fence(.Acquire)
case .release: intrinsics.atomic_signal_fence(.Release)
case .acq_rel: intrinsics.atomic_signal_fence(.Acq_Rel)
case .seq_cst: intrinsics.atomic_signal_fence(.Seq_Cst)
}
}
// 7.17.5 Lock-free property
atomic_is_lock_free :: #force_inline proc(obj: ^$T) -> bool {
return size_of(T) <= 8 && (intrinsics.type_is_integer(T) || intrinsics.type_is_pointer(T))
return intrinsics.atomic_type_is_lock_free(T)
}
// 7.17.6 Atomic integer types
@@ -121,13 +122,10 @@ atomic_store_explicit :: #force_inline proc(object: ^$T, desired: T, order: memo
assert(order != .acquire)
assert(order != .acq_rel)
#partial switch (order) {
case .relaxed:
intrinsics.atomic_store_relaxed(object, desired)
case .release:
intrinsics.atomic_store_rel(object, desired)
case .seq_cst:
intrinsics.atomic_store(object, desired)
#partial switch order {
case .relaxed: intrinsics.atomic_store_explicit(object, desired, .Relaxed)
case .release: intrinsics.atomic_store_explicit(object, desired, .Release)
case .seq_cst: intrinsics.atomic_store_explicit(object, desired, .Seq_Cst)
}
}
@@ -139,36 +137,26 @@ atomic_load_explicit :: #force_inline proc(object: ^$T, order: memory_order) {
assert(order != .release)
assert(order != .acq_rel)
#partial switch (order) {
case .relaxed:
return intrinsics.atomic_load_relaxed(object)
case .consume:
return intrinsics.atomic_load_acq(object)
case .acquire:
return intrinsics.atomic_load_acq(object)
case .seq_cst:
return intrinsics.atomic_load(object)
#partial switch order {
case .relaxed: return intrinsics.atomic_load_explicit(object, .Relaxed)
case .consume: return intrinsics.atomic_load_explicit(object, .Consume)
case .acquire: return intrinsics.atomic_load_explicit(object, .Acquire)
case .seq_cst: return intrinsics.atomic_load_explicit(object, .Seq_Cst)
}
}
atomic_exchange :: #force_inline proc(object: ^$T, desired: T) -> T {
return intrinsics.atomic_xchg(object, desired)
return intrinsics.atomic_exchange(object, desired)
}
atomic_exchange_explicit :: #force_inline proc(object: ^$T, desired: T, order: memory_order) -> T {
switch (order) {
case .relaxed:
return intrinsics.atomic_xchg_relaxed(object, desired)
case .consume:
return intrinsics.atomic_xchg_acq(object, desired)
case .acquire:
return intrinsics.atomic_xchg_acq(object, desired)
case .release:
return intrinsics.atomic_xchg_rel(object, desired)
case .acq_rel:
return intrinsics.atomic_xchg_acqrel(object, desired)
case .seq_cst:
return intrinsics.atomic_xchg(object, desired)
switch order {
case .relaxed: return intrinsics.atomic_exchange_explicit(object, desired, .Relaxed)
case .consume: return intrinsics.atomic_exchange_explicit(object, desired, .Consume)
case .acquire: return intrinsics.atomic_exchange_explicit(object, desired, .Acquire)
case .release: return intrinsics.atomic_exchange_explicit(object, desired, .Release)
case .acq_rel: return intrinsics.atomic_exchange_explicit(object, desired, .Acq_Rel)
case .seq_cst: return intrinsics.atomic_exchange_explicit(object, desired, .Seq_Cst)
}
return false
}
@@ -189,102 +177,104 @@ atomic_exchange_explicit :: #force_inline proc(object: ^$T, desired: T, order: m
// [success = seq_cst, failure = acquire] => failacq
// [success = acquire, failure = relaxed] => acq_failrelaxed
// [success = acq_rel, failure = relaxed] => acqrel_failrelaxed
atomic_compare_exchange_strong :: #force_inline proc(object, expected: ^$T, desired: T) {
value, ok := intrinsics.atomic_cxchg(object, expected^, desired)
atomic_compare_exchange_strong :: #force_inline proc(object, expected: ^$T, desired: T) -> bool {
value, ok := intrinsics.atomic_compare_exchange_strong(object, expected^, desired)
if !ok { expected^ = value }
return ok
}
atomic_compare_exchange_strong_explicit :: #force_inline proc(object, expected: ^$T, desired: T, success, failure: memory_order) {
atomic_compare_exchange_strong_explicit :: #force_inline proc(object, expected: ^$T, desired: T, success, failure: memory_order) -> bool {
assert(failure != .release)
assert(failure != .acq_rel)
value: T; ok: bool
#partial switch (failure) {
#partial switch failure {
case .seq_cst:
assert(success != .relaxed)
#partial switch (success) {
#partial switch success {
case .seq_cst:
value, ok := intrinsics.atomic_cxchg(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Seq_Cst, .Seq_Cst)
case .acquire:
value, ok := intrinsics.atomic_cxchg_acq(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Acquire, .Seq_Cst)
case .consume:
value, ok := intrinsics.atomic_cxchg_acq(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Consume, .Seq_Cst)
case .release:
value, ok := intrinsics.atomic_cxchg_rel(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Release, .Seq_Cst)
case .acq_rel:
value, ok := intrinsics.atomic_cxchg_acqrel(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Acq_Rel, .Seq_Cst)
}
case .relaxed:
assert(success != .release)
#partial switch (success) {
#partial switch success {
case .relaxed:
value, ok := intrinsics.atomic_cxchg_relaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Relaxed, .Relaxed)
case .seq_cst:
value, ok := intrinsics.atomic_cxchg_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Seq_Cst, .Relaxed)
case .acquire:
value, ok := intrinsics.atomic_cxchg_acq_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Acquire, .Relaxed)
case .consume:
value, ok := intrinsics.atomic_cxchg_acq_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Consume, .Relaxed)
case .acq_rel:
value, ok := intrinsics.atomic_cxchg_acqrel_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Acq_Rel, .Relaxed)
}
case .consume:
fallthrough
assert(success == .seq_cst)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Seq_Cst, .Consume)
case .acquire:
assert(success == .seq_cst)
value, ok := intrinsics.atomic_cxchg_failacq(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_strong_explicit(object, expected^, desired, .Seq_Cst, .Acquire)
}
if !ok { expected^ = value }
return ok
}
atomic_compare_exchange_weak :: #force_inline proc(object, expected: ^$T, desired: T) {
value, ok := intrinsics.atomic_cxchgweak(object, expected^, desired)
atomic_compare_exchange_weak :: #force_inline proc(object, expected: ^$T, desired: T) -> bool {
value, ok := intrinsics.atomic_compare_exchange_weak(object, expected^, desired)
if !ok { expected^ = value }
return ok
}
atomic_compare_exchange_weak_explicit :: #force_inline proc(object, expected: ^$T, desited: T, success, failure: memory_order) {
atomic_compare_exchange_weak_explicit :: #force_inline proc(object, expected: ^$T, desited: T, success, failure: memory_order) -> bool {
assert(failure != .release)
assert(failure != .acq_rel)
value: T; ok: bool
#partial switch (failure) {
#partial switch failure {
case .seq_cst:
assert(success != .relaxed)
#partial switch (success) {
#partial switch success {
case .seq_cst:
value, ok := intrinsics.atomic_cxchgweak(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Seq_Cst, .Seq_Cst)
case .acquire:
value, ok := intrinsics.atomic_cxchgweak_acq(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Acquire, .Seq_Cst)
case .consume:
value, ok := intrinsics.atomic_cxchgweak_acq(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Consume, .Seq_Cst)
case .release:
value, ok := intrinsics.atomic_cxchgweak_rel(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Release, .Seq_Cst)
case .acq_rel:
value, ok := intrinsics.atomic_cxchgweak_acqrel(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Acq_Rel, .Seq_Cst)
}
case .relaxed:
assert(success != .release)
#partial switch (success) {
#partial switch success {
case .relaxed:
value, ok := intrinsics.atomic_cxchgweak_relaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Relaxed, .Relaxed)
case .seq_cst:
value, ok := intrinsics.atomic_cxchgweak_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Seq_Cst, .Relaxed)
case .acquire:
value, ok := intrinsics.atomic_cxchgweak_acq_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Acquire, .Relaxed)
case .consume:
value, ok := intrinsics.atomic_cxchgweak_acq_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Consume, .Relaxed)
case .acq_rel:
value, ok := intrinsics.atomic_cxchgweak_acqrel_failrelaxed(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Acq_Rel, .Relaxed)
}
case .consume:
fallthrough
assert(success == .seq_cst)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Seq_Cst, .Consume)
case .acquire:
assert(success == .seq_cst)
value, ok := intrinsics.atomic_cxchgweak_failacq(object, expected^, desired)
value, ok = intrinsics.atomic_compare_exchange_weak_explicit(object, expected^, desired, .Seq_Cst, .Acquire)
}
if !ok { expected^ = value }
@@ -297,19 +287,14 @@ atomic_fetch_add :: #force_inline proc(object: ^$T, operand: T) -> T {
}
atomic_fetch_add_explicit :: #force_inline proc(object: ^$T, operand: T, order: memory_order) -> T {
switch (order) {
case .relaxed:
return intrinsics.atomic_add_relaxed(object, operand)
case .consume:
return intrinsics.atomic_add_acq(object, operand)
case .acquire:
return intrinsics.atomic_add_acq(object, operand)
case .release:
return intrinsics.atomic_add_rel(object, operand)
case .acq_rel:
return intrinsics.atomic_add_acqrel(object, operand)
case .seq_cst:
return intrinsics.atomic_add(object, operand)
switch order {
case .relaxed: return intrinsics.atomic_add_explicit(object, operand, .Relaxed)
case .consume: return intrinsics.atomic_add_explicit(object, operand, .Consume)
case .acquire: return intrinsics.atomic_add_explicit(object, operand, .Acquire)
case .release: return intrinsics.atomic_add_explicit(object, operand, .Release)
case .acq_rel: return intrinsics.atomic_add_explicit(object, operand, .Acq_Rel)
case: fallthrough
case .seq_cst: return intrinsics.atomic_add_explicit(object, operand, .Seq_Cst)
}
}
@@ -318,19 +303,14 @@ atomic_fetch_sub :: #force_inline proc(object: ^$T, operand: T) -> T {
}
atomic_fetch_sub_explicit :: #force_inline proc(object: ^$T, operand: T, order: memory_order) -> T {
switch (order) {
case .relaxed:
return intrinsics.atomic_sub_relaxed(object, operand)
case .consume:
return intrinsics.atomic_sub_acq(object, operand)
case .acquire:
return intrinsics.atomic_sub_acq(object, operand)
case .release:
return intrinsics.atomic_sub_rel(object, operand)
case .acq_rel:
return intrinsics.atomic_sub_acqrel(object, operand)
case .seq_cst:
return intrinsics.atomic_sub(object, operand)
switch order {
case .relaxed: return intrinsics.atomic_sub_explicit(object, operand, .Relaxed)
case .consume: return intrinsics.atomic_sub_explicit(object, operand, .Consume)
case .acquire: return intrinsics.atomic_sub_explicit(object, operand, .Acquire)
case .release: return intrinsics.atomic_sub_explicit(object, operand, .Release)
case .acq_rel: return intrinsics.atomic_sub_explicit(object, operand, .Acq_Rel)
case: fallthrough
case .seq_cst: return intrinsics.atomic_sub_explicit(object, operand, .Seq_Cst)
}
}
@@ -339,19 +319,14 @@ atomic_fetch_or :: #force_inline proc(object: ^$T, operand: T) -> T {
}
atomic_fetch_or_explicit :: #force_inline proc(object: ^$T, operand: T, order: memory_order) -> T {
switch (order) {
case .relaxed:
return intrinsics.atomic_or_relaxed(object, operand)
case .consume:
return intrinsics.atomic_or_acq(object, operand)
case .acquire:
return intrinsics.atomic_or_acq(object, operand)
case .release:
return intrinsics.atomic_or_rel(object, operand)
case .acq_rel:
return intrinsics.atomic_or_acqrel(object, operand)
case .seq_cst:
return intrinsics.atomic_or(object, operand)
switch order {
case .relaxed: return intrinsics.atomic_or_explicit(object, operand, .Relaxed)
case .consume: return intrinsics.atomic_or_explicit(object, operand, .Consume)
case .acquire: return intrinsics.atomic_or_explicit(object, operand, .Acquire)
case .release: return intrinsics.atomic_or_explicit(object, operand, .Release)
case .acq_rel: return intrinsics.atomic_or_explicit(object, operand, .Acq_Rel)
case: fallthrough
case .seq_cst: return intrinsics.atomic_or_explicit(object, operand, .Seq_Cst)
}
}
@@ -360,19 +335,14 @@ atomic_fetch_xor :: #force_inline proc(object: ^$T, operand: T) -> T {
}
atomic_fetch_xor_explicit :: #force_inline proc(object: ^$T, operand: T, order: memory_order) -> T {
switch (order) {
case .relaxed:
return intrinsics.atomic_xor_relaxed(object, operand)
case .consume:
return intrinsics.atomic_xor_acq(object, operand)
case .acquire:
return intrinsics.atomic_xor_acq(object, operand)
case .release:
return intrinsics.atomic_xor_rel(object, operand)
case .acq_rel:
return intrinsics.atomic_xor_acqrel(object, operand)
case .seq_cst:
return intrinsics.atomic_xor(object, operand)
switch order {
case .relaxed: return intrinsics.atomic_xor_explicit(object, operand, .Relaxed)
case .consume: return intrinsics.atomic_xor_explicit(object, operand, .Consume)
case .acquire: return intrinsics.atomic_xor_explicit(object, operand, .Acquire)
case .release: return intrinsics.atomic_xor_explicit(object, operand, .Release)
case .acq_rel: return intrinsics.atomic_xor_explicit(object, operand, .Acq_Rel)
case: fallthrough
case .seq_cst: return intrinsics.atomic_xor_explicit(object, operand, .Seq_Cst)
}
}
@@ -380,19 +350,14 @@ atomic_fetch_and :: #force_inline proc(object: ^$T, operand: T) -> T {
return intrinsics.atomic_and(object, operand)
}
atomic_fetch_and_explicit :: #force_inline proc(object: ^$T, operand: T, order: memory_order) -> T {
switch (order) {
case .relaxed:
return intrinsics.atomic_and_relaxed(object, operand)
case .consume:
return intrinsics.atomic_and_acq(object, operand)
case .acquire:
return intrinsics.atomic_and_acq(object, operand)
case .release:
return intrinsics.atomic_and_rel(object, operand)
case .acq_rel:
return intrinsics.atomic_and_acqrel(object, operand)
case .seq_cst:
return intrinsics.atomic_and(object, operand)
switch order {
case .relaxed: return intrinsics.atomic_and_explicit(object, operand, .Relaxed)
case .consume: return intrinsics.atomic_and_explicit(object, operand, .Consume)
case .acquire: return intrinsics.atomic_and_explicit(object, operand, .Acquire)
case .release: return intrinsics.atomic_and_explicit(object, operand, .Release)
case .acq_rel: return intrinsics.atomic_and_explicit(object, operand, .Acq_Rel)
case: fallthrough
case .seq_cst: return intrinsics.atomic_and_explicit(object, operand, .Seq_Cst)
}
}

62
core/c/libc/stdio.odin
View File

@@ -1,8 +1,8 @@
package libc
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
@@ -13,7 +13,7 @@ when ODIN_OS == "windows" {
FILE :: struct {}
// MSVCRT compatible.
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
_IOFBF :: 0x0000
_IONBF :: 0x0004
_IOLBF :: 0x0040
@@ -48,7 +48,7 @@ when ODIN_OS == "windows" {
}
// GLIBC and MUSL compatible.
when ODIN_OS == "linux" {
when ODIN_OS == .Linux {
fpos_t :: struct #raw_union { _: [16]char, _: longlong, _: double, }
_IOFBF :: 0
@@ -78,7 +78,57 @@ when ODIN_OS == "linux" {
}
}
when ODIN_OS == "darwin" {
when ODIN_OS == .OpenBSD {
fpos_t :: distinct i64
_IOFBF :: 0
_IOLBF :: 1
_IONBF :: 1
BUFSIZ :: 1024
EOF :: int(-1)
FOPEN_MAX :: 20
FILENAME_MAX :: 1024
SEEK_SET :: 0
SEEK_CUR :: 1
SEEK_END :: 2
foreign libc {
stderr: ^FILE
stdin: ^FILE
stdout: ^FILE
}
}
when ODIN_OS == .FreeBSD {
fpos_t :: distinct i64
_IOFBF :: 0
_IOLBF :: 1
_IONBF :: 1
BUFSIZ :: 1024
EOF :: int(-1)
FOPEN_MAX :: 20
FILENAME_MAX :: 1024
SEEK_SET :: 0
SEEK_CUR :: 1
SEEK_END :: 2
foreign libc {
stderr: ^FILE
stdin: ^FILE
stdout: ^FILE
}
}
when ODIN_OS == .Darwin {
fpos_t :: distinct i64
_IOFBF :: 0
@@ -146,7 +196,7 @@ foreign libc {
getc :: proc(stream: ^FILE) -> int ---
getchar :: proc() -> int ---
putc :: proc(c: int, stream: ^FILE) -> int ---
putchar :: proc() -> int ---
putchar :: proc(c: int) -> int ---
puts :: proc(s: cstring) -> int ---
ungetc :: proc(c: int, stream: ^FILE) -> int ---
fread :: proc(ptr: rawptr, size: size_t, nmemb: size_t, stream: ^FILE) -> size_t ---

10
core/c/libc/stdlib.odin
View File

@@ -2,15 +2,15 @@ package libc
// 7.22 General utilities
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
}
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
RAND_MAX :: 0x7fff
@(private="file")
@@ -24,7 +24,7 @@ when ODIN_OS == "windows" {
}
}
when ODIN_OS == "linux" {
when ODIN_OS == .Linux {
RAND_MAX :: 0x7fffffff
// GLIBC and MUSL only
@@ -40,7 +40,7 @@ when ODIN_OS == "linux" {
}
when ODIN_OS == "darwin" {
when ODIN_OS == .Darwin {
RAND_MAX :: 0x7fffffff
// GLIBC and MUSL only

4
core/c/libc/string.odin
View File

@@ -4,9 +4,9 @@ import "core:runtime"
// 7.24 String handling
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"

10
core/c/libc/threads.odin
View File

@@ -5,10 +5,10 @@ package libc
thrd_start_t :: proc "c" (rawptr) -> int
tss_dtor_t :: proc "c" (rawptr)
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc {
"system:libucrt.lib",
"system:msvcprt.lib"
"system:msvcprt.lib",
}
thrd_success :: 0 // _Thrd_success
@@ -74,10 +74,10 @@ when ODIN_OS == "windows" {
}
// GLIBC and MUSL compatible constants and types.
when ODIN_OS == "linux" {
when ODIN_OS == .Linux {
foreign import libc {
"system:c",
"system:pthread"
"system:pthread",
}
thrd_success :: 0
@@ -138,6 +138,6 @@ when ODIN_OS == "linux" {
}
when ODIN_OS == "darwin" {
when ODIN_OS == .Darwin {
// TODO: find out what this is meant to be!
}

15
core/c/libc/time.odin
View File

@@ -2,9 +2,9 @@ package libc
// 7.27 Date and time
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
@@ -12,7 +12,7 @@ when ODIN_OS == "windows" {
// We enforce 64-bit time_t and timespec as there is no reason to use 32-bit as
// we approach the 2038 problem. Windows has defaulted to this since VC8 (2005).
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign libc {
// 7.27.2 Time manipulation functions
clock :: proc() -> clock_t ---
@@ -45,7 +45,7 @@ when ODIN_OS == "windows" {
}
}
when ODIN_OS == "linux" || ODIN_OS == "freebsd" || ODIN_OS == "darwin" {
when ODIN_OS == .Linux || ODIN_OS == .FreeBSD || ODIN_OS == .Darwin || ODIN_OS == .OpenBSD {
@(default_calling_convention="c")
foreign libc {
// 7.27.2 Time manipulation functions
@@ -63,7 +63,12 @@ when ODIN_OS == "linux" || ODIN_OS == "freebsd" || ODIN_OS == "darwin" {
strftime :: proc(s: [^]char, maxsize: size_t, format: cstring, timeptr: ^tm) -> size_t ---
}
CLOCKS_PER_SEC :: 1000000
when ODIN_OS == .OpenBSD {
CLOCKS_PER_SEC :: 100
} else {
CLOCKS_PER_SEC :: 1000000
}
TIME_UTC :: 1
time_t :: distinct i64

4
core/c/libc/uchar.odin
View File

@@ -2,9 +2,9 @@ package libc
// 7.28 Unicode utilities
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"

4
core/c/libc/wchar.odin
View File

@@ -2,9 +2,9 @@ package libc
// 7.29 Extended multibyte and wide character utilities
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"

21
core/c/libc/wctype.odin
View File

@@ -2,27 +2,34 @@ package libc
// 7.30 Wide character classification and mapping utilities
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
foreign import libc "system:libucrt.lib"
} else when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
foreign import libc "system:System.framework"
} else {
foreign import libc "system:c"
}
when ODIN_OS == "windows" {
when ODIN_OS == .Windows {
wctrans_t :: distinct wchar_t
wctype_t :: distinct ushort
}
when ODIN_OS == "linux" {
} else when ODIN_OS == .Linux {
wctrans_t :: distinct intptr_t
wctype_t :: distinct ulong
}
when ODIN_OS == "darwin" {
} else when ODIN_OS == .Darwin {
wctrans_t :: distinct int
wctype_t :: distinct u32
} else when ODIN_OS == .OpenBSD {
wctrans_t :: distinct rawptr
wctype_t :: distinct rawptr
} else when ODIN_OS == .FreeBSD {
wctrans_t :: distinct int
wctype_t :: distinct ulong
}
@(default_calling_convention="c")

19
core/compress/common.odin
View File

@@ -47,7 +47,7 @@ when size_of(uintptr) == 8 {
}
Error :: union {
Error :: union #shared_nil {
General_Error,
Deflate_Error,
ZLIB_Error,
@@ -58,6 +58,7 @@ Error :: union {
}
General_Error :: enum {
None = 0,
File_Not_Found,
Cannot_Open_File,
File_Too_Short,
@@ -76,6 +77,7 @@ General_Error :: enum {
}
GZIP_Error :: enum {
None = 0,
Invalid_GZIP_Signature,
Reserved_Flag_Set,
Invalid_Extra_Data,
@@ -100,6 +102,7 @@ GZIP_Error :: enum {
}
ZIP_Error :: enum {
None = 0,
Invalid_ZIP_File_Signature,
Unexpected_Signature,
Insert_Next_Disk,
@@ -107,6 +110,7 @@ ZIP_Error :: enum {
}
ZLIB_Error :: enum {
None = 0,
Unsupported_Window_Size,
FDICT_Unsupported,
Unsupported_Compression_Level,
@@ -114,6 +118,7 @@ ZLIB_Error :: enum {
}
Deflate_Error :: enum {
None = 0,
Huffman_Bad_Sizes,
Huffman_Bad_Code_Lengths,
Inflate_Error,
@@ -123,7 +128,6 @@ Deflate_Error :: enum {
BType_3,
}
// General I/O context for ZLIB, LZW, etc.
Context_Memory_Input :: struct #packed {
input_data: []u8,
@@ -139,7 +143,12 @@ Context_Memory_Input :: struct #packed {
size_packed: i64,
size_unpacked: i64,
}
#assert(size_of(Context_Memory_Input) == 64)
when size_of(rawptr) == 8 {
#assert(size_of(Context_Memory_Input) == 64)
} else {
// e.g. `-target:windows_i386`
#assert(size_of(Context_Memory_Input) == 52)
}
Context_Stream_Input :: struct #packed {
input_data: []u8,
@@ -174,8 +183,6 @@ Context_Stream_Input :: struct #packed {
This simplifies end-of-stream handling where bits may be left in the bit buffer.
*/
// TODO: Make these return compress.Error errors.
input_size_from_memory :: proc(z: ^Context_Memory_Input) -> (res: i64, err: Error) {
return i64(len(z.input_data)), nil
}
@@ -473,4 +480,4 @@ discard_to_next_byte_lsb_from_stream :: proc(z: ^Context_Stream_Input) {
consume_bits_lsb(z, discard)
}
discard_to_next_byte_lsb :: proc{discard_to_next_byte_lsb_from_memory, discard_to_next_byte_lsb_from_stream};
discard_to_next_byte_lsb :: proc{discard_to_next_byte_lsb_from_memory, discard_to_next_byte_lsb_from_stream}

2
core/compress/gzip/example.odin
View File

@@ -45,7 +45,7 @@ main :: proc() {
if len(args) < 2 {
stderr("No input file specified.\n")
err := load(slice=TEST, buf=&buf, known_gzip_size=len(TEST))
err := load(data=TEST, buf=&buf, known_gzip_size=len(TEST))
if err == nil {
stdout("Displaying test vector: ")
stdout(bytes.buffer_to_string(&buf))

15
core/compress/gzip/gzip.odin
View File

@@ -66,7 +66,8 @@ OS :: enum u8 {
_Unknown = 14,
Unknown = 255,
}
OS_Name :: #partial [OS]string{
OS_Name :: #sparse[OS]string{
._Unknown = "",
.FAT = "FAT",
.Amiga = "Amiga",
.VMS = "VMS/OpenVMS",
@@ -99,9 +100,9 @@ E_GZIP :: compress.GZIP_Error
E_ZLIB :: compress.ZLIB_Error
E_Deflate :: compress.Deflate_Error
GZIP_MAX_PAYLOAD_SIZE :: int(max(u32le))
GZIP_MAX_PAYLOAD_SIZE :: i64(max(u32le))
load :: proc{load_from_slice, load_from_file, load_from_context}
load :: proc{load_from_bytes, load_from_file, load_from_context}
load_from_file :: proc(filename: string, buf: ^bytes.Buffer, expected_output_size := -1, allocator := context.allocator) -> (err: Error) {
context.allocator = allocator
@@ -111,16 +112,16 @@ load_from_file :: proc(filename: string, buf: ^bytes.Buffer, expected_output_siz
err = E_General.File_Not_Found
if ok {
err = load_from_slice(data, buf, len(data), expected_output_size)
err = load_from_bytes(data, buf, len(data), expected_output_size)
}
return
}
load_from_slice :: proc(slice: []u8, buf: ^bytes.Buffer, known_gzip_size := -1, expected_output_size := -1, allocator := context.allocator) -> (err: Error) {
load_from_bytes :: proc(data: []byte, buf: ^bytes.Buffer, known_gzip_size := -1, expected_output_size := -1, allocator := context.allocator) -> (err: Error) {
buf := buf
z := &compress.Context_Memory_Input{
input_data = slice,
input_data = data,
output = buf,
}
return load_from_context(z, buf, known_gzip_size, expected_output_size, allocator)
@@ -135,7 +136,7 @@ load_from_context :: proc(z: ^$C, buf: ^bytes.Buffer, known_gzip_size := -1, exp
z.output = buf
if expected_output_size > GZIP_MAX_PAYLOAD_SIZE {
if i64(expected_output_size) > i64(GZIP_MAX_PAYLOAD_SIZE) {
return E_GZIP.Payload_Size_Exceeds_Max_Payload
}

148
core/compress/shoco/model.odin Normal file
View File

@@ -0,0 +1,148 @@
/*
This file was generated, so don't edit this by hand.
Transliterated from https://github.com/Ed-von-Schleck/shoco/blob/master/shoco_model.h,
which is an English word model.
*/
// package shoco is an implementation of the shoco short string compressor
package shoco
DEFAULT_MODEL :: Shoco_Model {
min_char = 39,
max_char = 122,
characters_by_id = {
'e', 'a', 'i', 'o', 't', 'h', 'n', 'r', 's', 'l', 'u', 'c', 'w', 'm', 'd', 'b', 'p', 'f', 'g', 'v', 'y', 'k', '-', 'H', 'M', 'T', '\'', 'B', 'x', 'I', 'W', 'L',
},
ids_by_character = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 26, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, -1, -1, -1, -1, -1, 23, 29, -1, -1, 31, 24, -1, -1, -1, -1, -1, -1, 25, -1, -1, 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 15, 11, 14, 0, 17, 18, 5, 2, -1, 21, 9, 13, 6, 3, 16, -1, 7, 8, 4, 10, 19, 12, 28, 20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
},
successors_by_bigram = {
7, 4, 12, -1, 6, -1, 1, 0, 3, 5, -1, 9, -1, 8, 2, -1, 15, 14, -1, 10, 11, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1,
1, -1, 6, -1, 1, -1, 0, 3, 2, 4, 15, 11, -1, 9, 5, 10, 13, -1, 12, 8, 7, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
9, 11, -1, 4, 2, -1, 0, 8, 1, 5, -1, 6, -1, 3, 7, 15, -1, 12, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, 14, 7, 5, -1, 1, 2, 8, 9, 0, 15, 6, 4, 11, -1, 12, 3, -1, 10, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
2, 4, 3, 1, 5, 0, -1, 6, 10, 9, 7, 12, 11, -1, -1, -1, -1, 13, -1, -1, 8, -1, 15, -1, -1, -1, 14, -1, -1, -1, -1, -1,
0, 1, 2, 3, 4, -1, -1, 5, 9, 10, 6, -1, -1, 8, 15, 11, -1, 14, -1, -1, 7, -1, 13, -1, -1, -1, 12, -1, -1, -1, -1, -1,
2, 8, 7, 4, 3, -1, 9, -1, 6, 11, -1, 5, -1, -1, 0, -1, -1, 14, 1, 15, 10, 12, -1, -1, -1, -1, 13, -1, -1, -1, -1, -1,
0, 3, 1, 2, 6, -1, 9, 8, 4, 12, 13, 10, -1, 11, 7, -1, -1, 15, 14, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 6, 3, 4, 1, 2, -1, -1, 5, 10, 7, 9, 11, 12, -1, -1, 8, 14, -1, -1, 15, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 6, 2, 5, 9, -1, -1, -1, 10, 1, 8, -1, 12, 14, 4, -1, 15, 7, -1, 13, 3, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
8, 10, 9, 15, 1, -1, 4, 0, 3, 2, -1, 6, -1, 12, 11, 13, 7, 14, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
1, 3, 6, 0, 4, 2, -1, 7, 13, 8, 9, 11, -1, -1, 15, -1, -1, -1, -1, -1, 10, 5, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1,
3, 0, 1, 4, -1, 2, 5, 6, 7, 8, -1, 14, -1, -1, 9, 15, -1, 12, -1, -1, -1, 10, 11, -1, -1, -1, 13, -1, -1, -1, -1, -1,
0, 1, 3, 2, 15, -1, 12, -1, 7, 14, 4, -1, -1, 9, -1, 8, 5, 10, -1, -1, 6, -1, 13, -1, -1, -1, 11, -1, -1, -1, -1, -1,
0, 3, 1, 2, -1, -1, 12, 6, 4, 9, 7, -1, -1, 14, 8, -1, -1, 15, 11, 13, 5, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 5, 7, 2, 10, 13, -1, 6, 8, 1, 3, -1, -1, 14, 15, 11, -1, -1, -1, 12, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 2, 6, 3, 7, 10, -1, 1, 9, 4, 8, -1, -1, 15, -1, 12, 5, -1, -1, -1, 11, -1, 13, -1, -1, -1, 14, -1, -1, -1, -1, -1,
1, 3, 4, 0, 7, -1, 12, 2, 11, 8, 6, 13, -1, -1, -1, -1, -1, 5, -1, -1, 10, 15, 9, -1, -1, -1, 14, -1, -1, -1, -1, -1,
1, 3, 5, 2, 13, 0, 9, 4, 7, 6, 8, -1, -1, 15, -1, 11, -1, -1, 10, -1, 14, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 2, 1, 3, -1, -1, -1, 6, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
1, 11, 4, 0, 3, -1, 13, 12, 2, 7, -1, -1, 15, 10, 5, 8, 14, -1, -1, -1, -1, -1, 9, -1, -1, -1, 6, -1, -1, -1, -1, -1,
0, 9, 2, 14, 15, 4, 1, 13, 3, 5, -1, -1, 10, -1, -1, -1, -1, 6, 12, -1, 7, -1, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1,
-1, 2, 14, -1, 1, 5, 8, 7, 4, 12, -1, 6, 9, 11, 13, 3, 10, 15, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 1, 3, 2, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
4, 3, 1, 5, -1, -1, -1, 0, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
2, 8, 4, 1, -1, 0, -1, 6, -1, -1, 5, -1, 7, -1, -1, -1, -1, -1, -1, -1, 10, -1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1,
12, 5, -1, -1, 1, -1, -1, 7, 0, 3, -1, 2, -1, 4, 6, -1, -1, -1, -1, 8, -1, -1, 15, -1, 13, 9, -1, -1, -1, -1, -1, 11,
1, 3, 2, 4, -1, -1, -1, 5, -1, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, -1,
5, 3, 4, 12, 1, 6, -1, -1, -1, -1, 8, 2, -1, -1, -1, -1, 0, 9, -1, -1, 11, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, 0, -1, 1, 12, 3, -1, -1, -1, -1, 5, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, 6, -1, 10,
2, 3, 1, 4, -1, 0, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 7, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1,
5, 1, 3, 0, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, 9, -1, -1, 6, -1, 7,
},
successors_reversed = {
's', 't', 'c', 'l', 'm', 'a', 'd', 'r', 'v', 'T', 'A', 'L', 'e', 'M', 'Y', '-',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'-', 't', 'a', 'b', 's', 'h', 'c', 'r', 'n', 'w', 'p', 'm', 'l', 'd', 'i', 'f',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'u', 'e', 'i', 'a', 'o', 'r', 'y', 'l', 'I', 'E', 'R', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'e', 'a', 'o', 'i', 'u', 'A', 'y', 'E', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
't', 'n', 'f', 's', '\'', 'm', 'I', 'N', 'A', 'E', 'L', 'Z', 'r', 'V', 'R', 'C',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'o', 'a', 'y', 'i', 'u', 'e', 'I', 'L', 'D', '\'', 'E', 'Y', '\x00', '\x00', '\x00', '\x00',
'r', 'i', 'y', 'a', 'e', 'o', 'u', 'Y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'h', 'o', 'e', 'E', 'i', 'u', 'r', 'w', 'a', 'H', 'y', 'R', 'Z', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'h', 'i', 'e', 'a', 'o', 'r', 'I', 'y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'n', 't', 's', 'r', 'l', 'd', 'i', 'y', 'v', 'm', 'b', 'c', 'g', 'p', 'k', 'u',
'e', 'l', 'o', 'u', 'y', 'a', 'r', 'i', 's', 'j', 't', 'b', 'v', 'h', 'm', 'd',
'o', 'e', 'h', 'a', 't', 'k', 'i', 'r', 'l', 'u', 'y', 'c', 'q', 's', '-', 'd',
'e', 'i', 'o', 'a', 's', 'y', 'r', 'u', 'd', 'l', '-', 'g', 'n', 'v', 'm', 'f',
'r', 'n', 'd', 's', 'a', 'l', 't', 'e', 'm', 'c', 'v', 'y', 'i', 'x', 'f', 'p',
'o', 'e', 'r', 'a', 'i', 'f', 'u', 't', 'l', '-', 'y', 's', 'n', 'c', '\'', 'k',
'h', 'e', 'o', 'a', 'r', 'i', 'l', 's', 'u', 'n', 'g', 'b', '-', 't', 'y', 'm',
'e', 'a', 'i', 'o', 't', 'r', 'u', 'y', 'm', 's', 'l', 'b', '\'', '-', 'f', 'd',
'n', 's', 't', 'm', 'o', 'l', 'c', 'd', 'r', 'e', 'g', 'a', 'f', 'v', 'z', 'b',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'e', 'n', 'i', 's', 'h', 'l', 'f', 'y', '-', 'a', 'w', '\'', 'g', 'r', 'o', 't',
'e', 'l', 'i', 'y', 'd', 'o', 'a', 'f', 'u', 't', 's', 'k', 'w', 'v', 'm', 'p',
'e', 'a', 'o', 'i', 'u', 'p', 'y', 's', 'b', 'm', 'f', '\'', 'n', '-', 'l', 't',
'd', 'g', 'e', 't', 'o', 'c', 's', 'i', 'a', 'n', 'y', 'l', 'k', '\'', 'f', 'v',
'u', 'n', 'r', 'f', 'm', 't', 'w', 'o', 's', 'l', 'v', 'd', 'p', 'k', 'i', 'c',
'e', 'r', 'a', 'o', 'l', 'p', 'i', 't', 'u', 's', 'h', 'y', 'b', '-', '\'', 'm',
'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'e', 'i', 'o', 'a', 's', 'y', 't', 'd', 'r', 'n', 'c', 'm', 'l', 'u', 'g', 'f',
'e', 't', 'h', 'i', 'o', 's', 'a', 'u', 'p', 'c', 'l', 'w', 'm', 'k', 'f', 'y',
'h', 'o', 'e', 'i', 'a', 't', 'r', 'u', 'y', 'l', 's', 'w', 'c', 'f', '\'', '-',
'r', 't', 'l', 's', 'n', 'g', 'c', 'p', 'e', 'i', 'a', 'd', 'm', 'b', 'f', 'o',
'e', 'i', 'a', 'o', 'y', 'u', 'r', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
'a', 'i', 'h', 'e', 'o', 'n', 'r', 's', 'l', 'd', 'k', '-', 'f', '\'', 'c', 'b',
'p', 't', 'c', 'a', 'i', 'e', 'h', 'q', 'u', 'f', '-', 'y', 'o', '\x00', '\x00', '\x00',
'o', 'e', 's', 't', 'i', 'd', '\'', 'l', 'b', '-', 'm', 'a', 'r', 'n', 'p', 'w',
},
character_count = 32,
successor_count = 16,
max_successor_n = 7,
packs = {
{ 0x80000000, 1, 2, { 26, 24, 24, 24, 24, 24, 24, 24 }, { 15, 3, 0, 0, 0, 0, 0, 0 }, 0xc0, 0x80 },
{ 0xc0000000, 2, 4, { 25, 22, 19, 16, 16, 16, 16, 16 }, { 15, 7, 7, 7, 0, 0, 0, 0 }, 0xe0, 0xc0 },
{ 0xe0000000, 4, 8, { 23, 19, 15, 11, 8, 5, 2, 0 }, { 31, 15, 15, 15, 7, 7, 7, 3 }, 0xf0, 0xe0 },
},
}

318
core/compress/shoco/shoco.odin Normal file
View File

@@ -0,0 +1,318 @@
/*
Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
List of contributors:
Jeroen van Rijn: Initial implementation.
An implementation of [shoco](https://github.com/Ed-von-Schleck/shoco) by Christian Schramm.
*/
// package shoco is an implementation of the shoco short string compressor
package shoco
import "core:intrinsics"
import "core:compress"
Shoco_Pack :: struct {
word: u32,
bytes_packed: i8,
bytes_unpacked: i8,
offsets: [8]u16,
masks: [8]i16,
header_mask: u8,
header: u8,
}
Shoco_Model :: struct {
min_char: u8,
max_char: u8,
characters_by_id: []u8,
ids_by_character: [256]i16,
successors_by_bigram: []i8,
successors_reversed: []u8,
character_count: u8,
successor_count: u8,
max_successor_n: i8,
packs: []Shoco_Pack,
}
compress_bound :: proc(uncompressed_size: int) -> (worst_case_compressed_size: int) {
// Worst case compression happens when input is non-ASCII (128-255)
// Encoded as 0x00 + the byte in question.
return uncompressed_size * 2
}
decompress_bound :: proc(compressed_size: int, model := DEFAULT_MODEL) -> (maximum_decompressed_size: int) {
// Best case compression is 2:1
most: f64
for pack in model.packs {
val := f64(compressed_size) / f64(pack.bytes_packed) * f64(pack.bytes_unpacked)
most = max(most, val)
}
return int(most)
}
find_best_encoding :: proc(indices: []i16, n_consecutive: i8, model := DEFAULT_MODEL) -> (res: int) {
for p := len(model.packs); p > 0; p -= 1 {
pack := model.packs[p - 1]
if n_consecutive >= pack.bytes_unpacked {
have_index := true
for i := 0; i < int(pack.bytes_unpacked); i += 1 {
if indices[i] > pack.masks[i] {
have_index = false
break
}
}
if have_index {
return p - 1
}
}
}
return -1
}
validate_model :: proc(model: Shoco_Model) -> (int, compress.Error) {
if len(model.characters_by_id) != int(model.character_count) {
return 0, .Unknown_Compression_Method
}
if len(model.successors_by_bigram) != int(model.character_count) * int(model.character_count) {
return 0, .Unknown_Compression_Method
}
if len(model.successors_reversed) != int(model.successor_count) * int(model.max_char - model.min_char) {
return 0, .Unknown_Compression_Method
}
// Model seems legit.
return 0, nil
}
// Decompresses into provided buffer.
decompress_slice_to_output_buffer :: proc(input: []u8, output: []u8, model := DEFAULT_MODEL) -> (size: int, err: compress.Error) {
inp, inp_end := 0, len(input)
out, out_end := 0, len(output)
validate_model(model) or_return
for inp < inp_end {
val := transmute(i8)input[inp]
mark := int(-1)
for val < 0 {
val <<= 1
mark += 1
}
if mark > len(model.packs) {
return out, .Unknown_Compression_Method
}
if mark < 0 {
if out >= out_end {
return out, .Output_Too_Short
}
// Ignore the sentinel value for non-ASCII chars
if input[inp] == 0x00 {
inp += 1
if inp >= inp_end {
return out, .Stream_Too_Short
}
}
output[out] = input[inp]
inp, out = inp + 1, out + 1
} else {
pack := model.packs[mark]
if out + int(pack.bytes_unpacked) > out_end {
return out, .Output_Too_Short
} else if inp + int(pack.bytes_packed) > inp_end {
return out, .Stream_Too_Short
}
code := intrinsics.unaligned_load((^u32)(&input[inp]))
when ODIN_ENDIAN == .Little {
code = intrinsics.byte_swap(code)
}
// Unpack the leading char
offset := pack.offsets[0]
mask := pack.masks[0]
last_chr := model.characters_by_id[(code >> offset) & u32(mask)]
output[out] = last_chr
// Unpack the successor chars
for i := 1; i < int(pack.bytes_unpacked); i += 1 {
offset = pack.offsets[i]
mask = pack.masks[i]
index_major := u32(last_chr - model.min_char) * u32(model.successor_count)
index_minor := (code >> offset) & u32(mask)
last_chr = model.successors_reversed[index_major + index_minor]
output[out + i] = last_chr
}
out += int(pack.bytes_unpacked)
inp += int(pack.bytes_packed)
}
}
return out, nil
}
decompress_slice_to_string :: proc(input: []u8, model := DEFAULT_MODEL, allocator := context.allocator) -> (res: string, err: compress.Error) {
context.allocator = allocator
if len(input) == 0 {
return "", .Stream_Too_Short
}
max_output_size := decompress_bound(len(input), model)
buf: [dynamic]u8
if !resize(&buf, max_output_size) {
return "", .Out_Of_Memory
}
length, result := decompress_slice_to_output_buffer(input, buf[:])
resize(&buf, length)
return string(buf[:]), result
}
decompress :: proc{decompress_slice_to_output_buffer, decompress_slice_to_string}
compress_string_to_buffer :: proc(input: string, output: []u8, model := DEFAULT_MODEL, allocator := context.allocator) -> (size: int, err: compress.Error) {
inp, inp_end := 0, len(input)
out, out_end := 0, len(output)
output := output
validate_model(model) or_return
indices := make([]i16, model.max_successor_n + 1)
defer delete(indices)
last_resort := false
encode: for inp < inp_end {
if last_resort {
last_resort = false
if input[inp] & 0x80 == 0x80 {
// Non-ASCII case
if out + 2 > out_end {
return out, .Output_Too_Short
}
// Put in a sentinel byte
output[out] = 0x00
out += 1
} else {
// An ASCII byte
if out + 1 > out_end {
return out, .Output_Too_Short
}
}
output[out] = input[inp]
out, inp = out + 1, inp + 1
} else {
// Find the longest string of known successors
indices[0] = model.ids_by_character[input[inp]]
last_chr_index := indices[0]
if last_chr_index < 0 {
last_resort = true
continue encode
}
rest := inp_end - inp
n_consecutive: i8 = 1
for ; n_consecutive <= model.max_successor_n; n_consecutive += 1 {
if inp_end > 0 && int(n_consecutive) == rest {
break
}
current_index := model.ids_by_character[input[inp + int(n_consecutive)]]
if current_index < 0 { // '\0' is always -1
break
}
successor_index := model.successors_by_bigram[last_chr_index * i16(model.character_count) + current_index]
if successor_index < 0 {
break
}
indices[n_consecutive] = i16(successor_index)
last_chr_index = current_index
}
if n_consecutive < 2 {
last_resort = true
continue encode
}
pack_n := find_best_encoding(indices, n_consecutive)
if pack_n >= 0 {
if out + int(model.packs[pack_n].bytes_packed) > out_end {
return out, .Output_Too_Short
}
pack := model.packs[pack_n]
code := pack.word
for i := 0; i < int(pack.bytes_unpacked); i += 1 {
code |= u32(indices[i]) << pack.offsets[i]
}
// In the little-endian world, we need to swap what's in the register to match the memory representation.
when ODIN_ENDIAN == .Little {
code = intrinsics.byte_swap(code)
}
out_ptr := raw_data(output[out:])
switch pack.bytes_packed {
case 4:
intrinsics.unaligned_store(transmute(^u32)out_ptr, code)
case 2:
intrinsics.unaligned_store(transmute(^u16)out_ptr, u16(code))
case 1:
intrinsics.unaligned_store(transmute(^u8)out_ptr, u8(code))
case:
return out, .Unknown_Compression_Method
}
out += int(pack.bytes_packed)
inp += int(pack.bytes_unpacked)
} else {
last_resort = true
continue encode
}
}
}
return out, nil
}
compress_string :: proc(input: string, model := DEFAULT_MODEL, allocator := context.allocator) -> (output: []u8, err: compress.Error) {
context.allocator = allocator
if len(input) == 0 {
return {}, .Stream_Too_Short
}
max_output_size := compress_bound(len(input))
buf: [dynamic]u8
if !resize(&buf, max_output_size) {
return {}, .Out_Of_Memory
}
length, result := compress_string_to_buffer(input, buf[:])
resize(&buf, length)
return buf[:length], result
}
compress :: proc{compress_string_to_buffer, compress_string}

83
core/compress/zlib/zlib.odin
View File

@@ -47,10 +47,10 @@ Options :: struct {
level: u8,
}
Error :: compress.Error
E_General :: compress.General_Error
E_ZLIB :: compress.ZLIB_Error
E_Deflate :: compress.Deflate_Error
Error :: compress.Error
General_Error :: compress.General_Error
ZLIB_Error :: compress.ZLIB_Error
Deflate_Error :: compress.Deflate_Error
DEFLATE_MAX_CHUNK_SIZE :: 65535
DEFLATE_MAX_LITERAL_SIZE :: 65535
@@ -111,9 +111,9 @@ ZFAST_MASK :: ((1 << ZFAST_BITS) - 1)
*/
Huffman_Table :: struct {
fast: [1 << ZFAST_BITS]u16,
firstcode: [16]u16,
firstcode: [17]u16,
maxcode: [17]int,
firstsymbol: [16]u16,
firstsymbol: [17]u16,
size: [288]u8,
value: [288]u16,
}
@@ -244,7 +244,7 @@ allocate_huffman_table :: proc(allocator := context.allocator) -> (z: ^Huffman_T
@(optimization_mode="speed")
build_huffman :: proc(z: ^Huffman_Table, code_lengths: []u8) -> (err: Error) {
sizes: [HUFFMAN_MAX_BITS+1]int
next_code: [HUFFMAN_MAX_BITS]int
next_code: [HUFFMAN_MAX_BITS+1]int
k := int(0)
@@ -256,21 +256,21 @@ build_huffman :: proc(z: ^Huffman_Table, code_lengths: []u8) -> (err: Error) {
}
sizes[0] = 0
for i in 1..<(HUFFMAN_MAX_BITS+1) {
for i in 1 ..< HUFFMAN_MAX_BITS {
if sizes[i] > (1 << uint(i)) {
return E_Deflate.Huffman_Bad_Sizes
return .Huffman_Bad_Sizes
}
}
code := int(0)
for i in 1..<HUFFMAN_MAX_BITS {
for i in 1 ..= HUFFMAN_MAX_BITS {
next_code[i] = code
z.firstcode[i] = u16(code)
z.firstsymbol[i] = u16(k)
code = code + sizes[i]
if sizes[i] != 0 {
if code - 1 >= (1 << u16(i)) {
return E_Deflate.Huffman_Bad_Code_Lengths
return .Huffman_Bad_Code_Lengths
}
}
z.maxcode[i] = code << (HUFFMAN_MAX_BITS - uint(i))
@@ -314,15 +314,15 @@ decode_huffman_slowpath :: proc(z: ^$C, t: ^Huffman_Table) -> (r: u16, err: Erro
s += 1
}
if s >= 16 {
return 0, E_Deflate.Bad_Huffman_Code
return 0, .Bad_Huffman_Code
}
// code size is s, so:
b := (k >> (16-s)) - int(t.firstcode[s]) + int(t.firstsymbol[s])
if b >= size_of(t.size) {
return 0, E_Deflate.Bad_Huffman_Code
return 0, .Bad_Huffman_Code
}
if t.size[b] != s {
return 0, E_Deflate.Bad_Huffman_Code
return 0, .Bad_Huffman_Code
}
compress.consume_bits_lsb(z, s)
@@ -335,11 +335,11 @@ decode_huffman_slowpath :: proc(z: ^$C, t: ^Huffman_Table) -> (r: u16, err: Erro
decode_huffman :: proc(z: ^$C, t: ^Huffman_Table) -> (r: u16, err: Error) #no_bounds_check {
if z.num_bits < 16 {
if z.num_bits > 63 {
return 0, E_ZLIB.Code_Buffer_Malformed
return 0, .Code_Buffer_Malformed
}
compress.refill_lsb(z)
if z.num_bits > 63 {
return 0, E_General.Stream_Too_Short
return 0, .Stream_Too_Short
}
}
#no_bounds_check b := t.fast[z.code_buffer & ZFAST_MASK]
@@ -361,7 +361,7 @@ parse_huffman_block :: proc(z: ^$C, z_repeat, z_offset: ^Huffman_Table) -> (err:
if value < 256 {
e := write_byte(z, u8(value))
if e != .None {
return E_General.Output_Too_Short
return .Output_Too_Short
}
} else {
if value == 256 {
@@ -377,7 +377,7 @@ parse_huffman_block :: proc(z: ^$C, z_repeat, z_offset: ^Huffman_Table) -> (err:
value, e = decode_huffman(z, z_offset)
if e != nil {
return E_Deflate.Bad_Huffman_Code
return .Bad_Huffman_Code
}
distance := Z_DIST_BASE[value]
@@ -387,7 +387,7 @@ parse_huffman_block :: proc(z: ^$C, z_repeat, z_offset: ^Huffman_Table) -> (err:
if z.bytes_written < i64(distance) {
// Distance is longer than we've decoded so far.
return E_Deflate.Bad_Distance
return .Bad_Distance
}
/*
@@ -405,14 +405,14 @@ parse_huffman_block :: proc(z: ^$C, z_repeat, z_offset: ^Huffman_Table) -> (err:
c := z.output.buf[z.bytes_written - i64(distance)]
e := repl_byte(z, length, c)
if e != .None {
return E_General.Output_Too_Short
return .Output_Too_Short
}
}
} else {
if length > 0 {
e := repl_bytes(z, length, distance)
if e != .None {
return E_General.Output_Too_Short
return .Output_Too_Short
}
}
}
@@ -432,25 +432,25 @@ inflate_from_context :: proc(using ctx: ^compress.Context_Memory_Input, raw := f
if !raw {
size, size_err := compress.input_size(ctx)
if size < 6 || size_err != nil {
return E_General.Stream_Too_Short
return .Stream_Too_Short
}
cmf, _ := compress.read_u8(ctx)
method := Compression_Method(cmf & 0xf)
if method != .DEFLATE {
return E_General.Unknown_Compression_Method
return .Unknown_Compression_Method
}
if cinfo := (cmf >> 4) & 0xf; cinfo > 7 {
return E_ZLIB.Unsupported_Window_Size
return .Unsupported_Window_Size
}
flg, _ := compress.read_u8(ctx)
fcheck := flg & 0x1f
fcheck_computed := (cmf << 8 | flg) & 0x1f
if fcheck != fcheck_computed {
return E_General.Checksum_Failed
return .Checksum_Failed
}
/*
@@ -458,7 +458,7 @@ inflate_from_context :: proc(using ctx: ^compress.Context_Memory_Input, raw := f
They're application specific and PNG doesn't use them.
*/
if fdict := (flg >> 5) & 1; fdict != 0 {
return E_ZLIB.FDICT_Unsupported
return .FDICT_Unsupported
}
// flevel := Compression_Level((flg >> 6) & 3);
@@ -485,7 +485,7 @@ inflate_from_context :: proc(using ctx: ^compress.Context_Memory_Input, raw := f
output_hash := hash.adler32(ctx.output.buf[:])
if output_hash != u32(adler) {
return E_General.Checksum_Failed
return .Checksum_Failed
}
}
return nil
@@ -538,23 +538,24 @@ inflate_raw :: proc(z: ^$C, expected_output_size := -1, allocator := context.all
final = compress.read_bits_lsb(z, 1)
type = compress.read_bits_lsb(z, 2)
// fmt.printf("Final: %v | Type: %v\n", final, type);
// fmt.printf("Final: %v | Type: %v\n", final, type)
switch type {
case 0:
// fmt.printf("Method 0: STORED\n")
// Uncompressed block
// Discard bits until next byte boundary
compress.discard_to_next_byte_lsb(z)
uncompressed_len := i16(compress.read_bits_lsb(z, 16))
length_check := i16(compress.read_bits_lsb(z, 16))
uncompressed_len := u16(compress.read_bits_lsb(z, 16))
length_check := u16(compress.read_bits_lsb(z, 16))
// fmt.printf("LEN: %v, ~LEN: %v, NLEN: %v, ~NLEN: %v\n", uncompressed_len, ~uncompressed_len, length_check, ~length_check);
// fmt.printf("LEN: %v, ~LEN: %v, NLEN: %v, ~NLEN: %v\n", uncompressed_len, ~uncompressed_len, length_check, ~length_check)
if ~uncompressed_len != length_check {
return E_Deflate.Len_Nlen_Mismatch
return .Len_Nlen_Mismatch
}
/*
@@ -567,10 +568,12 @@ inflate_raw :: proc(z: ^$C, expected_output_size := -1, allocator := context.all
write_byte(z, u8(lit))
uncompressed_len -= 1
}
assert(uncompressed_len == 0)
case 3:
return E_Deflate.BType_3
return .BType_3
case:
// log.debugf("Err: %v | Final: %v | Type: %v\n", err, final, type);
// fmt.printf("Err: %v | Final: %v | Type: %v\n", err, final, type)
if type == 1 {
// Use fixed code lengths.
build_huffman(z_repeat, Z_FIXED_LENGTH[:]) or_return
@@ -601,7 +604,7 @@ inflate_raw :: proc(z: ^$C, expected_output_size := -1, allocator := context.all
c = decode_huffman(z, codelength_ht) or_return
if c < 0 || c >= 19 {
return E_Deflate.Huffman_Bad_Code_Lengths
return .Huffman_Bad_Code_Lengths
}
if c < 16 {
lencodes[n] = u8(c)
@@ -613,7 +616,7 @@ inflate_raw :: proc(z: ^$C, expected_output_size := -1, allocator := context.all
case 16:
c = u16(compress.read_bits_no_refill_lsb(z, 2) + 3)
if n == 0 {
return E_Deflate.Huffman_Bad_Code_Lengths
return .Huffman_Bad_Code_Lengths
}
fill = lencodes[n - 1]
case 17:
@@ -621,11 +624,11 @@ inflate_raw :: proc(z: ^$C, expected_output_size := -1, allocator := context.all
case 18:
c = u16(compress.read_bits_no_refill_lsb(z, 7) + 11)
case:
return E_Deflate.Huffman_Bad_Code_Lengths
return .Huffman_Bad_Code_Lengths
}
if ntot - n < u32(c) {
return E_Deflate.Huffman_Bad_Code_Lengths
return .Huffman_Bad_Code_Lengths
}
nc := n + u32(c)
@@ -636,7 +639,7 @@ inflate_raw :: proc(z: ^$C, expected_output_size := -1, allocator := context.all
}
if n != ntot {
return E_Deflate.Huffman_Bad_Code_Lengths
return .Huffman_Bad_Code_Lengths
}
build_huffman(z_repeat, lencodes[:hlit]) or_return
@@ -674,4 +677,4 @@ inflate_from_byte_array_raw :: proc(input: []u8, buf: ^bytes.Buffer, raw := fals
return inflate_raw(z=&ctx, expected_output_size=expected_output_size)
}
inflate :: proc{inflate_from_context, inflate_from_byte_array};
inflate :: proc{inflate_from_context, inflate_from_byte_array}

160
core/container/bit_array/bit_array.odin
View File

@@ -1,6 +1,7 @@
package dynamic_bit_array
import "core:intrinsics"
import "core:mem"
/*
Note that these constants are dependent on the backing being a u64.
@@ -11,11 +12,120 @@ INDEX_SHIFT :: 6
@(private="file")
INDEX_MASK :: 63
@(private="file")
NUM_BITS :: 64
Bit_Array :: struct {
bits: [dynamic]u64,
bias: int,
bits: [dynamic]u64,
bias: int,
max_index: int,
free_pointer: bool,
}
Bit_Array_Iterator :: struct {
array: ^Bit_Array,
word_idx: int,
bit_idx: uint,
}
/*
In:
- ba: ^Bit_Array - the array to iterate over
Out:
- it: ^Bit_Array_Iterator - the iterator that holds iteration state
*/
make_iterator :: proc (ba: ^Bit_Array) -> (it: Bit_Array_Iterator) {
return Bit_Array_Iterator { array = ba }
}
/*
In:
- it: ^Bit_Array_Iterator - the iterator struct that holds the state.
Out:
- set: bool - the state of the bit at `index`
- index: int - the next bit of the Bit_Array referenced by `it`.
- ok: bool - `true` if the iterator returned a valid index,
`false` if there were no more bits
*/
iterate_by_all :: proc (it: ^Bit_Array_Iterator) -> (set: bool, index: int, ok: bool) {
index = it.word_idx * NUM_BITS + int(it.bit_idx) + it.array.bias
if index > it.array.max_index { return false, 0, false }
word := it.array.bits[it.word_idx] if len(it.array.bits) > it.word_idx else 0
set = (word >> it.bit_idx & 1) == 1
it.bit_idx += 1
if it.bit_idx >= NUM_BITS {
it.bit_idx = 0
it.word_idx += 1
}
return set, index, true
}
/*
In:
- it: ^Bit_Array_Iterator - the iterator struct that holds the state.
Out:
- index: int - the next set bit of the Bit_Array referenced by `it`.
- ok: bool - `true` if the iterator returned a valid index,
`false` if there were no more bits set
*/
iterate_by_set :: proc (it: ^Bit_Array_Iterator) -> (index: int, ok: bool) {
return iterate_internal_(it, true)
}
/*
In:
- it: ^Bit_Array_Iterator - the iterator struct that holds the state.
Out:
- index: int - the next unset bit of the Bit_Array referenced by `it`.
- ok: bool - `true` if the iterator returned a valid index,
`false` if there were no more unset bits
*/
iterate_by_unset:: proc (it: ^Bit_Array_Iterator) -> (index: int, ok: bool) {
return iterate_internal_(it, false)
}
@(private="file")
iterate_internal_ :: proc (it: ^Bit_Array_Iterator, $ITERATE_SET_BITS: bool) -> (index: int, ok: bool) {
word := it.array.bits[it.word_idx] if len(it.array.bits) > it.word_idx else 0
when ! ITERATE_SET_BITS { word = ~word }
// if the word is empty or we have already gone over all the bits in it,
// b.bit_idx is greater than the index of any set bit in the word,
// meaning that word >> b.bit_idx == 0.
for it.word_idx < len(it.array.bits) && word >> it.bit_idx == 0 {
it.word_idx += 1
it.bit_idx = 0
word = it.array.bits[it.word_idx] if len(it.array.bits) > it.word_idx else 0
when ! ITERATE_SET_BITS { word = ~word }
}
// if we are iterating the set bits, reaching the end of the array means we have no more bits to check
when ITERATE_SET_BITS {
if it.word_idx >= len(it.array.bits) {
return 0, false
}
}
// reaching here means that the word has some set bits
it.bit_idx += uint(intrinsics.count_trailing_zeros(word >> it.bit_idx))
index = it.word_idx * NUM_BITS + int(it.bit_idx) + it.array.bias
it.bit_idx += 1
if it.bit_idx >= NUM_BITS {
it.bit_idx = 0
it.word_idx += 1
}
return index, index <= it.array.max_index
}
/*
In:
- ba: ^Bit_Array - a pointer to the Bit Array
@@ -70,14 +180,42 @@ set :: proc(ba: ^Bit_Array, #any_int index: uint, allocator := context.allocator
resize_if_needed(ba, leg_index) or_return
ba.max_index = max(idx, ba.max_index)
ba.bits[leg_index] |= 1 << uint(bit_index)
return true
}
/*
In:
- ba: ^Bit_Array - a pointer to the Bit Array
- index: The bit index. Can be an enum member.
Out:
- ok: Whether or not we managed to unset requested bit.
`unset` automatically resizes the Bit Array to accommodate the requested index if needed.
*/
unset :: proc(ba: ^Bit_Array, #any_int index: uint, allocator := context.allocator) -> (ok: bool) {
idx := int(index) - ba.bias
if ba == nil || int(index) < ba.bias { return false }
context.allocator = allocator
leg_index := idx >> INDEX_SHIFT
bit_index := idx & INDEX_MASK
resize_if_needed(ba, leg_index) or_return
ba.max_index = max(idx, ba.max_index)
ba.bits[leg_index] &= ~(1 << uint(bit_index))
return true
}
/*
A helper function to create a Bit Array with optional bias, in case your smallest index is non-zero (including negative).
*/
create :: proc(max_index: int, min_index := 0, allocator := context.allocator) -> (res: Bit_Array, ok: bool) #optional_ok {
create :: proc(max_index: int, min_index := 0, allocator := context.allocator) -> (res: ^Bit_Array, ok: bool) #optional_ok {
context.allocator = allocator
size_in_bits := max_index - min_index
@@ -85,10 +223,11 @@ create :: proc(max_index: int, min_index := 0, allocator := context.allocator) -
legs := size_in_bits >> INDEX_SHIFT
res = Bit_Array{
bias = min_index,
}
return res, resize_if_needed(&res, size_in_bits)
res = new(Bit_Array)
res.bias = min_index
res.max_index = max_index
res.free_pointer = true
return res, resize_if_needed(res, legs)
}
/*
@@ -96,7 +235,7 @@ create :: proc(max_index: int, min_index := 0, allocator := context.allocator) -
*/
clear :: proc(ba: ^Bit_Array) {
if ba == nil { return }
ba.bits = {}
mem.zero_slice(ba.bits[:])
}
/*
@@ -105,6 +244,9 @@ clear :: proc(ba: ^Bit_Array) {
destroy :: proc(ba: ^Bit_Array) {
if ba == nil { return }
delete(ba.bits)
if ba.free_pointer { // Only free if this Bit_Array was created using `create`, not when on the stack.
free(ba)
}
}
/*
@@ -121,4 +263,4 @@ resize_if_needed :: proc(ba: ^Bit_Array, legs: int, allocator := context.allocat
resize(&ba.bits, legs + 1)
}
return len(ba.bits) > legs
}
}

13
core/container/bit_array/doc.odin
View File

@@ -21,6 +21,7 @@ package dynamic_bit_array
// returns `false`, `false`, because this Bit Array wasn't created to allow negative indices.
was_set, was_retrieved := get(&bits, -1)
fmt.println(was_set, was_retrieved)
destroy(&bits)
}
-- A Bit Array can optionally allow for negative indices, if the mininum value was given during creation:
@@ -40,13 +41,13 @@ package dynamic_bit_array
using bit_array
bits := create(int(max(Foo)), int(min(Foo)))
defer destroy(&bits)
defer destroy(bits)
fmt.printf("Set(Bar): %v\n", set(&bits, Foo.Bar))
fmt.printf("Get(Bar): %v, %v\n", get(&bits, Foo.Bar))
fmt.printf("Set(Negative_Test): %v\n", set(&bits, Foo.Negative_Test))
fmt.printf("Get(Leaves): %v, %v\n", get(&bits, Foo.Leaves))
fmt.printf("Get(Negative_Test): %v, %v\n", get(&bits, Foo.Negative_Test))
fmt.printf("Set(Bar): %v\n", set(bits, Foo.Bar))
fmt.printf("Get(Bar): %v, %v\n", get(bits, Foo.Bar))
fmt.printf("Set(Negative_Test): %v\n", set(bits, Foo.Negative_Test))
fmt.printf("Get(Leaves): %v, %v\n", get(bits, Foo.Leaves))
fmt.printf("Get(Negative_Test): %v, %v\n", get(bits, Foo.Negative_Test))
fmt.printf("Freed.\n")
}
*/

173
core/container/intrusive/list/intrusive_list.odin Normal file
View File

@@ -0,0 +1,173 @@
package container_intrusive_list
import "core:intrinsics"
// An intrusive doubly-linked list
//
// As this is an intrusive container, a `Node` must be embedded in your own
// structure which is conventionally called a "link". The use of `push_front`
// and `push_back` take the address of this node. Retrieving the data
// associated with the node requires finding the relative offset of the node
// of the parent structure. The parent type and field name are given to
// `iterator_*` procedures, or to the built-in `container_of` procedure.
//
// This data structure is two-pointers in size:
// 8 bytes on 32-bit platforms and 16 bytes on 64-bit platforms
List :: struct {
head: ^Node,
tail: ^Node,
}
Node :: struct {
next, prev: ^Node,
}
push_front :: proc(list: ^List, node: ^Node) {
if list.head != nil {
list.head.prev = node
node.prev, node.next = nil, list.head
list.head = node
} else {
list.head, list.tail = node, node
node.prev, node.next = nil, nil
}
}
push_back :: proc(list: ^List, node: ^Node) {
if list.tail != nil {
list.tail.next = node
node.prev, node.next = list.tail, nil
list.tail = node
} else {
list.head, list.tail = node, node
node.prev, node.next = nil, nil
}
}
remove :: proc(list: ^List, node: ^Node) {
if node != nil {
if node.next != nil {
node.next.prev = node.prev
}
if node.prev != nil {
node.prev.next = node.next
}
if list.head == node {
list.head = node.next
}
if list.tail == node {
list.tail = node.prev
}
}
}
remove_by_proc :: proc(list: ^List, to_erase: proc(^Node) -> bool) {
for node := list.head; node != nil; {
next := node.next
if to_erase(node) {
if node.next != nil {
node.next.prev = node.prev
}
if node.prev != nil {
node.prev.next = node.next
}
if list.head == node {
list.head = node.next
}
if list.tail == node {
list.tail = node.prev
}
}
node = next
}
}
is_empty :: proc(list: ^List) -> bool {
return list.head == nil
}
pop_front :: proc(list: ^List) -> ^Node {
link := list.head
if link == nil {
return nil
}
if link.next != nil {
link.next.prev = link.prev
}
if link.prev != nil {
link.prev.next = link.next
}
if link == list.head {
list.head = link.next
}
if link == list.tail {
list.tail = link.prev
}
return link
}
pop_back :: proc(list: ^List) -> ^Node {
link := list.tail
if link == nil {
return nil
}
if link.next != nil {
link.next.prev = link.prev
}
if link.prev != nil {
link.prev.next = link.next
}
if link == list.head {
list.head = link.next
}
if link == list.tail {
list.tail = link.prev
}
return link
}
Iterator :: struct($T: typeid) {
curr: ^Node,
offset: uintptr,
}
iterator_head :: proc(list: List, $T: typeid, $field_name: string) -> Iterator(T)
where intrinsics.type_has_field(T, field_name),
intrinsics.type_field_type(T, field_name) == Node {
return {list.head, offset_of_by_string(T, field_name)}
}
iterator_tail :: proc(list: List, $T: typeid, $field_name: string) -> Iterator(T)
where intrinsics.type_has_field(T, field_name),
intrinsics.type_field_type(T, field_name) == Node {
return {list.tail, offset_of_by_string(T, field_name)}
}
iterator_from_node :: proc(node: ^Node, $T: typeid, $field_name: string) -> Iterator(T)
where intrinsics.type_has_field(T, field_name),
intrinsics.type_field_type(T, field_name) == Node {
return {node, offset_of_by_string(T, field_name)}
}
iterate_next :: proc(it: ^Iterator($T)) -> (ptr: ^T, ok: bool) {
node := it.curr
if node == nil {
return nil, false
}
it.curr = node.next
return (^T)(uintptr(node) - it.offset), true
}
iterate_prev :: proc(it: ^Iterator($T)) -> (ptr: ^T, ok: bool) {
node := it.curr
if node == nil {
return nil, false
}
it.curr = node.prev
return (^T)(uintptr(node) - it.offset), true
}

201
core/container/lru/lru_cache.odin Normal file
View File

@@ -0,0 +1,201 @@
package container_lru
import "core:runtime"
import "core:intrinsics"
_ :: runtime
_ :: intrinsics
Node :: struct($Key, $Value: typeid) where intrinsics.type_is_valid_map_key(Key) {
prev, next: ^Node(Key, Value),
key: Key,
value: Value,
}
// Cache is an LRU cache. It automatically removes entries as new entries are
// added if the capacity is reached. Entries are removed based on how recently
// they were used where the oldest entries are removed first.
Cache :: struct($Key, $Value: typeid) where intrinsics.type_is_valid_map_key(Key) {
head: ^Node(Key, Value),
tail: ^Node(Key, Value),
entries: map[Key]^Node(Key, Value),
count: int,
capacity: int,
node_allocator: runtime.Allocator,
on_remove: proc(key: Key, value: Value, user_data: rawptr),
on_remove_user_data: rawptr,
}
// init initializes a Cache
init :: proc(c: ^$C/Cache($Key, $Value), capacity: int, entries_allocator := context.allocator, node_allocator := context.allocator) {
c.entries.allocator = entries_allocator
c.node_allocator = node_allocator
c.capacity = capacity
}
// destroy deinitializes a Cachem
destroy :: proc(c: ^$C/Cache($Key, $Value), call_on_remove: bool) {
clear(c, call_on_remove)
delete(c.entries)
}
// clear the contents of a Cache
clear :: proc(c: ^$C/Cache($Key, $Value), call_on_remove: bool) {
for _, node in c.entries {
if call_on_remove {
_call_on_remove(c, node)
}
free(node, c.node_allocator)
}
runtime.clear(&c.entries)
c.head = nil
c.tail = nil
c.count = 0
}
// set the given key value pair. This operation updates the recent usage of the item.
set :: proc(c: ^$C/Cache($Key, $Value), key: Key, value: Value) -> runtime.Allocator_Error {
if e, ok := c.entries[key]; ok {
e.value = value
_pop_node(c, e)
_push_front_node(c, e)
return nil
}
e : ^Node(Key, Value) = nil
assert(c.count <= c.capacity)
if c.count == c.capacity {
e = c.tail
_remove_node(c, e)
}
else {
c.count += 1
e = new(Node(Key, Value), c.node_allocator) or_return
}
e.key = key
e.value = value
_push_front_node(c, e)
c.entries[key] = e
return nil
}
// get a value from the cache from a given key. This operation updates the usage of the item.
get :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> (value: Value, ok: bool) #optional_ok {
e: ^Node(Key, Value)
e, ok = c.entries[key]
if !ok {
return
}
_pop_node(c, e)
_push_front_node(c, e)
return e.value, true
}
// get_ptr gets the pointer to a value the cache from a given key. This operation updates the usage of the item.
get_ptr :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> (value: ^Value, ok: bool) #optional_ok {
e: ^Node(Key, Value)
e, ok = c.entries[key]
if !ok {
return
}
_pop_node(c, e)
_push_front_node(c, e)
return &e.value, true
}
// peek gets the value from the cache from a given key without updating the recent usage.
peek :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> (value: Value, ok: bool) #optional_ok {
e: ^Node(Key, Value)
e, ok = c.entries[key]
if !ok {
return
}
return e.value, true
}
// exists checks for the existence of a value from a given key without updating the recent usage.
exists :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> bool {
return key in c.entries
}
// remove removes an item from the cache.
remove :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> bool {
e, ok := c.entries[key]
if !ok {
return false
}
_remove_node(c, e)
free(node, c.node_allocator)
c.count -= 1
return true
}
@(private)
_remove_node :: proc(c: ^$C/Cache($Key, $Value), node: ^Node(Key, Value)) {
if c.head == node {
c.head = node.next
}
if c.tail == node {
c.tail = node.prev
}
if node.prev != nil {
node.prev.next = node.next
}
if node.next != nil {
node.next.prev = node.prev
}
node.prev = nil
node.next = nil
delete_key(&c.entries, node.key)
_call_on_remove(c, node)
}
@(private)
_call_on_remove :: proc(c: ^$C/Cache($Key, $Value), node: ^Node(Key, Value)) {
if c.on_remove != nil {
c.on_remove(node.key, node.value, c.on_remove_user_data)
}
}
@(private)
_push_front_node :: proc(c: ^$C/Cache($Key, $Value), e: ^Node(Key, Value)) {
if c.head != nil {
e.next = c.head
e.next.prev = e
}
c.head = e
if c.tail == nil {
c.tail = e
}
e.prev = nil
}
@(private)
_pop_node :: proc(c: ^$C/Cache($Key, $Value), e: ^Node(Key, Value)) {
if e == nil {
return
}
if c.head == e {
c.head = e.next
}
if c.tail == e {
c.tail = e.prev
}
if e.prev != nil {
e.prev.next = e.next
}
if e.next != nil {
e.next.prev = e.prev
}
e.prev = nil
e.next = nil
}

11
core/container/queue/queue.odin
View File

@@ -2,6 +2,7 @@ package container_queue
import "core:builtin"
import "core:runtime"
_ :: runtime
// Dynamically resizable double-ended queue/ring-buffer
Queue :: struct($T: typeid) {
@@ -68,6 +69,16 @@ get :: proc(q: ^$Q/Queue($T), #any_int i: int, loc := #caller_location) -> T {
idx := (uint(i)+q.offset)%builtin.len(q.data)
return q.data[idx]
}
front :: proc(q: ^$Q/Queue($T)) -> T {
return q.data[q.offset]
}
back :: proc(q: ^$Q/Queue($T)) -> T {
idx := (q.offset+uint(q.len))%builtin.len(q.data)
return q.data[idx]
}
set :: proc(q: ^$Q/Queue($T), #any_int i: int, val: T, loc := #caller_location) {
runtime.bounds_check_error_loc(loc, i, builtin.len(q.data))

10
core/container/small_array/small_array.odin
View File

@@ -25,14 +25,14 @@ slice :: proc(a: ^$A/Small_Array($N, $T)) -> []T {
}
get :: proc(a: $A/Small_Array($N, $T), index: int, loc := #caller_location) -> T {
get :: proc(a: $A/Small_Array($N, $T), index: int) -> T {
return a.data[index]
}
get_ptr :: proc(a: $A/Small_Array($N, $T), index: int, loc := #caller_location) -> ^T {
get_ptr :: proc(a: ^$A/Small_Array($N, $T), index: int) -> ^T {
return &a.data[index]
}
set :: proc(a: ^$A/Small_Array($N, $T), index: int, item: T, loc := #caller_location) {
set :: proc(a: ^$A/Small_Array($N, $T), index: int, item: T) {
a.data[index] = item
}
@@ -86,7 +86,7 @@ pop_back_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (item: T, ok: bool) {
return
}
pop_front_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (T, bool) {
pop_front_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (item: T, ok: bool) {
if N > 0 && a.len > 0 {
item = a.data[0]
s := slice(a)
@@ -114,4 +114,4 @@ push_back_elems :: proc(a: ^$A/Small_Array($N, $T), items: ..T) {
append_elem :: push_back
append_elems :: push_back_elems
push :: proc{push_back, push_back_elems}
append :: proc{push_back, push_back_elems}
append :: proc{push_back, push_back_elems}

98
core/container/topological_sort/topological_sort.odin Normal file
View File

@@ -0,0 +1,98 @@
// The following is a generic O(V+E) topological sorter implementation.
// This is the fastest known method for topological sorting and Odin's
// map type is being used to accelerate lookups.
package container_topological_sort
import "core:intrinsics"
import "core:runtime"
_ :: intrinsics
_ :: runtime
Relations :: struct($K: typeid) where intrinsics.type_is_valid_map_key(K) {
dependents: map[K]bool,
dependencies: int,
}
Sorter :: struct(K: typeid) where intrinsics.type_is_valid_map_key(K) {
relations: map[K]Relations(K),
dependents_allocator: runtime.Allocator,
}
@(private="file")
make_relations :: proc(sorter: ^$S/Sorter($K)) -> (r: Relations(K)) {
r.dependents.allocator = sorter.dependents_allocator
return
}
init :: proc(sorter: ^$S/Sorter($K)) {
sorter.relations = make(map[K]Relations(K))
sorter.dependents_allocator = context.allocator
}
destroy :: proc(sorter: ^$S/Sorter($K)) {
for _, v in &sorter.relations {
delete(v.dependents)
}
delete(sorter.relations)
}
add_key :: proc(sorter: ^$S/Sorter($K), key: K) -> bool {
if key in sorter.relations {
return false
}
sorter.relations[key] = make_relations(sorter)
return true
}
add_dependency :: proc(sorter: ^$S/Sorter($K), key, dependency: K) -> bool {
if key == dependency {
return false
}
find := &sorter.relations[dependency]
if find == nil {
find = map_insert(&sorter.relations, dependency, make_relations(sorter))
}
if find.dependents[key] {
return true
}
find.dependents[key] = true
find = &sorter.relations[key]
if find == nil {
find = map_insert(&sorter.relations, key, make_relations(sorter))
}
find.dependencies += 1
return true
}
sort :: proc(sorter: ^$S/Sorter($K)) -> (sorted, cycled: [dynamic]K) {
relations := &sorter.relations
for k, v in relations {
if v.dependencies == 0 {
append(&sorted, k)
}
}
for root in &sorted do for k, _ in relations[root].dependents {
relation := &relations[k]
relation.dependencies -= 1
if relation.dependencies == 0 {
append(&sorted, k)
}
}
for k, v in relations {
if v.dependencies != 0 {
append(&cycled, k)
}
}
return
}

2
core/crypto/_fiat/field_poly1305/field.odin
View File

@@ -22,7 +22,7 @@ fe_from_bytes :: #force_inline proc (out1: ^Tight_Field_Element, arg1: []byte, a
assert(len(arg1) == 16)
when ODIN_ARCH == "i386" || ODIN_ARCH == "amd64" {
when ODIN_ARCH == .i386 || ODIN_ARCH == .amd64 {
// While it may be unwise to do deserialization here on our
// own when fiat-crypto provides equivalent functionality,
// doing it this way provides a little under 3x performance

8
core/crypto/blake/blake.odin
View File

@@ -44,7 +44,7 @@ hash_bytes_224 :: proc "contextless" (data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224 will hash the given input and write the
@@ -123,7 +123,7 @@ hash_bytes_256 :: proc "contextless" (data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -202,7 +202,7 @@ hash_bytes_384 :: proc "contextless" (data: []byte) -> [DIGEST_SIZE_384]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_384 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_384(transmute([]byte)(data), hash);
hash_bytes_to_buffer_384(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_384 will hash the given input and write the
@@ -281,7 +281,7 @@ hash_bytes_512 :: proc "contextless" (data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the

2
core/crypto/blake2b/blake2b.odin
View File

@@ -46,7 +46,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

2
core/crypto/blake2s/blake2s.odin
View File

@@ -47,7 +47,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

2
core/crypto/chacha20/chacha20.odin
View File

@@ -346,7 +346,7 @@ _do_blocks :: proc (ctx: ^Context, dst, src: []byte, nr_blocks: int) {
// Until dedicated assembly can be written leverage the fact that
// the callers of this routine ensure that src/dst are valid.
when ODIN_ARCH == "i386" || ODIN_ARCH == "amd64" {
when ODIN_ARCH == .i386 || ODIN_ARCH == .amd64 {
// util.PUT_U32_LE/util.U32_LE are not required on little-endian
// systems that also happen to not be strict about aligned
// memory access.

2
core/crypto/gost/gost.odin
View File

@@ -41,7 +41,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

8
core/crypto/groestl/groestl.odin
View File

@@ -44,7 +44,7 @@ hash_bytes_224 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224 will hash the given input and write the
@@ -123,7 +123,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -202,7 +202,7 @@ hash_bytes_384 :: proc(data: []byte) -> [DIGEST_SIZE_384]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_384 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_384(transmute([]byte)(data), hash);
hash_bytes_to_buffer_384(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_384 will hash the given input and write the
@@ -281,7 +281,7 @@ hash_bytes_512 :: proc(data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the

30
core/crypto/haval/haval.odin
View File

@@ -50,7 +50,7 @@ hash_bytes_128_3 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128_3 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128_3(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128_3(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128_3 will hash the given input and write the
@@ -135,7 +135,7 @@ hash_bytes_128_4 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128_4 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128_4(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128_4(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128_4 will hash the given input and write the
@@ -220,7 +220,7 @@ hash_bytes_128_5 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128_5 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128_5(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128_5(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128_5 will hash the given input and write the
@@ -305,7 +305,7 @@ hash_bytes_160_3 :: proc(data: []byte) -> [DIGEST_SIZE_160]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_160_3 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_160_3(transmute([]byte)(data), hash);
hash_bytes_to_buffer_160_3(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_160_3 will hash the given input and write the
@@ -390,7 +390,7 @@ hash_bytes_160_4 :: proc(data: []byte) -> [DIGEST_SIZE_160]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_160_4 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_160_4(transmute([]byte)(data), hash);
hash_bytes_to_buffer_160_4(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_160_4 will hash the given input and write the
@@ -475,7 +475,7 @@ hash_bytes_160_5 :: proc(data: []byte) -> [DIGEST_SIZE_160]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_160_5 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_160_5(transmute([]byte)(data), hash);
hash_bytes_to_buffer_160_5(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_160_5 will hash the given input and write the
@@ -560,7 +560,7 @@ hash_bytes_192_3 :: proc(data: []byte) -> [DIGEST_SIZE_192]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_192_3 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_192_3(transmute([]byte)(data), hash);
hash_bytes_to_buffer_192_3(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_192_3 will hash the given input and write the
@@ -645,7 +645,7 @@ hash_bytes_192_4 :: proc(data: []byte) -> [DIGEST_SIZE_192]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_192_4 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_192_4(transmute([]byte)(data), hash);
hash_bytes_to_buffer_192_4(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_192_4 will hash the given input and write the
@@ -730,7 +730,7 @@ hash_bytes_192_5 :: proc(data: []byte) -> [DIGEST_SIZE_192]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_192_5 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_192_5(transmute([]byte)(data), hash);
hash_bytes_to_buffer_192_5(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_192_5 will hash the given input and write the
@@ -815,7 +815,7 @@ hash_bytes_224_3 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224_3 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224_3(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224_3(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224_3 will hash the given input and write the
@@ -900,7 +900,7 @@ hash_bytes_224_4 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224_4 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224_4(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224_4(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224_4 will hash the given input and write the
@@ -985,7 +985,7 @@ hash_bytes_224_5 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224_5 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224_5(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224_5(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224_5 will hash the given input and write the
@@ -1070,7 +1070,7 @@ hash_bytes_256_3 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256_3 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256_3(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256_3(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256_3 will hash the given input and write the
@@ -1155,7 +1155,7 @@ hash_bytes_256_4 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256_4 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256_4(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256_4(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256_4 will hash the given input and write the
@@ -1240,7 +1240,7 @@ hash_bytes_256_5 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256_5 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256_5(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256_5(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256_5 will hash the given input and write the

8
core/crypto/jh/jh.odin
View File

@@ -44,7 +44,7 @@ hash_bytes_224 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224 will hash the given input and write the
@@ -123,7 +123,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -202,7 +202,7 @@ hash_bytes_384 :: proc(data: []byte) -> [DIGEST_SIZE_384]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_384 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_384(transmute([]byte)(data), hash);
hash_bytes_to_buffer_384(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_384 will hash the given input and write the
@@ -281,7 +281,7 @@ hash_bytes_512 :: proc(data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the

8
core/crypto/keccak/keccak.odin
View File

@@ -49,7 +49,7 @@ hash_bytes_224 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224 will hash the given input and write the
@@ -131,7 +131,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -213,7 +213,7 @@ hash_bytes_384 :: proc(data: []byte) -> [DIGEST_SIZE_384]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_384 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_384(transmute([]byte)(data), hash);
hash_bytes_to_buffer_384(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_384 will hash the given input and write the
@@ -295,7 +295,7 @@ hash_bytes_512 :: proc(data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the

2
core/crypto/md2/md2.odin
View File

@@ -40,7 +40,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

2
core/crypto/md4/md4.odin
View File

@@ -44,7 +44,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

2
core/crypto/md5/md5.odin
View File

@@ -43,7 +43,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

2
core/crypto/rand_generic.odin
View File

@@ -1,6 +1,6 @@
package crypto
when ODIN_OS != "linux" {
when ODIN_OS != .Linux && ODIN_OS != .OpenBSD && ODIN_OS != .Windows {
_rand_bytes :: proc (dst: []byte) {
unimplemented("crypto: rand_bytes not supported on this OS")
}

12
core/crypto/rand_openbsd.odin Normal file
View File

@@ -0,0 +1,12 @@
package crypto
import "core:c"
foreign import libc "system:c"
foreign libc {
arc4random_buf :: proc "c" (buf: rawptr, nbytes: c.size_t) ---
}
_rand_bytes :: proc (dst: []byte) {
arc4random_buf(raw_data(dst), len(dst))
}

23
core/crypto/rand_windows.odin Normal file
View File

@@ -0,0 +1,23 @@
package crypto
import win32 "core:sys/windows"
import "core:os"
import "core:fmt"
_rand_bytes :: proc(dst: []byte) {
ret := (os.Errno)(win32.BCryptGenRandom(nil, raw_data(dst), u32(len(dst)), win32.BCRYPT_USE_SYSTEM_PREFERRED_RNG))
if ret != os.ERROR_NONE {
switch ret {
case os.ERROR_INVALID_HANDLE:
// The handle to the first parameter is invalid.
// This should not happen here, since we explicitly pass nil to it
panic("crypto: BCryptGenRandom Invalid handle for hAlgorithm")
case os.ERROR_INVALID_PARAMETER:
// One of the parameters was invalid
panic("crypto: BCryptGenRandom Invalid parameter")
case:
// Unknown error
panic(fmt.tprintf("crypto: BCryptGenRandom failed: %d\n", ret))
}
}
}

8
core/crypto/ripemd/ripemd.odin
View File

@@ -45,7 +45,7 @@ hash_bytes_128 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128 will hash the given input and write the
@@ -121,7 +121,7 @@ hash_bytes_160 :: proc(data: []byte) -> [DIGEST_SIZE_160]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_160 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_160(transmute([]byte)(data), hash);
hash_bytes_to_buffer_160(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_160 will hash the given input and write the
@@ -197,7 +197,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -273,7 +273,7 @@ hash_bytes_320 :: proc(data: []byte) -> [DIGEST_SIZE_320]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_320 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_320(transmute([]byte)(data), hash);
hash_bytes_to_buffer_320(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_320 will hash the given input and write the

2
core/crypto/sha1/sha1.odin
View File

@@ -43,7 +43,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

14
core/crypto/sha2/sha2.odin
View File

@@ -48,7 +48,7 @@ hash_bytes_224 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224 will hash the given input and write the
@@ -127,7 +127,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -206,7 +206,7 @@ hash_bytes_384 :: proc(data: []byte) -> [DIGEST_SIZE_384]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_384 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_384(transmute([]byte)(data), hash);
hash_bytes_to_buffer_384(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_384 will hash the given input and write the
@@ -285,7 +285,7 @@ hash_bytes_512 :: proc(data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the
@@ -419,8 +419,10 @@ update :: proc(ctx: ^$T, data: []byte) {
sha2_transf(ctx, shifted_message, block_nb)
rem_len = new_len % CURR_BLOCK_SIZE
when T == Sha256_Context {copy(ctx.block[:], shifted_message[block_nb << 6:rem_len])}
else when T == Sha512_Context {copy(ctx.block[:], shifted_message[block_nb << 7:rem_len])}
if rem_len > 0 {
when T == Sha256_Context {copy(ctx.block[:], shifted_message[block_nb << 6:rem_len])}
else when T == Sha512_Context {copy(ctx.block[:], shifted_message[block_nb << 7:rem_len])}
}
ctx.length = rem_len
when T == Sha256_Context {ctx.tot_len += (block_nb + 1) << 6}

8
core/crypto/sha3/sha3.odin
View File

@@ -47,7 +47,7 @@ hash_bytes_224 :: proc(data: []byte) -> [DIGEST_SIZE_224]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_224 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_224(transmute([]byte)(data), hash);
hash_bytes_to_buffer_224(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_224 will hash the given input and write the
@@ -126,7 +126,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -205,7 +205,7 @@ hash_bytes_384 :: proc(data: []byte) -> [DIGEST_SIZE_384]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_384 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_384(transmute([]byte)(data), hash);
hash_bytes_to_buffer_384(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_384 will hash the given input and write the
@@ -284,7 +284,7 @@ hash_bytes_512 :: proc(data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the

4
core/crypto/shake/shake.odin
View File

@@ -46,7 +46,7 @@ hash_bytes_128 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128 will hash the given input and write the
@@ -128,7 +128,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the

335
core/crypto/siphash/siphash.odin Normal file
View File

@@ -0,0 +1,335 @@
package siphash
/*
Copyright 2022 zhibog
Made available under the BSD-3 license.
List of contributors:
zhibog: Initial implementation.
Implementation of the SipHash hashing algorithm, as defined at <https://github.com/veorq/SipHash> and <https://www.aumasson.jp/siphash/siphash.pdf>
Use the specific procedures for a certain setup. The generic procdedures will default to Siphash 2-4
*/
import "core:crypto"
import "core:crypto/util"
/*
High level API
*/
KEY_SIZE :: 16
DIGEST_SIZE :: 8
// sum_string_1_3 will hash the given message with the key and return
// the computed hash as a u64
sum_string_1_3 :: proc(msg, key: string) -> u64 {
return sum_bytes_1_3(transmute([]byte)(msg), transmute([]byte)(key))
}
// sum_bytes_1_3 will hash the given message with the key and return
// the computed hash as a u64
sum_bytes_1_3 :: proc (msg, key: []byte) -> u64 {
ctx: Context
hash: u64
init(&ctx, key, 1, 3)
update(&ctx, msg)
final(&ctx, &hash)
return hash
}
// sum_string_to_buffer_1_3 will hash the given message with the key and write
// the computed hash into the provided destination buffer
sum_string_to_buffer_1_3 :: proc(msg, key: string, dst: []byte) {
sum_bytes_to_buffer_1_3(transmute([]byte)(msg), transmute([]byte)(key), dst)
}
// sum_bytes_to_buffer_1_3 will hash the given message with the key and write
// the computed hash into the provided destination buffer
sum_bytes_to_buffer_1_3 :: proc(msg, key, dst: []byte) {
assert(len(dst) >= DIGEST_SIZE, "crypto/siphash: Destination buffer needs to be at least of size 8")
hash := sum_bytes_1_3(msg, key)
_collect_output(dst[:], hash)
}
sum_1_3 :: proc {
sum_string_1_3,
sum_bytes_1_3,
sum_string_to_buffer_1_3,
sum_bytes_to_buffer_1_3,
}
// verify_u64_1_3 will check if the supplied tag matches with the output you
// will get from the provided message and key
verify_u64_1_3 :: proc (tag: u64 msg, key: []byte) -> bool {
return sum_bytes_1_3(msg, key) == tag
}
// verify_bytes will check if the supplied tag matches with the output you
// will get from the provided message and key
verify_bytes_1_3 :: proc (tag, msg, key: []byte) -> bool {
derived_tag: [8]byte
sum_bytes_to_buffer_1_3(msg, key, derived_tag[:])
return crypto.compare_constant_time(derived_tag[:], tag) == 1
}
verify_1_3 :: proc {
verify_bytes_1_3,
verify_u64_1_3,
}
// sum_string_2_4 will hash the given message with the key and return
// the computed hash as a u64
sum_string_2_4 :: proc(msg, key: string) -> u64 {
return sum_bytes_2_4(transmute([]byte)(msg), transmute([]byte)(key))
}
// sum_bytes_2_4 will hash the given message with the key and return
// the computed hash as a u64
sum_bytes_2_4 :: proc (msg, key: []byte) -> u64 {
ctx: Context
hash: u64
init(&ctx, key, 2, 4)
update(&ctx, msg)
final(&ctx, &hash)
return hash
}
// sum_string_to_buffer_2_4 will hash the given message with the key and write
// the computed hash into the provided destination buffer
sum_string_to_buffer_2_4 :: proc(msg, key: string, dst: []byte) {
sum_bytes_to_buffer_2_4(transmute([]byte)(msg), transmute([]byte)(key), dst)
}
// sum_bytes_to_buffer_2_4 will hash the given message with the key and write
// the computed hash into the provided destination buffer
sum_bytes_to_buffer_2_4 :: proc(msg, key, dst: []byte) {
assert(len(dst) >= DIGEST_SIZE, "crypto/siphash: Destination buffer needs to be at least of size 8")
hash := sum_bytes_2_4(msg, key)
_collect_output(dst[:], hash)
}
sum_2_4 :: proc {
sum_string_2_4,
sum_bytes_2_4,
sum_string_to_buffer_2_4,
sum_bytes_to_buffer_2_4,
}
sum_string :: sum_string_2_4
sum_bytes :: sum_bytes_2_4
sum_string_to_buffer :: sum_string_to_buffer_2_4
sum_bytes_to_buffer :: sum_bytes_to_buffer_2_4
sum :: proc {
sum_string,
sum_bytes,
sum_string_to_buffer,
sum_bytes_to_buffer,
}
// verify_u64_2_4 will check if the supplied tag matches with the output you
// will get from the provided message and key
verify_u64_2_4 :: proc (tag: u64 msg, key: []byte) -> bool {
return sum_bytes_2_4(msg, key) == tag
}
// verify_bytes will check if the supplied tag matches with the output you
// will get from the provided message and key
verify_bytes_2_4 :: proc (tag, msg, key: []byte) -> bool {
derived_tag: [8]byte
sum_bytes_to_buffer_2_4(msg, key, derived_tag[:])
return crypto.compare_constant_time(derived_tag[:], tag) == 1
}
verify_2_4 :: proc {
verify_bytes_2_4,
verify_u64_2_4,
}
verify_bytes :: verify_bytes_2_4
verify_u64 :: verify_u64_2_4
verify :: proc {
verify_bytes,
verify_u64,
}
// sum_string_4_8 will hash the given message with the key and return
// the computed hash as a u64
sum_string_4_8 :: proc(msg, key: string) -> u64 {
return sum_bytes_4_8(transmute([]byte)(msg), transmute([]byte)(key))
}
// sum_bytes_4_8 will hash the given message with the key and return
// the computed hash as a u64
sum_bytes_4_8 :: proc (msg, key: []byte) -> u64 {
ctx: Context
hash: u64
init(&ctx, key, 4, 8)
update(&ctx, msg)
final(&ctx, &hash)
return hash
}
// sum_string_to_buffer_4_8 will hash the given message with the key and write
// the computed hash into the provided destination buffer
sum_string_to_buffer_4_8 :: proc(msg, key: string, dst: []byte) {
sum_bytes_to_buffer_4_8(transmute([]byte)(msg), transmute([]byte)(key), dst)
}
// sum_bytes_to_buffer_4_8 will hash the given message with the key and write
// the computed hash into the provided destination buffer
sum_bytes_to_buffer_4_8 :: proc(msg, key, dst: []byte) {
assert(len(dst) >= DIGEST_SIZE, "crypto/siphash: Destination buffer needs to be at least of size 8")
hash := sum_bytes_4_8(msg, key)
_collect_output(dst[:], hash)
}
sum_4_8 :: proc {
sum_string_4_8,
sum_bytes_4_8,
sum_string_to_buffer_4_8,
sum_bytes_to_buffer_4_8,
}
// verify_u64_4_8 will check if the supplied tag matches with the output you
// will get from the provided message and key
verify_u64_4_8 :: proc (tag: u64 msg, key: []byte) -> bool {
return sum_bytes_4_8(msg, key) == tag
}
// verify_bytes will check if the supplied tag matches with the output you
// will get from the provided message and key
verify_bytes_4_8 :: proc (tag, msg, key: []byte) -> bool {
derived_tag: [8]byte
sum_bytes_to_buffer_4_8(msg, key, derived_tag[:])
return crypto.compare_constant_time(derived_tag[:], tag) == 1
}
verify_4_8 :: proc {
verify_bytes_4_8,
verify_u64_4_8,
}
/*
Low level API
*/
init :: proc(ctx: ^Context, key: []byte, c_rounds, d_rounds: int) {
assert(len(key) == KEY_SIZE, "crypto/siphash: Invalid key size, want 16")
ctx.c_rounds = c_rounds
ctx.d_rounds = d_rounds
is_valid_setting := (ctx.c_rounds == 1 && ctx.d_rounds == 3) ||
(ctx.c_rounds == 2 && ctx.d_rounds == 4) ||
(ctx.c_rounds == 4 && ctx.d_rounds == 8)
assert(is_valid_setting, "crypto/siphash: Incorrect rounds set up. Valid pairs are (1,3), (2,4) and (4,8)")
ctx.k0 = util.U64_LE(key[:8])
ctx.k1 = util.U64_LE(key[8:])
ctx.v0 = 0x736f6d6570736575 ~ ctx.k0
ctx.v1 = 0x646f72616e646f6d ~ ctx.k1
ctx.v2 = 0x6c7967656e657261 ~ ctx.k0
ctx.v3 = 0x7465646279746573 ~ ctx.k1
ctx.is_initialized = true
}
update :: proc(ctx: ^Context, data: []byte) {
assert(ctx.is_initialized, "crypto/siphash: Context is not initalized")
ctx.last_block = len(data) / 8 * 8
ctx.buf = data
i := 0
m: u64
for i < ctx.last_block {
m = u64(ctx.buf[i] & 0xff)
i += 1
for r in u64(1)..<8 {
m |= u64(ctx.buf[i] & 0xff) << (r * 8)
i += 1
}
ctx.v3 ~= m
for _ in 0..<ctx.c_rounds {
_compress(ctx)
}
ctx.v0 ~= m
}
}
final :: proc(ctx: ^Context, dst: ^u64) {
m: u64
for i := len(ctx.buf) - 1; i >= ctx.last_block; i -= 1 {
m <<= 8
m |= u64(ctx.buf[i] & 0xff)
}
m |= u64(len(ctx.buf) << 56)
ctx.v3 ~= m
for _ in 0..<ctx.c_rounds {
_compress(ctx)
}
ctx.v0 ~= m
ctx.v2 ~= 0xff
for _ in 0..<ctx.d_rounds {
_compress(ctx)
}
dst^ = ctx.v0 ~ ctx.v1 ~ ctx.v2 ~ ctx.v3
reset(ctx)
}
reset :: proc(ctx: ^Context) {
ctx.k0, ctx.k1 = 0, 0
ctx.v0, ctx.v1 = 0, 0
ctx.v2, ctx.v3 = 0, 0
ctx.last_block = 0
ctx.c_rounds = 0
ctx.d_rounds = 0
ctx.is_initialized = false
}
Context :: struct {
v0, v1, v2, v3: u64, // State values
k0, k1: u64, // Split key
c_rounds: int, // Number of message rounds
d_rounds: int, // Number of finalization rounds
buf: []byte, // Provided data
last_block: int, // Offset from the last block
is_initialized: bool,
}
_get_byte :: #force_inline proc "contextless" (byte_num: byte, into: u64) -> byte {
return byte(into >> (((~byte_num) & (size_of(u64) - 1)) << 3))
}
_collect_output :: #force_inline proc "contextless" (dst: []byte, hash: u64) {
dst[0] = _get_byte(7, hash)
dst[1] = _get_byte(6, hash)
dst[2] = _get_byte(5, hash)
dst[3] = _get_byte(4, hash)
dst[4] = _get_byte(3, hash)
dst[5] = _get_byte(2, hash)
dst[6] = _get_byte(1, hash)
dst[7] = _get_byte(0, hash)
}
_compress :: #force_inline proc "contextless" (ctx: ^Context) {
ctx.v0 += ctx.v1
ctx.v1 = util.ROTL64(ctx.v1, 13)
ctx.v1 ~= ctx.v0
ctx.v0 = util.ROTL64(ctx.v0, 32)
ctx.v2 += ctx.v3
ctx.v3 = util.ROTL64(ctx.v3, 16)
ctx.v3 ~= ctx.v2
ctx.v0 += ctx.v3
ctx.v3 = util.ROTL64(ctx.v3, 21)
ctx.v3 ~= ctx.v0
ctx.v2 += ctx.v1
ctx.v1 = util.ROTL64(ctx.v1, 17)
ctx.v1 ~= ctx.v2
ctx.v2 = util.ROTL64(ctx.v2, 32)
}

2
core/crypto/sm3/sm3.odin
View File

@@ -42,7 +42,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

4
core/crypto/streebog/streebog.odin
View File

@@ -44,7 +44,7 @@ hash_bytes_256 :: proc(data: []byte) -> [DIGEST_SIZE_256]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_256 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_256(transmute([]byte)(data), hash);
hash_bytes_to_buffer_256(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_256 will hash the given input and write the
@@ -122,7 +122,7 @@ hash_bytes_512 :: proc(data: []byte) -> [DIGEST_SIZE_512]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_512 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_512(transmute([]byte)(data), hash);
hash_bytes_to_buffer_512(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_512 will hash the given input and write the

6
core/crypto/tiger/tiger.odin
View File

@@ -45,7 +45,7 @@ hash_bytes_128 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128 will hash the given input and write the
@@ -124,7 +124,7 @@ hash_bytes_160 :: proc(data: []byte) -> [DIGEST_SIZE_160]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_160 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_160(transmute([]byte)(data), hash);
hash_bytes_to_buffer_160(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_160 will hash the given input and write the
@@ -203,7 +203,7 @@ hash_bytes_192 :: proc(data: []byte) -> [DIGEST_SIZE_192]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_192 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_192(transmute([]byte)(data), hash);
hash_bytes_to_buffer_192(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_192 will hash the given input and write the

6
core/crypto/tiger2/tiger2.odin
View File

@@ -45,7 +45,7 @@ hash_bytes_128 :: proc(data: []byte) -> [DIGEST_SIZE_128]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_128 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_128(transmute([]byte)(data), hash);
hash_bytes_to_buffer_128(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_128 will hash the given input and write the
@@ -124,7 +124,7 @@ hash_bytes_160 :: proc(data: []byte) -> [DIGEST_SIZE_160]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_160 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_160(transmute([]byte)(data), hash);
hash_bytes_to_buffer_160(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_160 will hash the given input and write the
@@ -203,7 +203,7 @@ hash_bytes_192 :: proc(data: []byte) -> [DIGEST_SIZE_192]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer_192 :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer_192(transmute([]byte)(data), hash);
hash_bytes_to_buffer_192(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer_192 will hash the given input and write the

2
core/crypto/whirlpool/whirlpool.odin
View File

@@ -42,7 +42,7 @@ hash_bytes :: proc(data: []byte) -> [DIGEST_SIZE]byte {
// computed hash to the second parameter.
// It requires that the destination buffer is at least as big as the digest size
hash_string_to_buffer :: proc(data: string, hash: []byte) {
hash_bytes_to_buffer(transmute([]byte)(data), hash);
hash_bytes_to_buffer(transmute([]byte)(data), hash)
}
// hash_bytes_to_buffer will hash the given input and write the

12
core/dynlib/lib.odin
View File

@@ -1,3 +1,15 @@
package dynlib
Library :: distinct rawptr
load_library :: proc(path: string, global_symbols := false) -> (Library, bool) {
return _load_library(path, global_symbols)
}
unload_library :: proc(library: Library) -> bool {
return _unload_library(library)
}
symbol_address :: proc(library: Library, symbol: string) -> (ptr: rawptr, found: bool) #optional_ok {
return _symbol_address(library, symbol)
}

29
core/dynlib/lib_unix.odin
View File

@@ -1,23 +1,24 @@
// +build linux, darwin, freebsd
//+build linux, darwin, freebsd, openbsd
//+private
package dynlib
import "core:os"
load_library :: proc(path: string, global_symbols := false) -> (Library, bool) {
flags := os.RTLD_NOW
if global_symbols {
flags |= os.RTLD_GLOBAL
}
lib := os.dlopen(path, flags)
return Library(lib), lib != nil
_load_library :: proc(path: string, global_symbols := false) -> (Library, bool) {
flags := os.RTLD_NOW
if global_symbols {
flags |= os.RTLD_GLOBAL
}
lib := os.dlopen(path, flags)
return Library(lib), lib != nil
}
unload_library :: proc(library: Library) {
os.dlclose(rawptr(library))
_unload_library :: proc(library: Library) -> bool {
return os.dlclose(rawptr(library))
}
symbol_address :: proc(library: Library, symbol: string) -> (ptr: rawptr, found: bool) {
ptr = os.dlsym(rawptr(library), symbol)
found = ptr != nil
return
_symbol_address :: proc(library: Library, symbol: string) -> (ptr: rawptr, found: bool) {
ptr = os.dlsym(rawptr(library), symbol)
found = ptr != nil
return
}

9
core/dynlib/lib_windows.odin
View File

@@ -1,10 +1,11 @@
// +build windows
//+build windows
//+private
package dynlib
import win32 "core:sys/windows"
import "core:strings"
load_library :: proc(path: string, global_symbols := false) -> (Library, bool) {
_load_library :: proc(path: string, global_symbols := false) -> (Library, bool) {
// NOTE(bill): 'global_symbols' is here only for consistency with POSIX which has RTLD_GLOBAL
wide_path := win32.utf8_to_wstring(path, context.temp_allocator)
@@ -12,12 +13,12 @@ load_library :: proc(path: string, global_symbols := false) -> (Library, bool) {
return handle, handle != nil
}
unload_library :: proc(library: Library) -> bool {
_unload_library :: proc(library: Library) -> bool {
ok := win32.FreeLibrary(cast(win32.HMODULE)library)
return bool(ok)
}
symbol_address :: proc(library: Library, symbol: string) -> (ptr: rawptr, found: bool) {
_symbol_address :: proc(library: Library, symbol: string) -> (ptr: rawptr, found: bool) {
c_str := strings.clone_to_cstring(symbol, context.temp_allocator)
ptr = win32.GetProcAddress(cast(win32.HMODULE)library, c_str)
found = ptr != nil

86
core/encoding/csv/reader.odin
View File

@@ -34,6 +34,10 @@ Reader :: struct {
// If lazy_quotes is true, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field
lazy_quotes: bool,
// multiline_fields, when set to true, will treat a field starting with a " as a multiline string
// therefore, instead of reading until the next \n, it'll read until the next "
multiline_fields: bool,
// reuse_record controls whether calls to 'read' may return a slice using the backing buffer
// for performance
// By default, each call to 'read' returns a newly allocated slice
@@ -194,32 +198,72 @@ is_valid_delim :: proc(r: rune) -> bool {
@private
_read_record :: proc(r: ^Reader, dst: ^[dynamic]string, allocator := context.allocator) -> ([]string, Error) {
read_line :: proc(r: ^Reader) -> ([]byte, io.Error) {
line, err := bufio.reader_read_slice(&r.r, '\n')
if err == .Buffer_Full {
clear(&r.raw_buffer)
append(&r.raw_buffer, ..line)
for err == .Buffer_Full {
line, err = bufio.reader_read_slice(&r.r, '\n')
if !r.multiline_fields {
line, err := bufio.reader_read_slice(&r.r, '\n')
if err == .Buffer_Full {
clear(&r.raw_buffer)
append(&r.raw_buffer, ..line)
for err == .Buffer_Full {
line, err = bufio.reader_read_slice(&r.r, '\n')
append(&r.raw_buffer, ..line)
}
line = r.raw_buffer[:]
}
line = r.raw_buffer[:]
}
if len(line) > 0 && err == .EOF {
err = nil
if line[len(line)-1] == '\r' {
line = line[:len(line)-1]
if len(line) > 0 && err == .EOF {
err = nil
if line[len(line)-1] == '\r' {
line = line[:len(line)-1]
}
}
}
r.line_count += 1
r.line_count += 1
// normalize \r\n to \n
n := len(line)
for n >= 2 && string(line[n-2:]) == "\r\n" {
line[n-2] = '\n'
line = line[:n-1]
}
// normalize \r\n to \n
n := len(line)
for n >= 2 && string(line[n-2:]) == "\r\n" {
line[n-2] = '\n'
line = line[:n-1]
}
return line, err
return line, err
} else {
// Reading a "line" that can possibly contain multiline fields.
// Unfortunately, this means we need to read a character at a time.
err: io.Error
cur: rune
is_quoted: bool
field_length := 0
clear(&r.raw_buffer)
read_loop: for err == .None {
cur, _, err = bufio.reader_read_rune(&r.r)
if err != .None { break read_loop }
switch cur {
case '"':
is_quoted = field_length == 0
field_length += 1
case '\n', '\r':
if !is_quoted { break read_loop }
case r.comma:
field_length = 0
case:
field_length += 1
}
rune_buf, rune_len := utf8.encode_rune(cur)
append(&r.raw_buffer, ..rune_buf[:rune_len])
}
return r.raw_buffer[:], err
}
unreachable()
}
length_newline :: proc(b: []byte) -> int {

23
core/encoding/endian/doc.odin Normal file
View File

@@ -0,0 +1,23 @@
/*
Package endian implements sa simple translation between bytes and numbers with
specific endian encodings.
buf: [100]u8
put_u16(buf[:], .Little, 16) or_return
You may ask yourself, why isn't `byte_order` platform Endianness by default, so we can write:
put_u16(buf[:], 16) or_return
The answer is that very few file formats are written in native/platform endianness. Most of them specify the endianness of
each of their fields, or use a header field which specifies it for the entire file.
e.g. a file which specifies it at the top for all fields could do this:
file_order := .Little if buf[0] == 0 else .Big
field := get_u16(buf[1:], file_order) or_return
If on the other hand a field is *always* Big-Endian, you're wise to explicitly state it for the benefit of the reader,
be that your future self or someone else.
field := get_u16(buf[:], .Big) or_return
*/
package encoding_endian

153
core/encoding/endian/endian.odin Normal file
View File

@@ -0,0 +1,153 @@
package encoding_endian
Byte_Order :: enum u8 {
Little,
Big,
}
PLATFORM_BYTE_ORDER :: Byte_Order.Little when ODIN_ENDIAN == .Little else Byte_Order.Big
get_u16 :: proc(b: []byte, order: Byte_Order) -> (v: u16, ok: bool) {
if len(b) < 2 {
return 0, false
}
#no_bounds_check if order == .Little {
v = u16(b[0]) | u16(b[1])<<8
} else {
v = u16(b[1]) | u16(b[0])<<8
}
return v, true
}
get_u32 :: proc(b: []byte, order: Byte_Order) -> (v: u32, ok: bool) {
if len(b) < 4 {
return 0, false
}
#no_bounds_check if order == .Little {
v = u32(b[0]) | u32(b[1])<<8 | u32(b[2])<<16 | u32(b[3])<<24
} else {
v = u32(b[3]) | u32(b[2])<<8 | u32(b[1])<<16 | u32(b[0])<<24
}
return v, true
}
get_u64 :: proc(b: []byte, order: Byte_Order) -> (v: u64, ok: bool) {
if len(b) < 8 {
return 0, false
}
#no_bounds_check if order == .Little {
v = u64(b[0]) | u64(b[1])<<8 | u64(b[2])<<16 | u64(b[3])<<24 |
u64(b[4])<<32 | u64(b[5])<<40 | u64(b[6])<<48 | u64(b[7])<<56
} else {
v = u64(b[7]) | u64(b[6])<<8 | u64(b[5])<<16 | u64(b[4])<<24 |
u64(b[3])<<32 | u64(b[2])<<40 | u64(b[1])<<48 | u64(b[0])<<56
}
return v, true
}
get_i16 :: proc(b: []byte, order: Byte_Order) -> (i16, bool) {
v, ok := get_u16(b, order)
return i16(v), ok
}
get_i32 :: proc(b: []byte, order: Byte_Order) -> (i32, bool) {
v, ok := get_u32(b, order)
return i32(v), ok
}
get_i64 :: proc(b: []byte, order: Byte_Order) -> (i64, bool) {
v, ok := get_u64(b, order)
return i64(v), ok
}
get_f16 :: proc(b: []byte, order: Byte_Order) -> (f16, bool) {
v, ok := get_u16(b, order)
return transmute(f16)v, ok
}
get_f32 :: proc(b: []byte, order: Byte_Order) -> (f32, bool) {
v, ok := get_u32(b, order)
return transmute(f32)v, ok
}
get_f64 :: proc(b: []byte, order: Byte_Order) -> (f64, bool) {
v, ok := get_u64(b, order)
return transmute(f64)v, ok
}
put_u16 :: proc(b: []byte, order: Byte_Order, v: u16) -> bool {
if len(b) < 2 {
return false
}
#no_bounds_check if order == .Little {
b[0] = byte(v)
b[1] = byte(v >> 8)
} else {
b[0] = byte(v >> 8)
b[1] = byte(v)
}
return true
}
put_u32 :: proc(b: []byte, order: Byte_Order, v: u32) -> bool {
if len(b) < 4 {
return false
}
#no_bounds_check if order == .Little {
b[0] = byte(v)
b[1] = byte(v >> 8)
b[2] = byte(v >> 16)
b[3] = byte(v >> 24)
} else {
b[0] = byte(v >> 24)
b[1] = byte(v >> 16)
b[2] = byte(v >> 8)
b[3] = byte(v)
}
return true
}
put_u64 :: proc(b: []byte, order: Byte_Order, v: u64) -> bool {
if len(b) < 8 {
return false
}
#no_bounds_check if order == .Little {
b[0] = byte(v >> 0)
b[1] = byte(v >> 8)
b[2] = byte(v >> 16)
b[3] = byte(v >> 24)
b[4] = byte(v >> 32)
b[5] = byte(v >> 40)
b[6] = byte(v >> 48)
b[7] = byte(v >> 56)
} else {
b[0] = byte(v >> 56)
b[1] = byte(v >> 48)
b[2] = byte(v >> 40)
b[3] = byte(v >> 32)
b[4] = byte(v >> 24)
b[5] = byte(v >> 16)
b[6] = byte(v >> 8)
b[7] = byte(v)
}
return true
}
put_i16 :: proc(b: []byte, order: Byte_Order, v: i16) -> bool {
return put_u16(b, order, u16(v))
}
put_i32 :: proc(b: []byte, order: Byte_Order, v: i32) -> bool {
return put_u32(b, order, u32(v))
}
put_i64 :: proc(b: []byte, order: Byte_Order, v: i64) -> bool {
return put_u64(b, order, u64(v))
}
put_f16 :: proc(b: []byte, order: Byte_Order, v: f16) -> bool {
return put_u16(b, order, transmute(u16)v)
}
put_f32 :: proc(b: []byte, order: Byte_Order, v: f32) -> bool {
return put_u32(b, order, transmute(u32)v)
}
put_f64 :: proc(b: []byte, order: Byte_Order, v: f64) -> bool {
return put_u64(b, order, transmute(u64)v)
}

21
core/encoding/entity/LICENSE_table.md Normal file
View File

@@ -0,0 +1,21 @@
# License
By obtaining, using and/or copying this work, you (the licensee) agree that you have read, understood, and will comply with the following terms and conditions.
Permission to copy, modify, and distribute this software and its documentation, with or without modification, for any purpose and without fee or royalty is hereby granted, provided that you include the following on ALL copies of the software and documentation or portions thereof, including modifications:
The full text of this NOTICE in a location viewable to users of the redistributed or derivative work.
Any pre-existing intellectual property disclaimers, notices, or terms and conditions. If none exist, the W3C Software Short Notice should be included (hypertext is preferred, text is permitted) within the body of any redistributed or derivative code.
Notice of any changes or modifications to the files, including the date changes were made. (We recommend you provide URIs to the location from which the code is derived.)
# Disclaimers
THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS.
COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR DOCUMENTATION.
The name and trademarks of copyright holders may NOT be used in advertising or publicity pertaining to the software without specific, written prior permission. Title to copyright in this software and any associated documentation will at all times remain with copyright holders.
# Notes
This version: http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231

374
core/encoding/entity/entity.odin Normal file
View File

@@ -0,0 +1,374 @@
package unicode_entity
/*
A unicode entity encoder/decoder
Copyright 2021 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
This code has several procedures to map unicode runes to/from different textual encodings.
- SGML/XML/HTML entity
-- &#<decimal>;
-- &#x<hexadecimal>;
-- &<entity name>; (If the lookup tables are compiled in).
Reference: https://www.w3.org/2003/entities/2007xml/unicode.xml
- URL encode / decode %hex entity
Reference: https://datatracker.ietf.org/doc/html/rfc3986/#section-2.1
List of contributors:
Jeroen van Rijn: Initial implementation.
*/
import "core:unicode/utf8"
import "core:unicode"
import "core:strings"
MAX_RUNE_CODEPOINT :: int(unicode.MAX_RUNE)
write_rune :: strings.write_rune
write_string :: strings.write_string
Error :: enum u8 {
None = 0,
Tokenizer_Is_Nil,
Illegal_NUL_Character,
Illegal_UTF_Encoding,
Illegal_BOM,
CDATA_Not_Terminated,
Comment_Not_Terminated,
Invalid_Entity_Encoding,
}
Tokenizer :: struct {
r: rune,
w: int,
src: string,
offset: int,
read_offset: int,
}
CDATA_START :: "<![CDATA["
CDATA_END :: "]]>"
COMMENT_START :: "<!--"
COMMENT_END :: "-->"
/*
Default: CDATA and comments are passed through unchanged.
*/
XML_Decode_Option :: enum u8 {
/*
Do not decode & entities. It decodes by default.
If given, overrides `Decode_CDATA`.
*/
No_Entity_Decode,
/*
CDATA is unboxed.
*/
Unbox_CDATA,
/*
Unboxed CDATA is decoded as well.
Ignored if `.Unbox_CDATA` is not given.
*/
Decode_CDATA,
/*
Comments are stripped.
*/
Comment_Strip,
}
XML_Decode_Options :: bit_set[XML_Decode_Option; u8]
/*
Decode a string that may include SGML/XML/HTML entities.
The caller has to free the result.
*/
decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) {
context.allocator = allocator
l := len(input)
if l == 0 { return "", .None }
builder := strings.builder_make()
defer strings.builder_destroy(&builder)
t := Tokenizer{src=input}
in_data := false
loop: for {
advance(&t) or_return
if t.r < 0 { break loop }
/*
Below here we're never inside a CDATA tag.
At most we'll see the start of one, but that doesn't affect the logic.
*/
switch t.r {
case '<':
/*
Might be the start of a CDATA tag or comment.
We don't need to check if we need to write a `<`, because if it isn't CDATA or a comment,
it couldn't have been part of an XML tag body to be decoded here.
Keep in mind that we could already *be* inside a CDATA tag.
If so, write `>` as a literal and continue.
*/
if in_data {
write_rune(&builder, '<')
continue
}
in_data = _handle_xml_special(&t, &builder, options) or_return
case ']':
/*
If we're unboxing _and_ decoding CDATA, we'll have to check for the end tag.
*/
if in_data {
if t.read_offset + len(CDATA_END) < len(t.src) {
if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END {
in_data = false
t.read_offset += len(CDATA_END) - 1
}
}
continue
} else {
write_rune(&builder, ']')
}
case:
if in_data && .Decode_CDATA not_in options {
/*
Unboxed, but undecoded.
*/
write_rune(&builder, t.r)
continue
}
if t.r == '&' {
if entity, entity_err := _extract_xml_entity(&t); entity_err != .None {
/*
We read to the end of the string without closing the entity.
Pass through as-is.
*/
write_string(&builder, entity)
} else {
if .No_Entity_Decode not_in options {
if decoded, ok := xml_decode_entity(entity); ok {
write_rune(&builder, decoded)
continue
}
}
/*
Literal passthrough because the decode failed or we want entities not decoded.
*/
write_string(&builder, "&")
write_string(&builder, entity)
write_string(&builder, ";")
}
} else {
write_rune(&builder, t.r)
}
}
}
return strings.clone(strings.to_string(builder), allocator), err
}
advance :: proc(t: ^Tokenizer) -> (err: Error) {
if t == nil { return .Tokenizer_Is_Nil }
using t
#no_bounds_check {
if read_offset < len(src) {
offset = read_offset
r, w = rune(src[read_offset]), 1
switch {
case r == 0:
return .Illegal_NUL_Character
case r >= utf8.RUNE_SELF:
r, w = utf8.decode_rune_in_string(src[read_offset:])
if r == utf8.RUNE_ERROR && w == 1 {
return .Illegal_UTF_Encoding
} else if r == utf8.RUNE_BOM && offset > 0 {
return .Illegal_BOM
}
}
read_offset += w
return .None
} else {
offset = len(src)
r = -1
return
}
}
}
xml_decode_entity :: proc(entity: string) -> (decoded: rune, ok: bool) {
entity := entity
if len(entity) == 0 { return -1, false }
switch entity[0] {
case '#':
base := 10
val := 0
entity = entity[1:]
if len(entity) == 0 { return -1, false }
if entity[0] == 'x' || entity[0] == 'X' {
base = 16
entity = entity[1:]
}
for len(entity) > 0 {
r := entity[0]
switch r {
case '0'..='9':
val *= base
val += int(r - '0')
case 'a'..='f':
if base == 10 { return -1, false }
val *= base
val += int(r - 'a' + 10)
case 'A'..='F':
if base == 10 { return -1, false }
val *= base
val += int(r - 'A' + 10)
case:
return -1, false
}
if val > MAX_RUNE_CODEPOINT { return -1, false }
entity = entity[1:]
}
return rune(val), true
case:
/*
Named entity.
*/
return named_xml_entity_to_rune(entity)
}
}
/*
Private XML helper to extract `&<stuff>;` entity.
*/
@(private="file")
_extract_xml_entity :: proc(t: ^Tokenizer) -> (entity: string, err: Error) {
assert(t != nil && t.r == '&')
/*
All of these would be in the ASCII range.
Even if one is not, it doesn't matter. All characters we need to compare to extract are.
*/
using t
length := len(t.src)
found := false
#no_bounds_check {
for read_offset < length {
if src[read_offset] == ';' {
found = true
read_offset += 1
break
}
read_offset += 1
}
}
if found {
return string(src[offset + 1 : read_offset - 1]), .None
}
return string(src[offset : read_offset]), .Invalid_Entity_Encoding
}
/*
Private XML helper for CDATA and comments.
*/
@(private="file")
_handle_xml_special :: proc(t: ^Tokenizer, builder: ^strings.Builder, options: XML_Decode_Options) -> (in_data: bool, err: Error) {
assert(t != nil && t.r == '<')
if t.read_offset + len(CDATA_START) >= len(t.src) { return false, .None }
if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START {
t.read_offset += len(CDATA_START) - 1
if .Unbox_CDATA in options && .Decode_CDATA in options {
/*
We're unboxing _and_ decoding CDATA
*/
return true, .None
}
/*
CDATA is passed through.
*/
offset := t.offset
/*
Scan until end of CDATA.
*/
for {
advance(t) or_return
if t.r < 0 { return true, .CDATA_Not_Terminated }
if t.read_offset + len(CDATA_END) < len(t.src) {
if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END {
t.read_offset += len(CDATA_END) - 1
cdata := string(t.src[offset : t.read_offset])
if .Unbox_CDATA in options {
cdata = cdata[len(CDATA_START):]
cdata = cdata[:len(cdata) - len(CDATA_END)]
}
write_string(builder, cdata)
return false, .None
}
}
}
} else if string(t.src[t.offset:][:len(COMMENT_START)]) == COMMENT_START {
t.read_offset += len(COMMENT_START)
/*
Comment is passed through by default.
*/
offset := t.offset
/*
Scan until end of Comment.
*/
for {
advance(t) or_return
if t.r < 0 { return true, .Comment_Not_Terminated }
if t.read_offset + len(COMMENT_END) < len(t.src) {
if string(t.src[t.offset:][:len(COMMENT_END)]) == COMMENT_END {
t.read_offset += len(COMMENT_END) - 1
if .Comment_Strip not_in options {
comment := string(t.src[offset : t.read_offset])
write_string(builder, comment)
}
return false, .None
}
}
}
}
return false, .None
}

76
core/encoding/entity/example/entity_example.odin Normal file
View File

@@ -0,0 +1,76 @@
package unicode_entity_example
import "core:encoding/xml"
import "core:strings"
import "core:mem"
import "core:fmt"
import "core:time"
doc_print :: proc(doc: ^xml.Document) {
buf: strings.Builder
defer strings.builder_destroy(&buf)
w := strings.to_writer(&buf)
xml.print(w, doc)
fmt.println(strings.to_string(buf))
}
_entities :: proc() {
doc: ^xml.Document
err: xml.Error
DOC :: #load("../../../../tests/core/assets/XML/unicode.xml")
OPTIONS :: xml.Options{
flags = {
.Ignore_Unsupported, .Intern_Comments,
},
expected_doctype = "",
}
parse_duration: time.Duration
{
time.SCOPED_TICK_DURATION(&parse_duration)
doc, err = xml.parse(DOC, OPTIONS)
}
defer xml.destroy(doc)
doc_print(doc)
ms := time.duration_milliseconds(parse_duration)
speed := (f64(1000.0) / ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
fmt.printf("Parse time: %.2f ms (%.2f MiB/s).\n", ms, speed)
fmt.printf("Error: %v\n", err)
}
_main :: proc() {
using fmt
options := xml.Options{ flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities }}
doc, _ := xml.parse(#load("test.html"), options)
defer xml.destroy(doc)
doc_print(doc)
}
main :: proc() {
using fmt
track: mem.Tracking_Allocator
mem.tracking_allocator_init(&track, context.allocator)
context.allocator = mem.tracking_allocator(&track)
// _main()
_entities()
if len(track.allocation_map) > 0 {
println()
for _, v in track.allocation_map {
printf("%v Leaked %v bytes.\n", v.location, v.size)
}
}
}

28
core/encoding/entity/example/test.html Normal file
View File

@@ -0,0 +1,28 @@
<html>
<head>
<title>Entity Reference Test</title>
<style>
body {
background: #000; color: #eee;
width: 40%;
margin-left: auto;
margin-right: auto;
font-size: 14pt;
}
</style>
</head>
<body>
<h1>Entity Reference Test</h1>
<div id="test_cdata_in_comment" foo="">
Foozle]!&#32;&copy;&#x20;<!-- <![CDATA[&#32;&reg;&#x20;]]> -->42&;1234&
</div>
<!-- EXPECTED: Foozle]! © 42&;1234& -->
<div id="test_cdata_unwrap_and_passthrough">
Foozle]!&#32;&copy;&#x20;<![CDATA[BOX&#32;&reg;&#x20;/BOX]]>42&;1234&
</div>
<!-- EXPECTED: Foozle]! © BOX ® /BOX42&;1234& -->
<div>
&verbar; &vert; &VerticalLine; &fjlig; &grave; &bsol; &reg; &rhov; &CounterClockwiseContourIntegral; &bsemi;
</div>
</body>
</html>

7493
core/encoding/entity/generated.odin Normal file
View File

File diff suppressed because it is too large Load Diff

29
core/encoding/hxa/read.odin
View File

@@ -39,6 +39,9 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
read_value :: proc(r: ^Reader, $T: typeid) -> (value: T, err: Read_Error) {
remaining := len(r.data) - r.offset
if remaining < size_of(T) {
if r.print_error {
fmt.eprintf("file '%s' failed to read value at offset %v\n", r.filename, r.offset)
}
err = .Short_Read
return
}
@@ -51,6 +54,10 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
read_array :: proc(r: ^Reader, $T: typeid, count: int) -> (value: []T, err: Read_Error) {
remaining := len(r.data) - r.offset
if remaining < size_of(T)*count {
if r.print_error {
fmt.eprintf("file '%s' failed to read array of %d elements at offset %v\n",
r.filename, count, r.offset)
}
err = .Short_Read
return
}
@@ -82,7 +89,8 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
type := read_value(r, Meta_Value_Type) or_return
if type > max(Meta_Value_Type) {
if r.print_error {
fmt.eprintf("HxA Error: file '%s' has meta value type %d. Maximum value is ", r.filename, u8(type), u8(max(Meta_Value_Type)))
fmt.eprintf("HxA Error: file '%s' has meta value type %d. Maximum value is %d\n",
r.filename, u8(type), u8(max(Meta_Value_Type)))
}
err = .Invalid_Data
return
@@ -114,7 +122,8 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
type := read_value(r, Layer_Data_Type) or_return
if type > max(type) {
if r.print_error {
fmt.eprintf("HxA Error: file '%s' has layer data type %d. Maximum value is ", r.filename, u8(type), u8(max(Layer_Data_Type)))
fmt.eprintf("HxA Error: file '%s' has layer data type %d. Maximum value is %d\n",
r.filename, u8(type), u8(max(Layer_Data_Type)))
}
err = .Invalid_Data
return
@@ -134,13 +143,23 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
}
if len(data) < size_of(Header) {
if print_error {
fmt.eprintf("HxA Error: file '%s' has no header\n", filename)
}
err = .Short_Read
return
}
context.allocator = allocator
header := cast(^Header)raw_data(data)
assert(header.magic_number == MAGIC_NUMBER)
if (header.magic_number != MAGIC_NUMBER) {
if print_error {
fmt.eprintf("HxA Error: file '%s' has invalid magic number 0x%x\n", filename, header.magic_number)
}
err = .Invalid_Data
return
}
r := &Reader{
filename = filename,
@@ -150,6 +169,7 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
}
node_count := 0
file.header = header^
file.nodes = make([]Node, header.internal_node_count)
defer if err != nil {
nodes_destroy(file.nodes)
@@ -162,7 +182,8 @@ read :: proc(data: []byte, filename := "<input>", print_error := false, allocato
type := read_value(r, Node_Type) or_return
if type > max(Node_Type) {
if r.print_error {
fmt.eprintf("HxA Error: file '%s' has node type %d. Maximum value is ", r.filename, u8(type), u8(max(Node_Type)))
fmt.eprintf("HxA Error: file '%s' has node type %d. Maximum value is %d\n",
r.filename, u8(type), u8(max(Node_Type)))
}
err = .Invalid_Data
return

6
core/encoding/hxa/write.odin
View File

@@ -84,7 +84,7 @@ write_internal :: proc(w: ^Writer, file: File) {
write_metadata :: proc(w: ^Writer, meta_data: []Meta) {
for m in meta_data {
name_len := max(len(m.name), 255)
name_len := min(len(m.name), 255)
write_value(w, u8(name_len))
write_string(w, m.name[:name_len])
@@ -127,7 +127,7 @@ write_internal :: proc(w: ^Writer, file: File) {
write_layer_stack :: proc(w: ^Writer, layers: Layer_Stack) {
write_value(w, u32(len(layers)))
for layer in layers {
name_len := max(len(layer.name), 255)
name_len := min(len(layer.name), 255)
write_value(w, u8(name_len))
write_string(w, layer .name[:name_len])
@@ -152,7 +152,7 @@ write_internal :: proc(w: ^Writer, file: File) {
return
}
write_value(w, &Header{
write_value(w, Header{
magic_number = MAGIC_NUMBER,
version = LATEST_VERSION,
internal_node_count = u32le(len(file.nodes)),

13
core/encoding/json/marshal.odin
View File

@@ -8,18 +8,19 @@ import "core:strings"
import "core:io"
Marshal_Data_Error :: enum {
None,
Unsupported_Type,
}
Marshal_Error :: union {
Marshal_Error :: union #shared_nil {
Marshal_Data_Error,
io.Error,
}
marshal :: proc(v: any, allocator := context.allocator) -> (data: []byte, err: Marshal_Error) {
b := strings.make_builder(allocator)
defer if err != .None {
strings.destroy_builder(&b)
b := strings.builder_make(allocator)
defer if err != nil {
strings.builder_destroy(&b)
}
marshal_to_builder(&b, v) or_return
@@ -27,7 +28,7 @@ marshal :: proc(v: any, allocator := context.allocator) -> (data: []byte, err: M
if len(b.buf) != 0 {
data = b.buf[:]
}
return data, .None
return data, nil
}
marshal_to_builder :: proc(b: ^strings.Builder, v: any) -> Marshal_Error {
@@ -48,7 +49,7 @@ marshal_to_writer :: proc(w: io.Writer, v: any) -> (err: Marshal_Error) {
unreachable()
case runtime.Type_Info_Integer:
buf: [21]byte
buf: [40]byte
u: u128
switch i in a {
case i8: u = u128(i)

8
core/encoding/json/parser.odin
View File

@@ -40,7 +40,7 @@ parse_string :: proc(data: string, spec := DEFAULT_SPECIFICATION, parse_integers
return parse_object(&p)
case .JSON5:
return parse_value(&p)
case .MJSON:
case .SJSON:
#partial switch p.curr_token.kind {
case .Ident, .String:
return parse_object_body(&p, .EOF)
@@ -354,6 +354,12 @@ unquote_string :: proc(token: Token, spec: Specification, allocator := context.a
b := bytes_make(len(s) + 2*utf8.UTF_MAX, 1, allocator) or_return
w := copy(b, s[0:i])
if len(b) == 0 && allocator.data == nil {
// `unmarshal_count_array` calls us with a nil allocator
return string(b[:w]), nil
}
loop: for i < len(s) {
c := s[i]
switch {

5
core/encoding/json/types.odin
View File

@@ -33,8 +33,9 @@ package json
Specification :: enum {
JSON,
JSON5, // https://json5.org/
MJSON, // https://bitsquid.blogspot.com/2009/10/simplified-json-notation.html
Bitsquid = MJSON,
SJSON, // https://bitsquid.blogspot.com/2009/10/simplified-json-notation.html
Bitsquid = SJSON,
MJSON = SJSON,
}

6
core/encoding/json/unmarshal.odin
View File

@@ -209,7 +209,7 @@ unmarshal_value :: proc(p: ^Parser, v: any) -> (err: Unmarshal_Error) {
variant := u.variants[0]
v.id = variant.id
ti = reflect.type_info_base(variant)
if !(u.maybe && reflect.is_pointer(variant)) {
if !reflect.is_pointer_internally(variant) {
tag := any{rawptr(uintptr(v.data) + u.tag_offset), u.tag_type.id}
assign_int(tag, 1)
}
@@ -325,7 +325,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm
UNSUPPORTED_TYPE := Unsupported_Type_Error{v.id, p.curr_token}
if end_token == .Close_Brace {
assert(expect_token(p, .Open_Brace) == nil)
unmarshal_expect_token(p, .Open_Brace)
}
v := v
@@ -473,7 +473,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm
}
if end_token == .Close_Brace {
assert(expect_token(p, .Close_Brace) == nil)
unmarshal_expect_token(p, .Close_Brace)
}
return
}

28
core/encoding/varint/doc.odin Normal file
View File

@@ -0,0 +1,28 @@
/*
Implementation of the LEB128 variable integer encoding as used by DWARF encoding and DEX files, among others.
Author of this Odin package: Jeroen van Rijn
Example:
```odin
import "core:encoding/varint"
import "core:fmt"
main :: proc() {
buf: [varint.LEB128_MAX_BYTES]u8
value := u128(42)
encode_size, encode_err := varint.encode_uleb128(buf[:], value)
assert(encode_size == 1 && encode_err == .None)
fmt.printf("Encoded as %v\n", buf[:encode_size])
decoded_val, decode_size, decode_err := varint.decode_uleb128(buf[:])
assert(decoded_val == value && decode_size == encode_size && decode_err == .None)
fmt.printf("Decoded as %v, using %v byte%v\n", decoded_val, decode_size, "" if decode_size == 1 else "s")
}
```
*/
package varint

163
core/encoding/varint/leb128.odin Normal file
View File

@@ -0,0 +1,163 @@
/*
Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
List of contributors:
Jeroen van Rijn: Initial implementation.
*/
// package varint implements variable length integer encoding and decoding using
// the LEB128 format as used by DWARF debug info, Android .dex and other file formats.
package varint
// In theory we should use the bigint package. In practice, varints bigger than this indicate a corrupted file.
// Instead we'll set limits on the values we'll encode/decode
// 18 * 7 bits = 126, which means that a possible 19th byte may at most be `0b0000_0011`.
LEB128_MAX_BYTES :: 19
Error :: enum {
None = 0,
Buffer_Too_Small = 1,
Value_Too_Large = 2,
}
// Decode a slice of bytes encoding an unsigned LEB128 integer into value and number of bytes used.
// Returns `size` == 0 for an invalid value, empty slice, or a varint > 18 bytes.
decode_uleb128_buffer :: proc(buf: []u8) -> (val: u128, size: int, err: Error) {
if len(buf) == 0 {
return 0, 0, .Buffer_Too_Small
}
for v in buf {
val, size, err = decode_uleb128_byte(v, size, val)
if err != .Buffer_Too_Small {
return
}
}
if err == .Buffer_Too_Small {
val, size = 0, 0
}
return
}
// Decodes an unsigned LEB128 integer into value a byte at a time.
// Returns `.None` when decoded properly, `.Value_Too_Large` when they value
// exceeds the limits of a u128, and `.Buffer_Too_Small` when it's not yet fully decoded.
decode_uleb128_byte :: proc(input: u8, offset: int, accumulator: u128) -> (val: u128, size: int, err: Error) {
size = offset + 1
// 18 * 7 bits = 126, which means that a possible 19th byte may at most be 0b0000_0011.
if size > LEB128_MAX_BYTES || size == LEB128_MAX_BYTES && input > 0b0000_0011 {
return 0, 0, .Value_Too_Large
}
val = accumulator | u128(input & 0x7f) << uint(offset * 7)
if input < 128 {
// We're done
return
}
// If the buffer runs out before the number ends, return an error.
return val, size, .Buffer_Too_Small
}
decode_uleb128 :: proc {decode_uleb128_buffer, decode_uleb128_byte}
// Decode a slice of bytes encoding a signed LEB128 integer into value and number of bytes used.
// Returns `size` == 0 for an invalid value, empty slice, or a varint > 18 bytes.
decode_ileb128_buffer :: proc(buf: []u8) -> (val: i128, size: int, err: Error) {
if len(buf) == 0 {
return 0, 0, .Buffer_Too_Small
}
for v in buf {
val, size, err = decode_ileb128_byte(v, size, val)
if err != .Buffer_Too_Small {
return
}
}
if err == .Buffer_Too_Small {
val, size = 0, 0
}
return
}
// Decode a a signed LEB128 integer into value and number of bytes used, one byte at a time.
// Returns `size` == 0 for an invalid value, empty slice, or a varint > 18 bytes.
decode_ileb128_byte :: proc(input: u8, offset: int, accumulator: i128) -> (val: i128, size: int, err: Error) {
size = offset + 1
shift := uint(offset * 7)
// 18 * 7 bits = 126, which including sign means we can have a 19th byte.
if size > LEB128_MAX_BYTES || size == LEB128_MAX_BYTES && input > 0x7f {
return 0, 0, .Value_Too_Large
}
val = accumulator | i128(input & 0x7f) << shift
if input < 128 {
if input & 0x40 == 0x40 {
val |= max(i128) << (shift + 7)
}
return val, size, .None
}
return val, size, .Buffer_Too_Small
}
decode_ileb128 :: proc{decode_ileb128_buffer, decode_ileb128_byte}
// Encode `val` into `buf` as an unsigned LEB128 encoded series of bytes.
// `buf` must be appropriately sized.
encode_uleb128 :: proc(buf: []u8, val: u128) -> (size: int, err: Error) {
val := val
for {
size += 1
if size > len(buf) {
return 0, .Buffer_Too_Small
}
low := val & 0x7f
val >>= 7
if val > 0 {
low |= 0x80 // more bytes to follow
}
buf[size - 1] = u8(low)
if val == 0 { break }
}
return
}
// Encode `val` into `buf` as a signed LEB128 encoded series of bytes.
// `buf` must be appropriately sized.
encode_ileb128 :: proc(buf: []u8, val: i128) -> (size: int, err: Error) {
SIGN_MASK :: i128(1) << 121 // sign extend mask
val, more := val, true
for more {
size += 1
if size > len(buf) {
return 0, .Buffer_Too_Small
}
low := val & 0x7f
val >>= 7
low = (low ~ SIGN_MASK) - SIGN_MASK
if (val == 0 && low & 0x40 != 0x40) || (val == -1 && low & 0x40 == 0x40) {
more = false
} else {
low |= 0x80
}
buf[size - 1] = u8(low)
}
return
}

86
core/encoding/xml/debug_print.odin Normal file
View File

@@ -0,0 +1,86 @@
/*
An XML 1.0 / 1.1 parser
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816).
List of contributors:
Jeroen van Rijn: Initial implementation.
*/
package xml
import "core:io"
import "core:fmt"
/*
Just for debug purposes.
*/
print :: proc(writer: io.Writer, doc: ^Document) -> (written: int, err: io.Error) {
if doc == nil { return }
using fmt
written += wprintf(writer, "[XML Prolog]\n")
for attr in doc.prologue {
written += wprintf(writer, "\t%v: %v\n", attr.key, attr.val)
}
written += wprintf(writer, "[Encoding] %v\n", doc.encoding)
if len(doc.doctype.ident) > 0 {
written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident)
if len(doc.doctype.rest) > 0 {
wprintf(writer, "\t%v\n", doc.doctype.rest)
}
}
for comment in doc.comments {
written += wprintf(writer, "[Pre-root comment] %v\n", comment)
}
if len(doc.elements) > 0 {
wprintln(writer, " --- ")
print_element(writer, doc, 0)
wprintln(writer, " --- ")
}
return written, .None
}
print_element :: proc(writer: io.Writer, doc: ^Document, element_id: Element_ID, indent := 0) -> (written: int, err: io.Error) {
using fmt
tab :: proc(writer: io.Writer, indent: int) {
for _ in 0..=indent {
wprintf(writer, "\t")
}
}
tab(writer, indent)
element := doc.elements[element_id]
if element.kind == .Element {
wprintf(writer, "<%v>\n", element.ident)
if len(element.value) > 0 {
tab(writer, indent + 1)
wprintf(writer, "[Value] %v\n", element.value)
}
for attr in element.attribs {
tab(writer, indent + 1)
wprintf(writer, "[Attr] %v: %v\n", attr.key, attr.val)
}
for child in element.children {
print_element(writer, doc, child, indent + 1)
}
} else if element.kind == .Comment {
wprintf(writer, "[COMMENT] %v\n", element.value)
}
return written, .None
}

112
core/encoding/xml/example/xml_example.odin Normal file
View File

@@ -0,0 +1,112 @@
package xml_example
import "core:encoding/xml"
import "core:mem"
import "core:fmt"
import "core:time"
import "core:strings"
import "core:hash"
N :: 1
example :: proc() {
using fmt
docs: [N]^xml.Document
errs: [N]xml.Error
times: [N]time.Duration
defer for round in 0..<N {
xml.destroy(docs[round])
}
DOC :: #load("../../../../tests/core/assets/XML/unicode.xml")
input := DOC
for round in 0..<N {
start := time.tick_now()
docs[round], errs[round] = xml.parse(input, xml.Options{
flags={.Ignore_Unsupported},
expected_doctype = "",
})
end := time.tick_now()
times[round] = time.tick_diff(start, end)
}
fastest := max(time.Duration)
slowest := time.Duration(0)
total := time.Duration(0)
for round in 0..<N {
fastest = min(fastest, times[round])
slowest = max(slowest, times[round])
total += times[round]
}
fastest_ms := time.duration_milliseconds(fastest)
slowest_ms := time.duration_milliseconds(slowest)
average_ms := time.duration_milliseconds(time.Duration(f64(total) / f64(N)))
fastest_speed := (f64(1000.0) / fastest_ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
slowest_speed := (f64(1000.0) / slowest_ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
average_speed := (f64(1000.0) / average_ms) * f64(len(DOC)) / 1_024.0 / 1_024.0
fmt.printf("N = %v\n", N)
fmt.printf("[Fastest]: %v bytes in %.2f ms (%.2f MiB/s).\n", len(input), fastest_ms, fastest_speed)
fmt.printf("[Slowest]: %v bytes in %.2f ms (%.2f MiB/s).\n", len(input), slowest_ms, slowest_speed)
fmt.printf("[Average]: %v bytes in %.2f ms (%.2f MiB/s).\n", len(input), average_ms, average_speed)
if errs[0] != .None {
printf("Load/Parse error: %v\n", errs[0])
if errs[0] == .File_Error {
println("\"unicode.xml\" not found. Did you run \"tests\\download_assets.py\"?")
}
return
}
charlist, charlist_ok := xml.find_child_by_ident(docs[0], 0, "charlist")
if !charlist_ok {
eprintln("Could not locate top-level `<charlist>` tag.")
return
}
printf("Found `<charlist>` with %v children, %v elements total\n", len(docs[0].elements[charlist].children), docs[0].element_count)
crc32 := doc_hash(docs[0])
printf("[%v] CRC32: 0x%08x\n", "🎉" if crc32 == 0xcaa042b9 else "🤬", crc32)
for round in 0..<N {
defer xml.destroy(docs[round])
}
}
doc_hash :: proc(doc: ^xml.Document, print := false) -> (crc32: u32) {
buf: strings.Builder
defer strings.builder_destroy(&buf)
w := strings.to_writer(&buf)
xml.print(w, doc)
tree := strings.to_string(buf)
if print { fmt.println(tree) }
return hash.crc32(transmute([]u8)tree)
}
main :: proc() {
using fmt
track: mem.Tracking_Allocator
mem.tracking_allocator_init(&track, context.allocator)
context.allocator = mem.tracking_allocator(&track)
example()
if len(track.allocation_map) > 0 {
println()
for _, v in track.allocation_map {
printf("%v Leaked %v bytes.\n", v.location, v.size)
}
}
println("Done and cleaned up!")
}

45
core/encoding/xml/helpers.odin Normal file
View File

@@ -0,0 +1,45 @@
/*
An XML 1.0 / 1.1 parser
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
This file contains helper functions.
*/
package xml
// Find parent's nth child with a given ident.
find_child_by_ident :: proc(doc: ^Document, parent_id: Element_ID, ident: string, nth := 0) -> (res: Element_ID, found: bool) {
tag := doc.elements[parent_id]
count := 0
for child_id in tag.children {
child := doc.elements[child_id]
/*
Skip commments. They have no name.
*/
if child.kind != .Element { continue }
/*
If the ident matches and it's the nth such child, return it.
*/
if child.ident == ident {
if count == nth { return child_id, true }
count += 1
}
}
return 0, false
}
// Find an attribute by key.
find_attribute_val_by_key :: proc(doc: ^Document, parent_id: Element_ID, key: string) -> (val: string, found: bool) {
tag := doc.elements[parent_id]
for attr in tag.attribs {
/*
If the ident matches, we're done. There can only ever be one attribute with the same name.
*/
if attr.key == key { return attr.val, true }
}
return "", false
}

436
core/encoding/xml/tokenizer.odin Normal file
View File

@@ -0,0 +1,436 @@
/*
An XML 1.0 / 1.1 parser
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816).
List of contributors:
Jeroen van Rijn: Initial implementation.
*/
package xml
import "core:fmt"
import "core:unicode"
import "core:unicode/utf8"
Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any)
Token :: struct {
kind: Token_Kind,
text: string,
pos: Pos,
}
Pos :: struct {
file: string,
offset: int, // starting at 0
line: int, // starting at 1
column: int, // starting at 1
}
Token_Kind :: enum {
Invalid,
Ident,
Literal,
Rune,
String,
Double_Quote, // "
Single_Quote, // '
Colon, // :
Eq, // =
Lt, // <
Gt, // >
Exclaim, // !
Question, // ?
Hash, // #
Slash, // /
Dash, // -
Open_Bracket, // [
Close_Bracket, // ]
EOF,
}
CDATA_START :: "<![CDATA["
CDATA_END :: "]]>"
COMMENT_START :: "<!--"
COMMENT_END :: "-->"
Tokenizer :: struct {
// Immutable data
path: string,
src: string,
err: Error_Handler,
// Tokenizing state
ch: rune,
offset: int,
read_offset: int,
line_offset: int,
line_count: int,
// Mutable data
error_count: int,
}
init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {
t.src = src
t.err = err
t.ch = ' '
t.offset = 0
t.read_offset = 0
t.line_offset = 0
t.line_count = len(src) > 0 ? 1 : 0
t.error_count = 0
t.path = path
advance_rune(t)
if t.ch == utf8.RUNE_BOM {
advance_rune(t)
}
}
@(private)
offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> Pos {
line := t.line_count
column := offset - t.line_offset + 1
return Pos {
file = t.path,
offset = offset,
line = line,
column = column,
}
}
default_error_handler :: proc(pos: Pos, msg: string, args: ..any) {
fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column)
fmt.eprintf(msg, ..args)
fmt.eprintf("\n")
}
error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
pos := offset_to_pos(t, offset)
if t.err != nil {
t.err(pos, msg, ..args)
}
t.error_count += 1
}
@(optimization_mode="speed")
advance_rune :: proc(using t: ^Tokenizer) {
#no_bounds_check {
/*
Already bounds-checked here.
*/
if read_offset < len(src) {
offset = read_offset
if ch == '\n' {
line_offset = offset
line_count += 1
}
r, w := rune(src[read_offset]), 1
switch {
case r == 0:
error(t, t.offset, "illegal character NUL")
case r >= utf8.RUNE_SELF:
r, w = #force_inline utf8.decode_rune_in_string(src[read_offset:])
if r == utf8.RUNE_ERROR && w == 1 {
error(t, t.offset, "illegal UTF-8 encoding")
} else if r == utf8.RUNE_BOM && offset > 0 {
error(t, t.offset, "illegal byte order mark")
}
}
read_offset += w
ch = r
} else {
offset = len(src)
if ch == '\n' {
line_offset = offset
line_count += 1
}
ch = -1
}
}
}
peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte {
if t.read_offset+offset < len(t.src) {
#no_bounds_check return t.src[t.read_offset+offset]
}
return 0
}
@(optimization_mode="speed")
skip_whitespace :: proc(t: ^Tokenizer) {
for {
switch t.ch {
case ' ', '\t', '\r', '\n':
advance_rune(t)
case:
return
}
}
}
@(optimization_mode="speed")
is_letter :: proc(r: rune) -> bool {
if r < utf8.RUNE_SELF {
switch r {
case '_':
return true
case 'A'..='Z', 'a'..='z':
return true
}
}
return unicode.is_letter(r)
}
is_valid_identifier_rune :: proc(r: rune) -> bool {
if r < utf8.RUNE_SELF {
switch r {
case '_', '-', ':': return true
case 'A'..='Z', 'a'..='z': return true
case '0'..='9': return true
case -1: return false
}
}
if unicode.is_letter(r) || unicode.is_digit(r) {
return true
}
return false
}
scan_identifier :: proc(t: ^Tokenizer) -> string {
offset := t.offset
namespaced := false
for is_valid_identifier_rune(t.ch) {
advance_rune(t)
if t.ch == ':' {
/*
A namespaced attr can have at most two parts, `namespace:ident`.
*/
if namespaced {
break
}
namespaced = true
}
}
return string(t.src[offset : t.offset])
}
/*
A comment ends when we see -->, preceded by a character that's not a dash.
"For compatibility, the string "--" (double-hyphen) must not occur within comments."
See: https://www.w3.org/TR/2006/REC-xml11-20060816/#dt-comment
Thanks to the length (4) of the comment start, we also have enough lookback,
and the peek at the next byte asserts that there's at least one more character
that's a `>`.
*/
scan_comment :: proc(t: ^Tokenizer) -> (comment: string, err: Error) {
offset := t.offset
for {
advance_rune(t)
ch := t.ch
if ch < 0 {
error(t, offset, "[parse] Comment was not terminated\n")
return "", .Unclosed_Comment
}
if string(t.src[t.offset - 1:][:2]) == "--" {
if peek_byte(t) == '>' {
break
} else {
error(t, t.offset - 1, "Invalid -- sequence in comment.\n")
return "", .Invalid_Sequence_In_Comment
}
}
}
expect(t, .Dash)
expect(t, .Gt)
return string(t.src[offset : t.offset - 1]), .None
}
/*
Skip CDATA
*/
skip_cdata :: proc(t: ^Tokenizer) -> (err: Error) {
if t.read_offset + len(CDATA_START) >= len(t.src) {
/*
Can't be the start of a CDATA tag.
*/
return .None
}
if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START {
t.read_offset += len(CDATA_START)
offset := t.offset
cdata_scan: for {
advance_rune(t)
if t.ch < 0 {
error(t, offset, "[scan_string] CDATA was not terminated\n")
return .Premature_EOF
}
/*
Scan until the end of a CDATA tag.
*/
if t.read_offset + len(CDATA_END) < len(t.src) {
if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END {
t.read_offset += len(CDATA_END)
break cdata_scan
}
}
}
}
return
}
@(optimization_mode="speed")
scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) {
err = .None
loop: for {
ch := t.ch
switch ch {
case -1:
error(t, t.offset, "[scan_string] Premature end of file.\n")
return "", .Premature_EOF
case '<':
if peek_byte(t) == '!' {
if peek_byte(t, 1) == '[' {
/*
Might be the start of a CDATA tag.
*/
skip_cdata(t) or_return
} else if peek_byte(t, 1) == '-' && peek_byte(t, 2) == '-' {
/*
Comment start. Eat comment.
*/
t.read_offset += 3
_ = scan_comment(t) or_return
}
}
case '\n':
if !multiline {
error(t, offset, string(t.src[offset : t.offset]))
error(t, offset, "[scan_string] Not terminated\n")
err = .Invalid_Tag_Value
break loop
}
}
if t.ch == close {
/*
If it's not a CDATA or comment, it's the end of this body.
*/
break loop
}
advance_rune(t)
}
/*
Strip trailing whitespace.
*/
lit := string(t.src[offset : t.offset])
end := len(lit)
eat: for ; end > 0; end -= 1 {
ch := lit[end - 1]
switch ch {
case ' ', '\t', '\r', '\n':
case:
break eat
}
}
lit = lit[:end]
if consume_close {
advance_rune(t)
}
/*
TODO: Handle decoding escape characters and unboxing CDATA.
*/
return lit, err
}
peek :: proc(t: ^Tokenizer) -> (token: Token) {
old := t^
token = scan(t)
t^ = old
return token
}
scan :: proc(t: ^Tokenizer) -> Token {
skip_whitespace(t)
offset := t.offset
kind: Token_Kind
err: Error
lit: string
pos := offset_to_pos(t, offset)
switch ch := t.ch; true {
case is_letter(ch):
lit = scan_identifier(t)
kind = .Ident
case:
advance_rune(t)
switch ch {
case -1:
kind = .EOF
case '<': kind = .Lt
case '>': kind = .Gt
case '!': kind = .Exclaim
case '?': kind = .Question
case '=': kind = .Eq
case '#': kind = .Hash
case '/': kind = .Slash
case '-': kind = .Dash
case ':': kind = .Colon
case '"', '\'':
kind = .Invalid
lit, err = scan_string(t, t.offset, ch, true, false)
if err == .None {
kind = .String
}
case '\n':
lit = "\n"
case:
kind = .Invalid
}
}
if kind != .String && lit == "" {
lit = string(t.src[offset : t.offset])
}
return Token{kind, lit, pos}
}

713
core/encoding/xml/xml_reader.odin Normal file
View File

@@ -0,0 +1,713 @@
/*
An XML 1.0 / 1.1 parser
Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
Made available under Odin's BSD-3 license.
A from-scratch XML implementation, loosely modelled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816).
Features:
- Supports enough of the XML 1.0/1.1 spec to handle the 99.9% of XML documents in common current usage.
- Simple to understand and use. Small.
Caveats:
- We do NOT support HTML in this package, as that may or may not be valid XML.
If it works, great. If it doesn't, that's not considered a bug.
- We do NOT support UTF-16. If you have a UTF-16 XML file, please convert it to UTF-8 first. Also, our condolences.
- <[!ELEMENT and <[!ATTLIST are not supported, and will be either ignored or return an error depending on the parser options.
MAYBE:
- XML writer?
- Serialize/deserialize Odin types?
List of contributors:
Jeroen van Rijn: Initial implementation.
*/
package xml
// An XML 1.0 / 1.1 parser
import "core:bytes"
import "core:encoding/entity"
import "core:intrinsics"
import "core:mem"
import "core:os"
import "core:strings"
likely :: intrinsics.expect
DEFAULT_OPTIONS :: Options{
flags = {.Ignore_Unsupported},
expected_doctype = "",
}
Option_Flag :: enum {
/*
If the caller says that input may be modified, we can perform in-situ parsing.
If this flag isn't provided, the XML parser first duplicates the input so that it can.
*/
Input_May_Be_Modified,
/*
Document MUST start with `<?xml` prologue.
*/
Must_Have_Prolog,
/*
Document MUST have a `<!DOCTYPE`.
*/
Must_Have_DocType,
/*
By default we skip comments. Use this option to intern a comment on a parented Element.
*/
Intern_Comments,
/*
How to handle unsupported parts of the specification, like <! other than <!DOCTYPE and <![CDATA[
*/
Error_on_Unsupported,
Ignore_Unsupported,
/*
By default CDATA tags are passed-through as-is.
This option unwraps them when encountered.
*/
Unbox_CDATA,
/*
By default SGML entities like `&gt;`, `&#32;` and `&#x20;` are passed-through as-is.
This option decodes them when encountered.
*/
Decode_SGML_Entities,
/*
If a tag body has a comment, it will be stripped unless this option is given.
*/
Keep_Tag_Body_Comments,
}
Option_Flags :: bit_set[Option_Flag; u16]
Document :: struct {
elements: [dynamic]Element,
element_count: Element_ID,
prologue: Attributes,
encoding: Encoding,
doctype: struct {
/*
We only scan the <!DOCTYPE IDENT part and skip the rest.
*/
ident: string,
rest: string,
},
/*
If we encounter comments before the root node, and the option to intern comments is given, this is where they'll live.
Otherwise they'll be in the element tree.
*/
comments: [dynamic]string,
/*
Internal
*/
tokenizer: ^Tokenizer,
allocator: mem.Allocator,
/*
Input. Either the original buffer, or a copy if `.Input_May_Be_Modified` isn't specified.
*/
input: []u8,
strings_to_free: [dynamic]string,
}
Element :: struct {
ident: string,
value: string,
attribs: Attributes,
kind: enum {
Element = 0,
Comment,
},
parent: Element_ID,
children: [dynamic]Element_ID,
}
Attribute :: struct {
key: string,
val: string,
}
Attributes :: [dynamic]Attribute
Options :: struct {
flags: Option_Flags,
expected_doctype: string,
}
Encoding :: enum {
Unknown,
UTF_8,
ISO_8859_1,
/*
Aliases
*/
LATIN_1 = ISO_8859_1,
}
Error :: enum {
/*
General return values.
*/
None = 0,
General_Error,
Unexpected_Token,
Invalid_Token,
/*
Couldn't find, open or read file.
*/
File_Error,
/*
File too short.
*/
Premature_EOF,
/*
XML-specific errors.
*/
No_Prolog,
Invalid_Prolog,
Too_Many_Prologs,
No_DocType,
Too_Many_DocTypes,
DocType_Must_Preceed_Elements,
/*
If a DOCTYPE is present _or_ the caller
asked for a specific DOCTYPE and the DOCTYPE
and root tag don't match, we return `.Invalid_DocType`.
*/
Invalid_DocType,
Invalid_Tag_Value,
Mismatched_Closing_Tag,
Unclosed_Comment,
Comment_Before_Root_Element,
Invalid_Sequence_In_Comment,
Unsupported_Version,
Unsupported_Encoding,
/*
<!FOO are usually skipped.
*/
Unhandled_Bang,
Duplicate_Attribute,
Conflicting_Options,
}
/*
Implementation starts here.
*/
parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
data := data
context.allocator = allocator
opts := validate_options(options) or_return
/*
If `.Input_May_Be_Modified` is not specified, we duplicate the input so that we can modify it in-place.
*/
if .Input_May_Be_Modified not_in opts.flags {
data = bytes.clone(data)
}
t := &Tokenizer{}
init(t, string(data), path, error_handler)
doc = new(Document)
doc.allocator = allocator
doc.tokenizer = t
doc.input = data
doc.elements = make([dynamic]Element, 1024, 1024, allocator)
// strings.intern_init(&doc.intern, allocator, allocator)
err = .Unexpected_Token
element, parent: Element_ID
tag_is_open := false
first_element := true
open: Token
/*
If a DOCTYPE is present, the root tag has to match.
If an expected DOCTYPE is given in options (i.e. it's non-empty), the DOCTYPE (if present) and root tag have to match.
*/
expected_doctype := options.expected_doctype
loop: for {
skip_whitespace(t)
// NOTE(Jeroen): This is faster as a switch.
switch t.ch {
case '<':
/*
Consume peeked `<`
*/
advance_rune(t)
open = scan(t)
// NOTE(Jeroen): We're not using a switch because this if-else chain ordered by likelihood is 2.5% faster at -o:size and -o:speed.
if likely(open.kind, Token_Kind.Ident) == .Ident {
/*
e.g. <odin - Start of new element.
*/
element = new_element(doc)
tag_is_open = true
if first_element {
/*
First element.
*/
parent = element
first_element = false
} else {
append(&doc.elements[parent].children, element)
}
doc.elements[element].parent = parent
doc.elements[element].ident = open.text
parse_attributes(doc, &doc.elements[element].attribs) or_return
/*
If a DOCTYPE is present _or_ the caller
asked for a specific DOCTYPE and the DOCTYPE
and root tag don't match, we return .Invalid_Root_Tag.
*/
if element == 0 { // Root tag?
if len(expected_doctype) > 0 && expected_doctype != open.text {
error(t, t.offset, "Root Tag doesn't match DOCTYPE. Expected: %v, got: %v\n", expected_doctype, open.text)
return doc, .Invalid_DocType
}
}
/*
One of these should follow:
- `>`, which means we've just opened this tag and expect a later element to close it.
- `/>`, which means this is an 'empty' or self-closing tag.
*/
end_token := scan(t)
#partial switch end_token.kind {
case .Gt:
/*
We're now the new parent.
*/
parent = element
case .Slash:
/*
Empty tag. Close it.
*/
expect(t, .Gt) or_return
parent = doc.elements[element].parent
element = parent
tag_is_open = false
case:
error(t, t.offset, "Expected close tag, got: %#v\n", end_token)
return
}
} else if open.kind == .Slash {
/*
Close tag.
*/
ident := expect(t, .Ident) or_return
_ = expect(t, .Gt) or_return
if doc.elements[element].ident != ident.text {
error(t, t.offset, "Mismatched Closing Tag. Expected %v, got %v\n", doc.elements[element].ident, ident.text)
return doc, .Mismatched_Closing_Tag
}
parent = doc.elements[element].parent
element = parent
tag_is_open = false
} else if open.kind == .Exclaim {
/*
<!
*/
next := scan(t)
#partial switch next.kind {
case .Ident:
switch next.text {
case "DOCTYPE":
if len(doc.doctype.ident) > 0 {
return doc, .Too_Many_DocTypes
}
if doc.element_count > 0 {
return doc, .DocType_Must_Preceed_Elements
}
parse_doctype(doc) or_return
if len(expected_doctype) > 0 && expected_doctype != doc.doctype.ident {
error(t, t.offset, "Invalid DOCTYPE. Expected: %v, got: %v\n", expected_doctype, doc.doctype.ident)
return doc, .Invalid_DocType
}
expected_doctype = doc.doctype.ident
case:
if .Error_on_Unsupported in opts.flags {
error(t, t.offset, "Unhandled: <!%v\n", next.text)
return doc, .Unhandled_Bang
}
skip_element(t) or_return
}
case .Dash:
/*
Comment: <!-- -->.
The grammar does not allow a comment to end in --->
*/
expect(t, .Dash)
comment := scan_comment(t) or_return
if .Intern_Comments in opts.flags {
if len(doc.elements) == 0 {
append(&doc.comments, comment)
} else {
el := new_element(doc)
doc.elements[el].parent = element
doc.elements[el].kind = .Comment
doc.elements[el].value = comment
append(&doc.elements[element].children, el)
}
}
case:
error(t, t.offset, "Invalid Token after <!. Expected .Ident, got %#v\n", next)
return
}
} else if open.kind == .Question {
/*
<?xml
*/
next := scan(t)
#partial switch next.kind {
case .Ident:
if len(next.text) == 3 && strings.to_lower(next.text, context.temp_allocator) == "xml" {
parse_prologue(doc) or_return
} else if len(doc.prologue) > 0 {
/*
We've already seen a prologue.
*/
return doc, .Too_Many_Prologs
} else {
/*
Could be `<?xml-stylesheet`, etc. Ignore it.
*/
skip_element(t) or_return
}
case:
error(t, t.offset, "Expected \"<?xml\", got \"<?%v\".", next.text)
return
}
} else {
error(t, t.offset, "Invalid Token after <: %#v\n", open)
return
}
case -1:
/*
End of file.
*/
if tag_is_open {
return doc, .Premature_EOF
}
break loop
case:
/*
This should be a tag's body text.
*/
body_text := scan_string(t, t.offset) or_return
needs_processing := .Unbox_CDATA in opts.flags
needs_processing |= .Decode_SGML_Entities in opts.flags
if !needs_processing {
doc.elements[element].value = body_text
continue
}
decode_opts := entity.XML_Decode_Options{}
if .Keep_Tag_Body_Comments not_in opts.flags {
decode_opts += { .Comment_Strip }
}
if .Decode_SGML_Entities not_in opts.flags {
decode_opts += { .No_Entity_Decode }
}
if .Unbox_CDATA in opts.flags {
decode_opts += { .Unbox_CDATA }
if .Decode_SGML_Entities in opts.flags {
decode_opts += { .Decode_CDATA }
}
}
decoded, decode_err := entity.decode_xml(body_text, decode_opts)
if decode_err == .None {
doc.elements[element].value = decoded
append(&doc.strings_to_free, decoded)
} else {
doc.elements[element].value = body_text
}
}
}
if .Must_Have_Prolog in opts.flags && len(doc.prologue) == 0 {
return doc, .No_Prolog
}
if .Must_Have_DocType in opts.flags && len(doc.doctype.ident) == 0 {
return doc, .No_DocType
}
resize(&doc.elements, int(doc.element_count))
return doc, .None
}
parse_string :: proc(data: string, options := DEFAULT_OPTIONS, path := "", error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
_data := transmute([]u8)data
return parse_bytes(_data, options, path, error_handler, allocator)
}
parse :: proc { parse_string, parse_bytes }
// Load an XML file
load_from_file :: proc(filename: string, options := DEFAULT_OPTIONS, error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
context.allocator = allocator
options := options
data, data_ok := os.read_entire_file(filename)
if !data_ok { return {}, .File_Error }
options.flags += { .Input_May_Be_Modified }
return parse_bytes(data, options, filename, error_handler, allocator)
}
destroy :: proc(doc: ^Document) {
if doc == nil { return }
for el in doc.elements {
delete(el.attribs)
delete(el.children)
}
delete(doc.elements)
delete(doc.prologue)
delete(doc.comments)
delete(doc.input)
for s in doc.strings_to_free {
delete(s)
}
delete(doc.strings_to_free)
free(doc)
}
/*
Helpers.
*/
validate_options :: proc(options: Options) -> (validated: Options, err: Error) {
validated = options
if .Error_on_Unsupported in validated.flags && .Ignore_Unsupported in validated.flags {
return options, .Conflicting_Options
}
return validated, .None
}
expect :: proc(t: ^Tokenizer, kind: Token_Kind) -> (tok: Token, err: Error) {
tok = scan(t)
if tok.kind == kind { return tok, .None }
error(t, t.offset, "Expected \"%v\", got \"%v\".", kind, tok.kind)
return tok, .Unexpected_Token
}
parse_attribute :: proc(doc: ^Document) -> (attr: Attribute, offset: int, err: Error) {
assert(doc != nil)
context.allocator = doc.allocator
t := doc.tokenizer
key := expect(t, .Ident) or_return
offset = t.offset - len(key.text)
_ = expect(t, .Eq) or_return
value := expect(t, .String) or_return
attr.key = key.text
attr.val = value.text
err = .None
return
}
check_duplicate_attributes :: proc(t: ^Tokenizer, attribs: Attributes, attr: Attribute, offset: int) -> (err: Error) {
for a in attribs {
if attr.key == a.key {
error(t, offset, "Duplicate attribute: %v\n", attr.key)
return .Duplicate_Attribute
}
}
return .None
}
parse_attributes :: proc(doc: ^Document, attribs: ^Attributes) -> (err: Error) {
assert(doc != nil)
context.allocator = doc.allocator
t := doc.tokenizer
for peek(t).kind == .Ident {
attr, offset := parse_attribute(doc) or_return
check_duplicate_attributes(t, attribs^, attr, offset) or_return
append(attribs, attr)
}
skip_whitespace(t)
return .None
}
parse_prologue :: proc(doc: ^Document) -> (err: Error) {
assert(doc != nil)
context.allocator = doc.allocator
t := doc.tokenizer
offset := t.offset
parse_attributes(doc, &doc.prologue) or_return
for attr in doc.prologue {
switch attr.key {
case "version":
switch attr.val {
case "1.0", "1.1":
case:
error(t, offset, "[parse_prologue] Warning: Unhandled XML version: %v\n", attr.val)
}
case "encoding":
switch strings.to_lower(attr.val, context.temp_allocator) {
case "utf-8", "utf8":
doc.encoding = .UTF_8
case "latin-1", "latin1", "iso-8859-1":
doc.encoding = .LATIN_1
case:
/*
Unrecognized encoding, assume UTF-8.
*/
error(t, offset, "[parse_prologue] Warning: Unrecognized encoding: %v\n", attr.val)
}
case:
// Ignored.
}
}
_ = expect(t, .Question) or_return
_ = expect(t, .Gt) or_return
return .None
}
skip_element :: proc(t: ^Tokenizer) -> (err: Error) {
close := 1
loop: for {
tok := scan(t)
#partial switch tok.kind {
case .EOF:
error(t, t.offset, "[skip_element] Premature EOF\n")
return .Premature_EOF
case .Lt:
close += 1
case .Gt:
close -= 1
if close == 0 {
break loop
}
case:
}
}
return .None
}
parse_doctype :: proc(doc: ^Document) -> (err: Error) {
/*
<!DOCTYPE greeting SYSTEM "hello.dtd">
<!DOCTYPE greeting [
<!ELEMENT greeting (#PCDATA)>
]>
*/
assert(doc != nil)
context.allocator = doc.allocator
t := doc.tokenizer
tok := expect(t, .Ident) or_return
doc.doctype.ident = tok.text
skip_whitespace(t)
offset := t.offset
skip_element(t) or_return
/*
-1 because the current offset is that of the closing tag, so the rest of the DOCTYPE tag ends just before it.
*/
doc.doctype.rest = string(t.src[offset : t.offset - 1])
return .None
}
Element_ID :: u32
new_element :: proc(doc: ^Document) -> (id: Element_ID) {
element_space := len(doc.elements)
// Need to resize
if int(doc.element_count) + 1 > element_space {
if element_space < 65536 {
element_space *= 2
} else {
element_space += 65536
}
resize(&doc.elements, element_space)
}
cur := doc.element_count
doc.element_count += 1
return cur
}

1442
core/fmt/fmt.odin
View File

File diff suppressed because it is too large Load Diff

7
core/fmt/fmt_js.odin
View File

@@ -34,11 +34,16 @@ stderr := io.Writer{
},
}
// print* procedures return the number of bytes written
// print formats using the default print settings and writes to stdout
print :: proc(args: ..any, sep := " ") -> int { return wprint(w=stdout, args=args, sep=sep) }
// println formats using the default print settings and writes to stdout
println :: proc(args: ..any, sep := " ") -> int { return wprintln(w=stdout, args=args, sep=sep) }
// printf formats according to the specififed format string and writes to stdout
printf :: proc(fmt: string, args: ..any) -> int { return wprintf(stdout, fmt, ..args) }
// eprint formats using the default print settings and writes to stderr
eprint :: proc(args: ..any, sep := " ") -> int { return wprint(w=stderr, args=args, sep=sep) }
// eprintln formats using the default print settings and writes to stderr
eprintln :: proc(args: ..any, sep := " ") -> int { return wprintln(w=stderr, args=args, sep=sep) }
// eprintf formats according to the specififed format string and writes to stderr
eprintf :: proc(fmt: string, args: ..any) -> int { return wprintf(stderr, fmt, ..args) }

10
core/fmt/fmt_os.odin
View File

@@ -5,15 +5,18 @@ import "core:runtime"
import "core:os"
import "core:io"
// fprint formats using the default print settings and writes to fd
fprint :: proc(fd: os.Handle, args: ..any, sep := " ") -> int {
w := io.to_writer(os.stream_from_handle(fd))
return wprint(w=w, args=args, sep=sep)
}
// fprintln formats using the default print settings and writes to fd
fprintln :: proc(fd: os.Handle, args: ..any, sep := " ") -> int {
w := io.to_writer(os.stream_from_handle(fd))
return wprintln(w=w, args=args, sep=sep)
}
// fprintf formats according to the specififed format string and writes to fd
fprintf :: proc(fd: os.Handle, fmt: string, args: ..any) -> int {
w := io.to_writer(os.stream_from_handle(fd))
return wprintf(w, fmt, ..args)
@@ -27,11 +30,16 @@ fprint_typeid :: proc(fd: os.Handle, id: typeid) -> (n: int, err: io.Error) {
return wprint_typeid(w, id)
}
// print* procedures return the number of bytes written
// print formats using the default print settings and writes to os.stdout
print :: proc(args: ..any, sep := " ") -> int { return fprint(fd=os.stdout, args=args, sep=sep) }
// println formats using the default print settings and writes to os.stdout
println :: proc(args: ..any, sep := " ") -> int { return fprintln(fd=os.stdout, args=args, sep=sep) }
// printf formats according to the specififed format string and writes to os.stdout
printf :: proc(fmt: string, args: ..any) -> int { return fprintf(os.stdout, fmt, ..args) }
// eprint formats using the default print settings and writes to os.stderr
eprint :: proc(args: ..any, sep := " ") -> int { return fprint(fd=os.stderr, args=args, sep=sep) }
// eprintln formats using the default print settings and writes to os.stderr
eprintln :: proc(args: ..any, sep := " ") -> int { return fprintln(fd=os.stderr, args=args, sep=sep) }
// eprintf formats according to the specififed format string and writes to os.stderr
eprintf :: proc(fmt: string, args: ..any) -> int { return fprintf(os.stderr, fmt, ..args) }

4
core/hash/crc.odin
View File

@@ -1,8 +1,8 @@
package hash
@(optimization_mode="speed")
crc64_ecma_182 :: proc(data: []byte, seed := u32(0)) -> u64 #no_bounds_check {
result := u64(seed)
crc64_ecma_182 :: proc(data: []byte, seed := u64(0)) -> (result: u64) #no_bounds_check {
result = seed
#no_bounds_check for b in data {
result = result<<8 ~ _crc64_table_ecma_182[((result>>56) ~ u64(b)) & 0xff]
}

10
core/hash/crc32.odin
View File

@@ -9,8 +9,8 @@ crc32 :: proc(data: []byte, seed := u32(0)) -> u32 #no_bounds_check {
length := len(data)
for length != 0 && uintptr(buffer) & 7 != 0 {
crc = crc32_table[0][byte(crc) ~ buffer^] ~ (crc >> 8)
buffer = intrinsics.ptr_offset(buffer, 1)
crc = crc32_table[0][byte(crc) ~ buffer[0]] ~ (crc >> 8)
buffer = buffer[1:]
length -= 1
}
@@ -28,14 +28,14 @@ crc32 :: proc(data: []byte, seed := u32(0)) -> u32 #no_bounds_check {
crc32_table[1][buf[6]] ~
crc32_table[0][buf[7]]
buffer = intrinsics.ptr_offset(buffer, 8)
buffer = buffer[8:]
length -= 8
}
for length != 0 {
crc = crc32_table[0][byte(crc) ~ buffer^] ~ (crc >> 8)
buffer = intrinsics.ptr_offset(buffer, 1)
crc = crc32_table[0][byte(crc) ~ buffer[0]] ~ (crc >> 8)
buffer = buffer[1:]
length -= 1
}

Some files were not shown because too many files have changed in this diff Show More